Commit e438ef55 authored by Penom Nom's avatar Penom Nom
Browse files

add blast filter(python version) in contamination component

add libraries NCBIXML.py and Record.py in src/ng6
parent 7241a6bc
This diff is collapsed.
# Copyright 1999-2000 by Jeffrey Chang. All rights reserved.
# This code is part of the Biopython distribution and governed by its
# license. Please see the LICENSE file that should have been included
# as part of this package.
"""Record classes to hold BLAST output.
Classes:
Blast Holds all the information from a blast search.
PSIBlast Holds all the information from a psi-blast search.
Header Holds information from the header.
Description Holds information about one hit description.
Alignment Holds information about one alignment hit.
HSP Holds information about one HSP.
MultipleAlignment Holds information about a multiple alignment.
DatabaseReport Holds information from the database report.
Parameters Holds information from the parameters.
"""
# XXX finish printable BLAST output
class Header(object):
"""Saves information from a blast header.
Members:
application The name of the BLAST flavor that generated this data.
version Version of blast used.
date Date this data was generated.
reference Reference for blast.
query Name of query sequence.
query_letters Number of letters in the query sequence. (int)
database Name of the database.
database_sequences Number of sequences in the database. (int)
database_letters Number of letters in the database. (int)
"""
def __init__(self):
self.application = ''
self.version = ''
self.date = ''
self.reference = ''
self.query = ''
self.query_letters = None
self.database = ''
self.database_sequences = None
self.database_letters = None
class Description(object):
"""Stores information about one hit in the descriptions section.
Members:
title Title of the hit.
score Number of bits. (int)
bits Bit score. (float)
e E value. (float)
num_alignments Number of alignments for the same subject. (int)
"""
def __init__(self):
self.title = ''
self.score = None
self.bits = None
self.e = None
self.num_alignments = None
def __str__(self):
return "%-66s %5s %s" % (self.title, self.score, self.e)
class Alignment(object):
"""Stores information about one hit in the alignments section.
Members:
title Name.
hit_id Hit identifier. (str)
hit_def Hit definition. (str)
length Length. (int)
hsps A list of HSP objects.
"""
def __init__(self):
self.title = ''
self.hit_id = ''
self.hit_def = ''
self.length = None
self.hsps = []
def __str__(self):
lines = self.title.split('\n')
lines.append("Length = %s\n" % self.length)
return '\n '.join(lines)
class HSP(object):
"""Stores information about one hsp in an alignment hit.
Members:
score BLAST score of hit. (float)
bits Number of bits for that score. (float)
expect Expect value. (float)
num_alignments Number of alignments for same subject. (int)
identities Number of identities (int) if using the XML parser.
Tuple of numer of identities/total aligned (int, int)
if using the (obsolete) plain text parser.
positives Number of positives (int) if using the XML parser.
Tuple of numer of positives/total aligned (int, int)
if using the (obsolete) plain text parser.
gaps Number of gaps (int) if using the XML parser.
Tuple of numer of gaps/total aligned (int, int) if
using the (obsolete) plain text parser.
align_length Length of the alignment. (int)
strand Tuple of (query, target) strand.
frame Tuple of 1 or 2 frame shifts, depending on the flavor.
query The query sequence.
query_start The start residue for the query sequence. (1-based)
query_end The end residue for the query sequence. (1-based)
match The match sequence.
sbjct The sbjct sequence.
sbjct_start The start residue for the sbjct sequence. (1-based)
sbjct_end The end residue for the sbjct sequence. (1-based)
Not all flavors of BLAST return values for every attribute:
score expect identities positives strand frame
BLASTP X X X X
BLASTN X X X X X
BLASTX X X X X X
TBLASTN X X X X X
TBLASTX X X X X X/X
Note: for BLASTX, the query sequence is shown as a protein sequence,
but the numbering is based on the nucleotides. Thus, the numbering
is 3x larger than the number of amino acid residues. A similar effect
can be seen for the sbjct sequence in TBLASTN, and for both sequences
in TBLASTX.
Also, for negative frames, the sequence numbering starts from
query_start and counts down.
"""
def __init__(self):
self.score = None
self.bits = None
self.expect = None
self.num_alignments = None
self.identities = (None, None)
self.positives = (None, None)
self.gaps = (None, None)
self.align_length = None
self.strand = (None, None)
self.frame = ()
self.query = ''
self.query_start = None
self.query_end = None
self.match = ''
self.sbjct = ''
self.sbjct_start = None
self.sbjct_end = None
def __str__(self):
lines = ["Score %i (%i bits), expectation %0.1e, alignment length %i" \
% (self.score, self.bits, self.expect, self.align_length)]
if self.align_length < 50:
lines.append("Query:%s %s %s" % (str(self.query_start).rjust(8),
str(self.query),
str(self.query_end)))
lines.append(" %s" \
% (str(self.match)))
lines.append("Sbjct:%s %s %s" % (str(self.sbjct_start).rjust(8),
str(self.sbjct),
str(self.sbjct_end)))
else:
lines.append("Query:%s %s...%s %s" \
% (str(self.query_start).rjust(8),
str(self.query)[:45],
str(self.query)[-3:],
str(self.query_end)))
lines.append(" %s...%s" \
% (str(self.match)[:45],
str(self.match)[-3:]))
lines.append("Sbjct:%s %s...%s %s" \
% (str(self.sbjct_start).rjust(8),
str(self.sbjct)[:45],
str(self.sbjct)[-3:],
str(self.sbjct_end)))
return "\n".join(lines)
class MultipleAlignment(object):
"""Holds information about a multiple alignment.
Members:
alignment A list of tuples (name, start residue, sequence, end residue).
The start residue is 1-based. It may be blank, if that sequence is
not aligned in the multiple alignment.
"""
def __init__(self):
self.alignment = []
class Round(object):
"""Holds information from a PSI-BLAST round.
Members:
number Round number. (int)
reused_seqs Sequences in model, found again. List of Description objects.
new_seqs Sequences not found, or below threshold. List of Description.
alignments A list of Alignment objects.
multiple_alignment A MultipleAlignment object.
"""
def __init__(self):
self.number = None
self.reused_seqs = []
self.new_seqs = []
self.alignments = []
self.multiple_alignment = None
class DatabaseReport(object):
"""Holds information about a database report.
Members:
database_name List of database names. (can have multiple dbs)
num_letters_in_database Number of letters in the database. (int)
num_sequences_in_database List of number of sequences in the database.
posted_date List of the dates the databases were posted.
ka_params A tuple of (lambda, k, h) values. (floats)
gapped # XXX this isn't set right!
ka_params_gap A tuple of (lambda, k, h) values. (floats)
"""
def __init__(self):
self.database_name = []
self.posted_date = []
self.num_letters_in_database = []
self.num_sequences_in_database = []
self.ka_params = (None, None, None)
self.gapped = 0
self.ka_params_gap = (None, None, None)
class Parameters(object):
"""Holds information about the parameters.
Members:
matrix Name of the matrix.
gap_penalties Tuple of (open, extend) penalties. (floats)
sc_match Match score for nucleotide-nucleotide comparison
sc_mismatch Mismatch penalty for nucleotide-nucleotide comparison
num_hits Number of hits to the database. (int)
num_sequences Number of sequences. (int)
num_good_extends Number of extensions. (int)
num_seqs_better_e Number of sequences better than e-value. (int)
hsps_no_gap Number of HSP's better, without gapping. (int)
hsps_prelim_gapped Number of HSP's gapped in prelim test. (int)
hsps_prelim_gapped_attemped Number of HSP's attempted in prelim. (int)
hsps_gapped Total number of HSP's gapped. (int)
query_length Length of the query. (int)
query_id Identifier of the query sequence. (str)
database_length Number of letters in the database. (int)
effective_hsp_length Effective HSP length. (int)
effective_query_length Effective length of query. (int)
effective_database_length Effective length of database. (int)
effective_search_space Effective search space. (int)
effective_search_space_used Effective search space used. (int)
frameshift Frameshift window. Tuple of (int, float)
threshold Threshold. (int)
window_size Window size. (int)
dropoff_1st_pass Tuple of (score, bits). (int, float)
gap_x_dropoff Tuple of (score, bits). (int, float)
gap_x_dropoff_final Tuple of (score, bits). (int, float)
gap_trigger Tuple of (score, bits). (int, float)
blast_cutoff Tuple of (score, bits). (int, float)
"""
def __init__(self):
self.matrix = ''
self.gap_penalties = (None, None)
self.sc_match = None
self.sc_mismatch = None
self.num_hits = None
self.num_sequences = None
self.num_good_extends = None
self.num_seqs_better_e = None
self.hsps_no_gap = None
self.hsps_prelim_gapped = None
self.hsps_prelim_gapped_attemped = None
self.hsps_gapped = None
self.query_id = None
self.query_length = None
self.database_length = None
self.effective_hsp_length = None
self.effective_query_length = None
self.effective_database_length = None
self.effective_search_space = None
self.effective_search_space_used = None
self.frameshift = (None, None)
self.threshold = None
self.window_size = None
self.dropoff_1st_pass = (None, None)
self.gap_x_dropoff = (None, None)
self.gap_x_dropoff_final = (None, None)
self.gap_trigger = (None, None)
self.blast_cutoff = (None, None)
#TODO - Add a friendly __str__ method to BLAST results
class Blast(Header, DatabaseReport, Parameters):
"""Saves the results from a blast search.
Members:
descriptions A list of Description objects.
alignments A list of Alignment objects.
multiple_alignment A MultipleAlignment object.
+ members inherited from base classes
"""
def __init__(self):
Header.__init__(self)
DatabaseReport.__init__(self)
Parameters.__init__(self)
self.descriptions = []
self.alignments = []
self.multiple_alignment = None
class PSIBlast(Header, DatabaseReport, Parameters):
"""Saves the results from a blastpgp search.
Members:
rounds A list of Round objects.
converged Whether the search converged.
+ members inherited from base classes
"""
def __init__(self):
Header.__init__(self)
DatabaseReport.__init__(self)
Parameters.__init__(self)
self.rounds = []
self.converged = 0
......@@ -26,84 +26,210 @@ from ng6.analysis import Analysis
from weaver.function import ShellFunction,PythonFunction
from weaver.abstraction import Map
from string import join
def extract_random_seq(sequence_files, output_fasta, extract_rate, min_threshold, max_threshold):
def extract_random_seq(input_fastq, output_fasta, extract_rate, min_threshold, max_threshold):
import ng6.seqio as seqio
extract_rate = float(extract_rate)
min_threshold = int(min_threshold)
max_threshold = int(max_threshold)
if extract_rate > 1 or extract_rate <= 0:
from subprocess import Popen, PIPE
if float(extract_rate) > 1 or float(extract_rate) <= 0:
os.sys.stderr.write("[ERROR] : the extract_rate value is not correct ! (Should be between 0 and 1)")
os.sys.exit()
if int(min_threshold) > int(max_threshold):
os.sys.stderr.write("[ERROR] : the threshold values are not correct ! (Minimum threshold is bigger than Maximum threshold)")
os.sys.exit()
# Compute min_threshold, max_threshold, nbsubseq, totalseq
reader = seqio.SequenceReader(sequence_files)
nb_seq, nb_seq_to_extract = 0, 0
for id, desc, seq, qual in reader :
nb_seq += 1
nb_seq_to_extract = int(nb_seq * float(extract_rate))
# Setting the thresholds if necessary
if nb_seq_to_extract > max_threshold:
nb_seq_to_extract = max_threshold
elif nb_seq_to_extract < min_threshold:
nb_seq_to_extract = min_threshold
if nb_seq < nb_seq_to_extract :
nb_seq_to_extract = nb_seq
step = nb_seq / nb_seq_to_extract
count = 0
# Writes result in fasta file
reader = seqio.SequenceReader(sequence_files)
with open(output_fasta, "w") as output:
#Compute min_threshold, max_threshold, nbsubseq, totalseq
cmd=["wc","-l",input_fastq]
p=Popen(cmd,stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
totalseq=int(int(stdout.split()[0])/4)
nbsubseq=int(totalseq * float(extract_rate))
#Setting the thresholds if necessary
if nbsubseq > max_threshold:
nbsubseq = max_threshold
elif nbsubseq < min_threshold :
nbsubseq = min_threshold
if totalseq < min_threshold:
nbsubseq = totalseq
step=int(totalseq / nbsubseq)
count=0
#Writting into the fasta file from fastq file
reader = seqio.SequenceReader(input_fastq)
with open(output_fasta,"w") as output:
for id, desc, seq, qual in reader :
count += 1
if count == step:
count = 0
count+=1
if count==step:
count=0
seqio.writefasta(output, [[id, desc, seq, qual]])
def blast_filter (blast_file, blast_filter_file, pci_default, pcq_default):
import ng6.NCBIXML as NCBIXML
blast_records = NCBIXML.parse(open(blast_file))
blast_output = open(blast_filter_file,"w")
for blast_record in blast_records:
query_subject_info={}
query_subject={}
keep_record = 0
pci_current_list=[]
bits_current_list=[]
for alignment in blast_record.alignments:
no_annot_regions_saved=[]
no_annot_regions_work=[]
nb_no_annot_region=0
for hsp in alignment.hsps:
keep_hsp = 0
if nb_no_annot_region == 0 :
no_annot_regions_saved.append([0,hsp.query_start])
no_annot_regions_saved.append([hsp.query_end,blast_record.query_letters])
nb_no_annot_region=nb_no_annot_region+1
keep_hsp = 1
else :
for [hole_start,hole_end] in no_annot_regions_work :
#for each hole
delete_no_annot_region = 0
# hsp include a no annot region
if hsp.query_start < hole_start and hsp.query_end > hole_end :
delete_no_annot_region = 1
keep_hsp = 1
#hsp start is in a no annot region
if hsp.query_start > hole_start and hsp.query_start < hole_end:
#hsp is in the none annotated region
if (hsp.query_end < hole_end) :
keep_hsp = 1
# adding the 2 new none annotated region to region table
no_annot_regions_saved.append([hole_start,hsp.query_start])
no_annot_regions_saved.append([hsp.query_end,hole_end])
else :
keep_hsp = 1;
# update end of the existing none annotated region
no_annot_regions_saved.append([hole_start,hsp.query_start])
else :
# no annot region include query end
if hsp.query_end > hole_start and hsp.query_end < hole_end :
if hsp.query_start <= hole_start :
keep_hsp = 1;
no_annot_regions_saved.append([hsp.query_end,hole_end])
else :
if delete_no_annot_region == 0 :
no_annot_regions_saved.append([hole_start,hole_end])
if keep_hsp:
perc_identity=float(hsp.identities*100/hsp.align_length)
pci_current_list.append(perc_identity)
bit_score = hsp.bits
bits_current_list.append(bit_score)
no_annot_regions_work=no_annot_regions_saved[:]
no_annot_regions_saved=[]
if nb_no_annot_region > 0 :
#info blast hit recovering
# query id, subject id, % identity, alignment length, mismatches, gap opens, q. start, q. end, s. start, s. end, evalue, bit score
query_id = blast_record.query
subject_id = alignment.hit_id
alignment_length = hsp.align_length
gap_opens = hsp.gaps
mismatches = blast_record.query_letters-gap_opens-perc_identity
q_start = hsp.query_start
q_end = hsp.query_end
s_start = hsp.sbjct_start
s_end = hsp.sbjct_end
evalue = hsp.expect
info = query_id+"$"+subject_id+"$"+str(perc_identity)+"$"+str(alignment_length)+"$"+str(mismatches)+"$"+str(gap_opens)+"$"+str(q_start)+"$"+str(q_end)+"$"+str(s_start)+"$"+str(s_end)+"$"+str(evalue)
key = blast_record.query +"$"+alignment.hit_id
if not query_subject.has_key(key):
query_subject_info[key]=[]
query_subject[key]=[]
hole_sum=0
for [hole_start,hole_end] in no_annot_regions_work :
hole_sum += hole_end - hole_start + 1
pcq = (blast_record.query_length - hole_sum ) *100 / blast_record.query_length
pci_mean = round(float(sum(pci_current_list))/float(len(pci_current_list)),2) if len(pci_current_list) > 0 else float('nan')
bits_mean = round(float(sum(bits_current_list))/float(len(bits_current_list)),2) if len(bits_current_list) > 0 else float('nan')
pci_current_list=[]
query_subject_info[key]=info
if pci_mean >= pci_default and pcq >= pcq_default :
query_subject[key]= bits_mean
max = [0,""]
for key in query_subject :
if query_subject[key] != []:
if query_subject[key] > max[0]:
max[0] = query_subject[key]
max[1] = key
if max[0] != 0 :
list = query_subject_info[max[1]].split("$")
ligne = str("\t".join(list)+"\t"+str(max[0]))
blast_output.write(ligne+"\n")
#print "END "+max[1], max[0]
blast_output.close()
class ContaminationSearch (Analysis):
def define_parameters(self, sequence_files, databank, evalue="1e-5", extract_rate=0.05, min_threshold=20000, max_threshold=1e6, archive_name=None):
self.sequence_files=InputFileList(sequence_files)
self.sub_fasta_files=OutputFileList(self.get_outputs('{basename_woext}_sub.fasta', self.sequence_files))
def define_parameters(self, fastq_files, databank, e_value=1e-5 ,extract_rate=0.05, min_threshold=20000, max_threshold=1e6, pci_default=90, pcq_default=90, archive_name=None):
self.fastq_files=InputFileList(fastq_files, Formats.FASTQ)
self.sub_fasta_files=OutputFileList(self.get_outputs('{basename_woext}_sub.fasta', self.fastq_files))
self.databank = databank
self.evalue = evalue
self.e_value=int(e_value)
self.extract_rate = float(extract_rate)
self.min_threshold = int(min_threshold)
self.max_threshold = int(max_threshold)
self.blast_files=OutputFileList(self.get_outputs('{basename_woext}.blast', self.sequence_files))
self.html_files=OutputFileList(self.get_outputs('{basename_woext}.html', self.sequence_files))
self.krona_stdout=OutputFileList(self.get_outputs('{basename_woext}.stdout', self.sequence_files))
self.pci_default = int(pci_default)
self.pcq_default = int(pcq_default)
self.blast_files=OutputFileList(self.get_outputs('{basename_woext}.blast', self.fastq_files))
self.blast_filter_files=OutputFileList(self.get_outputs('{basename_woext}_filter.blast', self.fastq_files))
self.html_files=OutputFileList(self.get_outputs('{basename_woext}.html', self.fastq_files))
self.krona_stdout=OutputFileList(self.get_outputs('{basename_woext}.stdout', self.fastq_files))
self.archive_name=archive_name
def define_analysis(self):
self.name = "ContaminationSearch"
self.description = "Contamination search against non redundant protein sequences."
self.software = "blastx"
self.options = "-evalue " + self.evalue + " -outfmt 6 -db " + os.path.basename(self.databank) + " ; " + \
"--extract-rate=" + str(self.extract_rate) + " --min-threshold=" + str(self.min_threshold) + \
" --max-threshold=" + str(self.max_threshold)
self.name = "Contamination"
self.description = "Contamination Distribution"
self.software = "blastn"
self.options = str(self.extract_rate)+" "+str(self.min_threshold)+" "+str(self.max_threshold)+" "+self.databank+" "+str(self.pci_default)+" "+str(self.pcq_default)
def post_process(self):
for html_file in self.html_files:
sample = os.path.splitext(os.path.basename(html_file))[0]
self._add_result_element(sample, "html", self._save_file(html_file))
def get_version(self):
cmd = [self.get_exec_path("blastx"),"-version"]
cmd = [self.get_exec_path("blastn"),"-version"]
p = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate()
return stdout.split()[1]
def process(self):
file_extraction_options= join(self.options.split()[0:3])
file_extraction_options= " ".join(self.options.split()[0:3])
file_extraction = PythonFunction(extract_random_seq, cmd_format="{EXE} {IN} {OUT} " + file_extraction_options)
file_extraction = Map(file_extraction, inputs=self.sequence_files, outputs=self.sub_fasta_files)
blastx = ShellFunction(self.get_exec_path("blastx") + " -evalue " + self.evalue + " -outfmt 6 -db " + self.databank + " -query $1 -out $2", cmd_format='{EXE} {IN} {OUT}')
blastx = Map(blastx, self.sub_fasta_files, self.blast_files)
file_extraction = Map(file_extraction, inputs=self.fastq_files, outputs=self.sub_fasta_files)
blastn = ShellFunction(self.get_exec_path("blastn") + " -task megablast -best_hit_overhang 0.25 -best_hit_score_edge 0.25 -evalue 1e-5 -perc_identity 90 -outfmt 5 -db " + self.databank + " -query $1 -out $2", cmd_format='{EXE} {IN} {OUT}')
blastn = Map(blastn, self.sub_fasta_files, self.blast_files)
blast_filter_options = " ".join(self.options.split()[-2:])
blastfilter = PythonFunction(blast_filter, cmd_format= "{EXE} {IN} {OUT} " + blast_filter_options)
blastfilter = Map(blastfilter, self.blast_files, self.blast_filter_files)
kronaImportBLAST = ShellFunction(self.get_exec_path("kronaImportBLAST") + " $1 -o $2 > $3", cmd_format='{EXE} {IN} {OUT}')
kronaImportBLAST = MultiMap(kronaImportBLAST, inputs = self.blast_files, outputs = [self.html_files, self.krona_stdout])
\ No newline at end of file
kronaImportBLAST = MultiMap(kronaImportBLAST, inputs = self.blast_filter_files, outputs = [self.html_files, self.krona_stdout])