From 7d2d49c92848acaae9ade2b87faf87bc0d32c0f2 Mon Sep 17 00:00:00 2001
From: Cedric Cabau <cedric.cabau@inra.fr>
Date: Fri, 23 Mar 2018 09:20:38 +0000
Subject: [PATCH] Fix various bugs
---
trunk/bin/AsmUtils.pm | 119 ++++++++++--------
trunk/bin/GenotoulGridRunner | 8 +-
trunk/bin/Workflow.pm | 2 -
trunk/bin/submit_jobs_files.pl | 2 +-
trunk/bin/write_jobs_files.pl | 9 +-
.../report/lib/js/drap-analysis-templates.js | 2 +-
6 files changed, 74 insertions(+), 68 deletions(-)
diff --git a/trunk/bin/AsmUtils.pm b/trunk/bin/AsmUtils.pm
index 1391c19..6fc59d4 100644
--- a/trunk/bin/AsmUtils.pm
+++ b/trunk/bin/AsmUtils.pm
@@ -64,8 +64,8 @@ sub process_cmd {
sub get_more_recent_file {
my $dir = shift;
- my $pattern = shift;
- my @files = sort { (stat($b))[9] <=> (stat($a))[9] } glob("$dir/$pattern");
+ my $regex = shift;
+ my @files = sort { (stat($b))[9] <=> (stat($a))[9] } grep(/$regex/, glob("$dir/*"));
return shift(@files);
}
@@ -79,6 +79,7 @@ sub set_drap_config {
sub get_drap_config {
my ($dir, $retrieve, $opt) = @_;
+ $dir =~ s|/$||;
local $/;
open(JSON, "$dir/.drap_conf.json") or croak "Can't open file $dir/.drap_conf.json";
my $json = <JSON>;
@@ -217,8 +218,16 @@ sub write_shell {
$file = $file->[0] if (ref($file) eq 'ARRAY');
open(SH, ">$file") or croak "Can't open file $file";
print SH "#!/bin/csh\n" unless ($opt->{step} eq 'dbg' && $opt->{dbg} eq 'oases'); # because oases jobs will be run using the qarray command or separately in local mode
- print SH $opt->{env}->{$opt->{step}.'_env'}."\n" if ($opt->{env}->{$opt->{step}.'_env'}); # get setup env command from drap.cfg for specific step
- print SH $opt->{cmd};
+ if ($opt->{env}->{$opt->{step}.'_env'}) { # get setup env command from drap.cfg for specific step
+ if ($opt->{step} eq 'dbg' && $opt->{dbg} eq 'oases') {
+ map { print SH $opt->{env}->{$opt->{step}.'_env'}."; $_\n" } split("\n", $opt->{cmd});
+ } else {
+ print SH $opt->{env}->{$opt->{step}.'_env'}."\n";
+ print SH $opt->{cmd};
+ }
+ } else {
+ print SH $opt->{cmd};
+ }
close SH;
}
delete $opt->{cmd};
@@ -350,8 +359,8 @@ sub get_more_recent_dir { get_more_recent_file(@_) };
sub preprocess_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j1-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j1-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j1-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j1-.*\.e\d+$');
unless ($opt->{no_trim} || -e "$opt->{outdir}/.trim.over") {
return [0, 1, "$opt->{outdir}/trim.log"];
}
@@ -381,8 +390,8 @@ sub preprocess_complete {
sub dbg_complete {
my $opt = shift;
if ($opt->{dbg} eq 'oases') {
- my $log = get_more_recent_file("$opt->{outdir}/err_log", "j2-*.o*.$opt->{task}");
- my $err = get_more_recent_file("$opt->{outdir}/err_log", "j2-*.e*.$opt->{task}");
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j2-.*\.o\d+\.'.$opt->{task}.'$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j2-.*\.e\d+\.'.$opt->{task}.'$');
return [0, 1, $log] if (count_inside_file($log, "^\\[.+\\] Exporting transcripts to $opt->{dir_list}->[0]/k$opt->{current_kmer}/transcripts.fa") == 0);
return [0, 2, $log] if (count_inside_file($log, "^\\[.+\\] Finished extracting transcripts") == 0);
# An empty transcripts.fa for a specific kmer could not be considerated as an error
@@ -391,8 +400,8 @@ sub dbg_complete {
return [0, 'qacct', get_job_id($log)] if (!$opt->{local} && qacct_status(get_job_id($log), $opt->{task}) == 1);
$opt->{kmer_status}->{$opt->{current_kmer}} = 1;
} else {
- my $log = get_more_recent_file("$opt->{outdir}/err_log", "j2-*.o*");
- my $err = get_more_recent_file("$opt->{outdir}/err_log", "j2-*.e*");
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j2-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j2-.*\.e\d+$');
return [0, 3, $log] if (count_inside_file($log, "^Butterfly assemblies are written to ") == 0);
return [0, 'empty', "$opt->{dir_list}->[0]/Trinity.fasta"] unless (-s "$opt->{dir_list}->[0]/Trinity.fasta");
return [0, 'error', $err] unless (-z $err);
@@ -403,8 +412,8 @@ sub dbg_complete {
sub merge_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j3-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j3-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j3-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j3-.*\.e\d+$');
if ($opt->{dbg} eq 'oases') {
foreach my $kmer_dir (glob("$opt->{dir_list}->[0]/k*")) {
my $kmer = basename($kmer_dir);
@@ -430,8 +439,8 @@ sub merge_complete {
sub clustering_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j4-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j4-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j4-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j4-.*\.e\d+$');
return [0, 'cd-hit', "$opt->{dir_list}->[2]/all_dbg.fa.cd-hit.log"] if (!-e "$opt->{dir_list}->[2]/all_dbg.fa.cd-hit.log" || count_inside_file("$opt->{dir_list}->[2]/all_dbg.fa.cd-hit.log", '^program completed !') == 0);
return [0, 'empty', "$opt->{dir_list}->[2]/all_dbg.fa"] unless (-s "$opt->{dir_list}->[2]/all_dbg.fa");
return [0, 'error', $err] unless (-z $err);
@@ -441,7 +450,7 @@ sub clustering_complete {
sub asm_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j5-*.o*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j5-.*\.o\d+$');
return [0, 'empty', "$opt->{dir_list}->[2]/all_contigs.raw.fa"] unless (-s "$opt->{dir_list}->[2]/all_contigs.raw.fa");
return [0, 'qacct', get_job_id($log)] if (!$opt->{local} && qacct_status(get_job_id($log)) == 1);
return [1];
@@ -449,8 +458,8 @@ sub asm_complete {
sub post_asm_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j6-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j6-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j6-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j6-.*\.e\d+$');
return [0, 'empty', "$opt->{dir_list}->[3]/all_contigs.fa"] unless (-s "$opt->{dir_list}->[3]/all_contigs.fa");
return [0, 'empty', "$opt->{dir_list}->[3]/all_contigs.fa.vecFilter.log"] unless (-e "$opt->{dir_list}->[3]/all_contigs.fa.vecFilter.log");
my $nb_ctgs_pre_filter;
@@ -477,8 +486,8 @@ sub post_asm_complete {
sub rmbt_editing_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j7-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j7-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j7-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j7-.*\.e\d+$');
map { unlink($_) } glob("$opt->{dir_list}->[4]/*_pass.over");
my $rmbt_checking = check_rmbt_directory($opt, $opt->{dir_list}->[4]);
@@ -509,8 +518,8 @@ sub rmbt_editing_complete {
sub rmbt_filtering_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j8-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j8-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j8-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j8-.*\.e\d+$');
my $rmbt_checking = check_rmbt_directory($opt, $opt->{dir_list}->[5]);
return $rmbt_checking unless ($rmbt_checking->[0] == 1);
my $valid_rmbt = 0;
@@ -534,8 +543,8 @@ sub rmbt_filtering_complete {
sub postprocess_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j9-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j9-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j9-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j9-.*\.e\d+$');
unless ($opt->{no_rate}) {
return [0, 'transrate', "$opt->{outdir}/00-ASSEMBLY_RATING"] if (count_inside_file("$opt->{outdir}/00-ASSEMBLY_RATING", 'Writing analysis results to assemblies.csv') == 0);
}
@@ -546,12 +555,12 @@ sub postprocess_complete {
sub reference_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j10-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j10-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j10-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j10-.*\.e\d+$');
my $run_dir = $opt->{tool} eq 'exonerate' ? 6 : 7;
my $tmp_dir = get_more_recent_dir("$opt->{dir_list}->[$run_dir]", "tmp_*");
return [0, 1, $tmp_dir] if (-e $tmp_dir); # runBlat and runExonerate cleans tmp dir when all tasks ended successfully
- my $out_file = get_more_recent_file("$opt->{dir_list}->[$run_dir]", "*.$opt->{tool}.best.tsv");
+ my $out_file = get_more_recent_file("$opt->{dir_list}->[$run_dir]", ".*\\.$opt->{tool}\\.best\\.tsv");
return [0, 'empty', $out_file] unless (-e $out_file);
unless ($opt->{local}) {
foreach my $run_job_id (find_inside_file($log, 'Your job (\d+) .* has been submitted')){
@@ -565,8 +574,8 @@ sub reference_complete {
sub meta_merge_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j1-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j1-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j1-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j1-.*\.e\d+$');
return [0, 'empty', "$opt->{dir_list}->[0]/all_conditions_contigs.fa"] unless (-s "$opt->{dir_list}->[0]/all_conditions_contigs.fa");
return [0, 'error', $err] unless (-z $err);
return [0, 'qacct', get_job_id($log)] if (!$opt->{local} && qacct_status(get_job_id($log)) == 1);
@@ -575,8 +584,8 @@ sub meta_merge_complete {
sub meta_longest_orf_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j2-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j2-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j2-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j2-.*\.e\d+$');
return [0, 'empty', "$opt->{dir_list}->[0]/all_conditions_contigs_longest_orf.faa"] unless (-s "$opt->{dir_list}->[0]/all_conditions_contigs_longest_orf.faa");
return [0, 'error', $err] unless (-z $err);
return [0, 'qacct', get_job_id($log)] if (!$opt->{local} && qacct_status(get_job_id($log)) == 1);
@@ -585,8 +594,8 @@ sub meta_longest_orf_complete {
sub meta_cluster_orf_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j3-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j3-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j3-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j3-.*\.e\d+$');
return [0, 'cd-hit', "$opt->{dir_list}->[0]/cd-hit_orfs.faa.log"] if (!-e "$opt->{dir_list}->[0]/cd-hit_orfs.faa.log" || count_inside_file("$opt->{dir_list}->[0]/cd-hit_orfs.faa.log", '^program completed !') == 0);
return [0, 'empty', "$opt->{dir_list}->[0]/cd-hit_orfs.faa"] unless (-s "$opt->{dir_list}->[0]/cd-hit_orfs.faa");
return [0, 'error', $err] unless (-z $err);
@@ -596,8 +605,8 @@ sub meta_cluster_orf_complete {
sub meta_longest_contig_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j4-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j4-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j4-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j4-.*\.e\d+$');
return [0, 'empty', "$opt->{dir_list}->[1]/cd-hit_contigs.fa"] unless (-s "$opt->{dir_list}->[1]/cd-hit_contigs.fa");
return [0, 'error', $err] unless (-z $err);
return [0, 'qacct', get_job_id($log)] if (!$opt->{local} && qacct_status(get_job_id($log)) == 1);
@@ -606,8 +615,8 @@ sub meta_longest_contig_complete {
sub meta_cluster_contig_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j5-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j5-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j5-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j5-.*\.e\d+$');
return [0, 'cd-hit', "$opt->{dir_list}->[1]/meta_contigs.fa.log"] if (!-e "$opt->{dir_list}->[1]/meta_contigs.fa.log" || count_inside_file("$opt->{dir_list}->[1]/meta_contigs.fa.log", '^program completed !') == 0);
return [0, 'empty', "$opt->{dir_list}->[1]/meta_contigs.fa"] unless (-s "$opt->{dir_list}->[1]/meta_contigs.fa");
return [0, 'error', $err] unless (-z $err);
@@ -617,8 +626,8 @@ sub meta_cluster_contig_complete {
sub meta_index_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j6-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j6-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j6-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j6-.*\.e\d+$');
if ($opt->{mapper} eq 'bwa') {
return [0, 'empty', "$opt->{dir_list}->[1]/meta_contigs.fa.sa"] unless (-s "$opt->{dir_list}->[1]/meta_contigs.fa.sa");
} else {
@@ -642,8 +651,8 @@ sub meta_rmbt_complete {
sub meta_filter_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j8-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j8-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j8-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j8-.*\.e\d+$');
return [0, 1, $err] if (count_inside_file($err, 'Parsing BAM header...') != scalar(@{[glob("$opt->{dir_list}->[2]/*.bam")]}));
return [0, 2, $err] if (count_inside_file($err, 'COMPLETED: Processed \d+ mapped fragments') == 0);
return [0, 3, $err] if (count_inside_file($err, 'transdecoder is finished.') == 0);
@@ -662,8 +671,8 @@ sub meta_filter_complete {
sub meta_postprocess_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j9-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j9-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j9-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j9-.*\.e\d+$');
return [0, 'transrate', "$opt->{outdir}/00-META-ASSEMBLY_RATING"] if (count_inside_file("$opt->{outdir}/00-META-ASSEMBLY_RATING", 'Writing analysis results to assemblies.csv') == 0);
return [0, 'error', $err] unless (-z $err);
return [0, 'qacct', get_job_id($log)] if (!$opt->{check_postprocess} && !$opt->{local} && qacct_status(get_job_id($log)) == 1);
@@ -672,12 +681,12 @@ sub meta_postprocess_complete {
sub meta_reference_complete {
my $opt = shift;
- my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j10-*.o*');
- my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j10-*.e*');
+ my $log = get_more_recent_file("$opt->{outdir}/err_log", 'j10-.*\.o\d+$');
+ my $err = get_more_recent_file("$opt->{outdir}/err_log", 'j10-.*\.e\d+$');
my $run_dir = $opt->{tool} eq 'exonerate' ? 4 : 5;
my $tmp_dir = get_more_recent_dir("$opt->{dir_list}->[$run_dir]", "tmp_*");
return [0, 1, $tmp_dir] if (-e $tmp_dir); # runBlat and runExonerate cleans tmp dir when all tasks ended successfully
- my $out_file = get_more_recent_file("$opt->{dir_list}->[$run_dir]", "*.$opt->{tool}.best.tsv");
+ my $out_file = get_more_recent_file("$opt->{dir_list}->[$run_dir]", ".*\\.$opt->{tool}\\.best\\.tsv");
return [0, 'empty', $out_file] unless (-e $out_file);
unless ($opt->{local}) {
foreach my $run_job_id (find_inside_file($log, 'Your job (\d+) .* has been submitted')){
@@ -738,7 +747,7 @@ sub check_rmbt_directory {
}
} else {
my $rmbt_err_dir = "$rmbt_directory/err_log_$rmbt_job_id";
- my $bam = get_more_recent_file("$rmbt_directory", "*.$rmbt_job_id.bam");
+ my $bam = get_more_recent_file("$rmbt_directory", ".*\\.$rmbt_job_id\\.bam\$");
my $flagstat = $bam ? "$bam.flagstat" : '';
if ($cmd =~ /runSampe\.\d+\.cmd$/) {
$opt->{check_alignR1} = shift(@{[find_inside_file("$rmbt_directory/runSampe.$rmbt_job_id.cmd", '-1 (\S+?)\s')]});
@@ -772,9 +781,9 @@ sub check_align_process {
my $nb_processed_sampe = 0;
my $processed_sampe;
if ($opt->{mapper} eq 'bwa') {
- my $sampe_err = get_more_recent_file($rmbt_err_dir, 'sampe.e*');
- my $aln_1_err = get_more_recent_file($rmbt_err_dir, 'aln.e*.1');
- my $aln_2_err = get_more_recent_file($rmbt_err_dir, 'aln.e*.2');
+ my $sampe_err = get_more_recent_file($rmbt_err_dir, 'sampe\.e\d+$');
+ my $aln_1_err = get_more_recent_file($rmbt_err_dir, 'aln\.e\d+\.1$');
+ my $aln_2_err = get_more_recent_file($rmbt_err_dir, 'aln\.e\d+\.2$');
return [0, "rmbt.2", $sampe_err] if (count_inside_file($sampe_err, '\[main\] Real time:') == 0);
return [0, "rmbt.2", $aln_1_err] if (count_inside_file($aln_1_err, '\[main\] Real time:') == 0);
return [0, "rmbt.2", $aln_2_err] if (count_inside_file($aln_2_err, '\[main\] Real time:') == 0);
@@ -787,7 +796,7 @@ sub check_align_process {
return [0, "rmbt.4", "$sampe_err and $flagstat"] unless ($nb_processed_sampe*2 == $nb_processed_flagstat);
$processed_sampe = $sampe_err;
} else {
- my $final_log = get_more_recent_file(dirname($rmbt_err_dir), "*.$rmbt_job_id.Log.final.out");
+ my $final_log = get_more_recent_file(dirname($rmbt_err_dir), ".*\\.$rmbt_job_id\\.Log\\.final\\.out");
return [0, "rmbt.5", dirname($rmbt_err_dir)] unless ($final_log);
$nb_processed_sampe = shift(@{[find_inside_file($final_log, '^\s*Number of input reads[\s|]+(\d+)')]});
$processed_sampe = $final_log;
@@ -795,14 +804,14 @@ sub check_align_process {
return [0, "rmbt.6", "$processed_sampe and $opt->{check_alignR1}"] unless ($nb_processed_sampe == $nb_alignR1);
return [0, "rmbt.6", "$processed_sampe and $opt->{check_alignR2}"] unless ($nb_processed_sampe == $nb_alignR2);
unless ($opt->{local}) {
- foreach my $runSampe_job_id (find_inside_file(get_more_recent_file($rmbt_err_dir, 'runSampe.*.jid'), '(\d+)')){
+ foreach my $runSampe_job_id (find_inside_file(get_more_recent_file($rmbt_err_dir, 'runSampe\.\d+\.jid'), '(\d+)')){
return [0, 'qacct', $runSampe_job_id] if (qacct_status($runSampe_job_id) == 1);
}
}
} else {
my ($nb_processed_samse, $nb_processed_sai, $processed_samse);
if ($opt->{mapper} eq 'bwa') {
- my $samse_err = get_more_recent_file($rmbt_err_dir, 'samse.e*');
+ my $samse_err = get_more_recent_file($rmbt_err_dir, 'samse\.e\d+$');
return [0, "rmbt.7", $samse_err] unless (count_inside_file($samse_err, '\[main\] Real time:') == 2);
my ($nb_processed_aln, $processed);
open (ERR, $samse_err);
@@ -819,14 +828,14 @@ sub check_align_process {
return [0, "rmbt.9", $samse_err] unless ($nb_processed_samse == $nb_processed_flagstat);
$processed_samse = $samse_err;
} else {
- my $final_log = get_more_recent_file(dirname($rmbt_err_dir), "*.$rmbt_job_id.Log.final.out");
+ my $final_log = get_more_recent_file(dirname($rmbt_err_dir), ".*\\.$rmbt_job_id\\.Log\\.final\\.out");
return [0, "rmbt.5", dirname($rmbt_err_dir)] unless ($final_log);
$nb_processed_samse = shift(@{[find_inside_file($final_log, '^\s*Number of input reads[\s|]+(\d+)')]});
$processed_samse = $final_log;
}
return [0, "rmbt.6", "$processed_samse and $opt->{check_alignR1}"] unless ($nb_processed_samse == $nb_alignR1);
unless ($opt->{local}) {
- my $runSamse_job_id = shift(@{[find_inside_file(get_more_recent_file($rmbt_err_dir, 'runSamse.*.jid'), '(\d+)')]});
+ my $runSamse_job_id = shift(@{[find_inside_file(get_more_recent_file($rmbt_err_dir, 'runSamse\.\d+\.jid'), '(\d+)')]});
return [0, 'qacct', $runSamse_job_id] if (qacct_status($runSamse_job_id) == 1);
}
}
@@ -869,7 +878,7 @@ sub set_env_variables {
if( -e $config_path && -r $config_path ){ # If configuration file exists and if it is readable
my $config = new ConfigFile( $config_path );
- foreach my $section ('ENV','DATABASE','SOFTWARE CONFIGURATION','SGE CONFIGURATION','SGE RESOURCES') {
+ foreach my $section ('ENV','DATABASE','SOFTWARE CONFIGURATION','SCHEDULER CONFIGURATION','SCHEDULER RESOURCES') {
if( defined($config->{$section}) ){
foreach my $key (keys %{$config->{$section}}) {
while ($config->{$section}{$key} =~ /\$(\w+)/) {
diff --git a/trunk/bin/GenotoulGridRunner b/trunk/bin/GenotoulGridRunner
index 71004b9..4641984 100755
--- a/trunk/bin/GenotoulGridRunner
+++ b/trunk/bin/GenotoulGridRunner
@@ -75,10 +75,10 @@ run_on_grid (@all_cmds);
sub run_on_grid {
my @cmds = @_;
- my $std_queue = $ENV{'DEFAULT_QUEUE'} || 'workq';
- my $mem_queue = $ENV{'MEM_QUEUE'} || 'hypermemq';
- my $max_chunk_jobs = $ENV{'TRINITY_MAX_CHUNK_JOBS'} || 100;
- my $node_max_mem = $ENV{'NODE_MAX_MEM'} || 240;
+ my $std_queue = $ENV{'default_queue'} || 'workq';
+ my $mem_queue = $ENV{'mem_queue'} || 'hypermemq';
+ my $max_chunk_jobs = $ENV{'trinity_max_chunk_jobs'} || 100;
+ my $node_max_mem = $ENV{'node_max_mem'} || 240;
my $qarray = 'qarray' ;
`which $qarray 2> /dev/null`;
if ($?) {
diff --git a/trunk/bin/Workflow.pm b/trunk/bin/Workflow.pm
index 00cabf1..6c6d223 100644
--- a/trunk/bin/Workflow.pm
+++ b/trunk/bin/Workflow.pm
@@ -122,8 +122,6 @@ sub add_component {
if (exists $env->{$component_name.'_env'}) {
# Add the env settings command
push (@cmds, new Cmd($env->{$component_name.'_env'}));
- #my @cmd_with_env = ( new Cmd($env->{$component_name.'_env'}), $command );
- #$submission_cmd = new CmdSet( \@cmd_with_env, "serial", "wf_cmpt_".$component_name_clean, SchedulerFactory->instantiate($component_dir) );
}
# Add the achieved file creation
diff --git a/trunk/bin/submit_jobs_files.pl b/trunk/bin/submit_jobs_files.pl
index 55c98f8..c606ffa 100755
--- a/trunk/bin/submit_jobs_files.pl
+++ b/trunk/bin/submit_jobs_files.pl
@@ -56,7 +56,7 @@ foreach my $step (@{$opt->{steps}}) {
$array = 1 if ($opt->{dbg} eq 'oases');
my $index = $#$scripts > 0 ? sprintf("_%d",$i+1) : '';
$reservation = $opt->{env}->{"$opt->{dbg}${index}_res"}||''; # get reservation from drap.cfg for specific step
- my $mem = $opt->{dbg} eq 'oases' ? $opt->{env}->{oases_ram} : ceil($opt->{dbg_mem}/$opt->{env}->{n_cpu});
+ my $mem = $opt->{dbg} eq 'oases' ? $opt->{dbg_mem} ? $opt->{dbg_mem} : $opt->{env}->{oases_ram} : ceil($opt->{dbg_mem}/$opt->{env}->{n_cpu});
$reservation =~ s/\%MEM\%/$mem/;
my $h_vmem = ceil($mem*3/2);
$reservation =~ s/\%H_VMEM\%/$h_vmem/;
diff --git a/trunk/bin/write_jobs_files.pl b/trunk/bin/write_jobs_files.pl
index 843ce42..b9596b1 100755
--- a/trunk/bin/write_jobs_files.pl
+++ b/trunk/bin/write_jobs_files.pl
@@ -68,7 +68,7 @@ unless (step_complete($opt)) {
my $local_restart = $opt->{restart};
if ($opt->{restart}) {
$clean_msg .= clean_directories($opt, 0);
- $opt->{kmer_status} = map { $_, 0 } @{$opt->{kmers}} if (ref($opt->{kmer_status}) eq 'HASH');
+ %{$opt->{kmer_status}} = map { $_, 0 } @{$opt->{kmers}} if (ref($opt->{kmer_status}) eq 'HASH');
$opt->{restart} = 0;
$resubmit = $opt->{step};
}
@@ -481,12 +481,13 @@ unless (-f "$opt->{dir_list}->[2]/all_contigs.raw.fa" && step_complete($opt)) {
."set pwd = `pwd`\n"
."cd $opt->{dir_list}->[2]\n./runAssembly.sh\n"
."cd \$pwd\n";
+ my $tgicl_cpu = $opt->{env}->{n_cpu} > 16 ? 16 : $opt->{env}->{n_cpu}; # tgicl (i.e psx) does not accept more than 16 cpu
open(SRC, "$opt->{binpath}/runAssembly.sh") or croak "Can't open file $opt->{binpath}/runAssembly.sh";
open(SH, ">$opt->{dir_list}->[2]/runAssembly.sh") or croak "Can't open file $opt->{dir_list}->[2]/runAssembly.sh";
while (<SRC>) {
s|BINPATH|$opt->{binpath}|;
s|CFGPATH|$opt->{cfgpath}|;
- s|TGICL_CPU|$opt->{env}->{n_cpu}|;
+ s|TGICL_CPU|$tgicl_cpu|;
print SH;
}
close SH;
@@ -715,7 +716,7 @@ unless (-f "$opt->{dir_list}->[5]/transcripts_fpkm_$lowest_fpkm.fa" && step_comp
$opt->{paired} ? "-1 $opt->{alignR1}->[$i] -2 $opt->{alignR2}->[$i]" : "-f $opt->{alignR1}->[$i]",
$opt->{dir_list}->[5], $opt->{mapper}, $opt->{env}->{n_cpu}, $opt->{filter} ? ' -y' : '', $opt->{local} ? ' --local' : ''
);
- my $flagstat_metrics_file = $report_db_folder."/".$aln_edit_analysis->get_or_create_step($opt->{mapper}." mapping")->get_or_create_metrics_filename('flagstatLog', $fastq);
+ my $flagstat_metrics_file = $report_db_folder."/".$aln_filter_analysis->get_or_create_step($opt->{mapper}." mapping")->get_or_create_metrics_filename('flagstatLog', $fastq);
my $flagstat_metrics_basename = basename($flagstat_metrics_file, '.json');
if ($opt->{local}) {
$opt->{cmd} .= "$opt->{binpath}/alignmentMetrics2json.pl \$bam.flagstat > $flagstat_metrics_file\n";
@@ -739,8 +740,6 @@ unless (-f "$opt->{dir_list}->[5]/transcripts_fpkm_$lowest_fpkm.fa" && step_comp
$opt->{cmd} .= sprintf("%s/cov_length_filter.sh -f %s/all_contigs.second_pass.fa -o %s -x %s/results.xprs -b %s/all_contigs.raw.fa.transdecoder_cutter.orf_length.tsv -t %s -l %d -c %s\n",
$opt->{binpath}, $opt->{dir_list}->[4], $opt->{dir_list}->[5], $opt->{dir_list}->[5], $opt->{dir_list}->[3], $opt->{type}, $opt->{length}, $opt->{optimize} ? $lowest_fpkm : $opt->{coverage}
);
- my $flagstat_metrics_file = $report_db_folder."/".$aln_filter_analysis->get_or_create_step($opt->{mapper}." mapping")->get_or_create_metrics_filename('flagstatLog');
- $opt->{cmd} .= "$opt->{binpath}/alignmentMetrics2json.pl \$bam.flagstat > $flagstat_metrics_file\n";
my $step_description = sprintf("Filter on coverage (fpkm: %s) and length (%s).",
$opt->{coverage}, $opt->{type} eq 'orf' ? "contigs >= $opt->{length} nucleotides" : "contigs with putative orf >= $opt->{length} nucleotides"
);
diff --git a/trunk/report/lib/js/drap-analysis-templates.js b/trunk/report/lib/js/drap-analysis-templates.js
index 4d0ca82..86a26e3 100644
--- a/trunk/report/lib/js/drap-analysis-templates.js
+++ b/trunk/report/lib/js/drap-analysis-templates.js
@@ -305,7 +305,7 @@ var geneSet_table = function( step_data ){
*/
var geneSet_img = function( ){
- var src = '../busco/busco_figure.png';
+ var src = 'database/busco_figure.png';
var width = '50%';
return '<img src="' + src + '" class="img-responsive center-block" width="' + width + '">';
}
--
GitLab