Skip to content

GitLab

Commit 6f920d96 authored by MARTIN Pierre's avatar MARTIN Pierre
Browse files

fix git to dev-test

parent d63977fe
Pipeline #36843 passed with stages
in 16 minutes and 59 seconds
Hide whitespace changes
Inline Side-by-side
conf/base.config
View file @ 6f920d96
......@@ -12,10 +12,11 @@ process {
cpus = { 1 * task.attempt }
memory = { 2.GB * task.attempt }
errorStrategy = { task.exitStatus in [1,143,137,104,134,139] ? 'retry' : 'finish' }
maxRetries = 4
errorStrategy = 'finish'
//{ task.exitStatus in [1,143,137,104,134,139] ? 'retry' : 'finish' }
maxRetries = 1
maxErrors = '-1'
container = 'file://metagwgs/env/metagwgs.sif'
container = '/home/pmartin2/work/metaG/singularity_img/metagwgs.sif'
withName: cutadapt {
cpus = 8
memory = { 8.GB * task.attempt }
......@@ -110,6 +111,9 @@ process {
memory = { 50.GB * task.attempt }
}
withLabel: eggnog {
container = 'file://metagwgs/env/eggnog_mapper.sif'
container = '/home/pmartin2/work/metaG/singularity_img/eggnog_mapper.sif'
}
withLabel: mosdepth {
container = '/home/pmartin2/work/metaG/singularity_img/mosdepth.sif'
}
}
conf/genotoul.config
View file @ 6f920d96
process.executor = 'slurm'
includeConfig 'singularity.config'
singularity.runOptions = "-B /work/bank/ -B /bank -B /work2 -B /work -B /save -B /home -B /work/project"
singularity.runOptions = "-B /work/bank/ -B /bank -B /work2 -B /work -B /save -B /home -B /work/project -B /usr/local/bioinfo"
process.queue = 'workq'
conf/test_genotoul_workq.config
View file @ 6f920d96
process.executor = 'slurm'
includeConfig 'singularity.config'
singularity.runOptions = "-B /work/bank/ -B /bank -B /work2 -B /work -B /save -B /home -B /work/project"
singularity.runOptions = "-B /work/bank/ -B /bank -B /work2 -B /save -B /home -B /work/project -B /usr/local/bioinfo"
  • Celine Noirot @celine.noirot ·
    Owner

    perso j'aurai garder le -B /work ... si je travaille dans le /work/cnoirot ca ne fonctionnera pas il me semble.

Please register or sign in to reply
process.queue = 'workq'
process {
......
env/Singularity_recipe_mosdepth 0 → 100644
View file @ 6f920d96
Bootstrap: docker
From: continuumio/miniconda3
IncludeCmd: yes
%files
env/mosdepth.yml /
%post
apt-get update && apt-get install -y procps && apt-get clean -y
/opt/conda/bin/conda env create -f /mosdepth.yml && /opt/conda/bin/conda clean -a
%environment
export PATH=/opt/conda/envs/mosdepth/bin:$PATH
%runscript
"$@"
env/mosdepth.yml 0 → 100644
View file @ 6f920d96
name: mosdepth
channels:
- bioconda
- conda-forge
- defaults
dependencies:
- mosdepth=0.3.1
\ No newline at end of file
functional_tests/functional_tests.py deleted 100755 → 0
View file @ d63977fe
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Usage
## python 01_clean_qc.py -w [root directory of metagwgs source folder] [optional: --skip_nextflow]
try:
import argparse
import sys
import re
import os
import os.path
import subprocess
import filecmp
except ImportError as error:
print(error)
exit(1)
# Lists of output files to compare
_01_1_cleaned_reads = [
("01_1_cleaned_reads/logs/a.no_filter.flagstat", "diff"),
("01_1_cleaned_reads/logs/a_cutadapt.log", "cut_diff"),
("01_1_cleaned_reads/logs/a_sickle.log", "diff"),
("01_1_cleaned_reads/logs/host_filter_flagstat/a.host_filter.flagstat", "diff"),
("01_1_cleaned_reads/logs/c.no_filter.flagstat", "diff"),
("01_1_cleaned_reads/logs/c_cutadapt.log", "cut_diff"),
("01_1_cleaned_reads/logs/c_sickle.log", "diff"),
("01_1_cleaned_reads/logs/host_filter_flagstat/c.host_filter.flagstat", "diff")
]
_01_2_qc = [
("01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R1_fastqc.html", "not_empty"),
("01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R1_fastqc.zip", "not_empty"),
("01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R2_fastqc.html", "not_empty"),
("01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R1_fastqc.zip", "not_empty"),
("01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R1_fastqc.html", "not_empty"),
("01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R1_fastqc.zip", "not_empty"),
("01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R2_fastqc.html", "not_empty"),
("01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R1_fastqc.zip", "not_empty"),
("01_2_qc/fastqc_raw/a/a_R1_fastqc.html", "not_empty"),
("01_2_qc/fastqc_raw/a/a_R1_fastqc.zip", "not_empty"),
("01_2_qc/fastqc_raw/a/a_R2_fastqc.html", "not_empty"),
("01_2_qc/fastqc_raw/a/a_R2_fastqc.zip", "not_empty"),
("01_2_qc/fastqc_raw/c/c_R1_fastqc.html", "not_empty"),
("01_2_qc/fastqc_raw/c/c_R1_fastqc.zip", "not_empty"),
("01_2_qc/fastqc_raw/c/c_R2_fastqc.html", "not_empty"),
("01_2_qc/fastqc_raw/c/c_R2_fastqc.zip", "not_empty")
]
_01_3_taxonomic_affiliation_reads = [
("01_3_taxonomic_affiliation_reads/a.krona.html", "not_empty"),
("01_3_taxonomic_affiliation_reads/c.krona.html", "not_empty"),
("01_3_taxonomic_affiliation_reads/taxo_affi_reads_class.tsv", "diff"),
("01_3_taxonomic_affiliation_reads/taxo_affi_reads_family.tsv", "diff"),
("01_3_taxonomic_affiliation_reads/taxo_affi_reads_genus.tsv", "diff"),
("01_3_taxonomic_affiliation_reads/taxo_affi_reads_order.tsv", "diff"),
("01_3_taxonomic_affiliation_reads/taxo_affi_reads_phylum.tsv", "diff"),
("01_3_taxonomic_affiliation_reads/taxo_affi_reads_species.tsv", "diff")
]
# List of steps:
steps_list = [
"01_clean_qc",
"02",
"03",
"04",
"05",
"06",
"07",
"08"
]
# Arguments parsing (3 required / 1 optional)
## Usage example:
#
# python functional_tests.py \
# -step 01_clean_qc \
# -exp_dir ../../functional_tests/expected \
# -obs_dir ../../functional_tests/observed/results \
# --script ../../functional_tests/test.sh
def parse_arguments():
parser = argparse.ArgumentParser()
parser.add_argument("-step", type = str, \
help = "(at least one required) step(s) of metagwgs you wish to perform a test on (multiple steps can be tested at once)")
parser.add_argument("-exp_dir", type = str, \
help = "(required) expected logs dir containing logs from a healthy metagwgs workflow")
parser.add_argument("-obs_dir", type = str, \
help = "(required) observed logs dir containing logs from the metagwgs workflow you wish to test")
parser.add_argument("--script", type = str, \
help = "(optional) script file containing metagwgs Nextflow launch command ")
if len(sys.argv) == 1:
parser.print_usage(sys.stderr)
sys.exit(1)
return parser.parse_args()
# Launch nextflow from script if given by user
def launch_nextflow(script):
script_file = os.path.abspath(script)
script_dir = os.path.dirname(script_file)
print("Launching test run with the provided file:\n\n{}\n".format(script_file))
process = subprocess.Popen(["sbatch", "{}".format(script_file)], cwd = script_dir, stdout = subprocess.PIPE, shell=True, executable = '/bin/bash')
output, error = process.communicate()
print("Test run completed")
# Do file comparisons for 01_clean_qc step and write output
def check_01_clean_qc(files_list):
f = open("ft_01_clean_qc.log", "w+")
expected_prefix = "{}/01_clean_qc/".format(os.path.abspath(args.exp_dir))
observed_prefix = "{}/01_clean_qc/".format(os.path.abspath(args.obs_dir))
f.write("Expected directory: {}\nObserved directory: {}\n".format(expected_prefix, observed_prefix))
max_cnt = len(files_list)
true_cnt = 0
false_cnt = 0
print("\nLaunching 01_clean_qc...\n")
for file in files_list:
file_path = file[0]
file_usage = file[1]
expected_path = "{}{}".format(expected_prefix, file_path)
observed_path = "{}{}".format(observed_prefix, file_path)
out = """
------------------------------------------------------------------------------
Path: {}
Test type: {}
""".format(file_path, file_usage)
f.write(out)
print(out)
if not os.path.exists("{}".format(expected_path)) or not os.path.exists("{}".format(observed_path)):
# Make log of non existing files
if not os.path.exists("{}".format(expected_path)):
sys.exit(
"\n{} doesn't exist\n\nPlease check expected/ directory for any missing file\n".format(expected_path)
)
elif not os.path.exists("{}".format(observed_path)):
sys.exit(
"\n{} doesn't exist\n\nPlease check observed/ directory for any missing file\n".format(observed_path)
)
else:
if file_usage == "diff":
test = filecmp.cmp(expected_path, observed_path)
if test:
true_cnt += 1
else:
false_cnt += 1
out = "Test result: {}\n".format(str(test))
f.write(out)
print(out)
elif file_usage == "cut_diff":
# Specifically remove header from cutadapt
command = "diff <(tail -n+6 {}) <(tail -n+6 {})".format(expected_path, observed_path)
process = subprocess.Popen(command, stdout = subprocess.PIPE, shell = True, executable='/bin/bash')
diff_out, error = process.communicate()
if diff_out.decode('ascii') != "":
test = "False : {}".format(diff_out)
false_cnt += 1
elif diff_out.decode('ascii') == "":
test = True
true_cnt += 1
else:
print(error)
out = "Test result: {}\n".format(str(test))
f.write(out)
print(out)
elif file_usage == "not_empty":
test = os.path.getsize(observed_path) > 0
if test:
true_cnt += 1
else:
false_cnt += 1
out = "Test result: {}\n".format(str(test))
print(out)
f.write(out)
else:
sys.exit("\n{} is not valid\n".format(file_usage))
continue
true_perc = (float(true_cnt) / float(max_cnt) * 100)
if false_cnt != 0:
false_perc = 100 - (float(true_cnt) / float(max_cnt) * 100)
else:
false_perc = 0
out = """
=========================================
-----------------------------------------
Testing the 01_clean_qc step of metagWGS:
Total: {}
Passed: {} ({}%)
Missed: {} ({}%)
-----------------------------------------
=========================================
""".format(max_cnt, true_cnt, true_perc, false_cnt, false_perc)
f.write(out)
print(out)
f.close()
# __main__
def main():
global args
args = parse_arguments()
if args.step not in steps_list:
sys.exit("-step doesn't exist, please chose from: {}".format(steps_list))
if args.script:
launch_nextflow(args.script)
if args.step == "01_clean_qc":
check_01_clean_qc(_01_1_cleaned_reads + _01_2_qc + _01_3_taxonomic_affiliation_reads)
else:
print("Step {}, nothing to do yet".format(args.step))
if __name__ == "__main__":
main()
\ No newline at end of file
functional_tests/functions.py 0 → 100644
View file @ 6f920d96
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Pierre MARTIN
# IE Genotoul
# MIAT - INRAe (Toulouse)
# 2021
# Functions of metagwgs functional_tests (main.py)
try:
import argparse
import sys
import re
import os
import os.path
import subprocess
except ImportError as error:
print(error)
exit(1)
# Arguments parsing (3 required / 1 optional)
## Usage example:
#
# python functional_tests.py \
# -step 01_clean_qc \
# -exp_dir ../../functional_tests/expected \
# -obs_dir ../../functional_tests/observed/results \
# --script ../../functional_tests/test.sh
def parse_arguments():
parser = argparse.ArgumentParser()
parser.add_argument('-step', type = str, \
help = '(at least one required) step(s) of metagwgs you wish to perform a test on (multiple steps can be tested at once)')
parser.add_argument('-exp_dir', type = str, \
help = '(required) expected logs dir containing logs from a healthy metagwgs workflow')
parser.add_argument('-obs_dir', type = str, \
help = '(required) observed logs dir containing logs from the metagwgs workflow you wish to test')
parser.add_argument('--script', type = str, \
help = '(optional) script file containing metagwgs Nextflow launch command ')
if len(sys.argv) == 1:
parser.print_usage(sys.stderr)
sys.exit(1)
return parser.parse_args()
# Launch nextflow from script if given by user
### Doesn't work right now, must launch .sh separately and then run functional tests on produced results
def launch_nextflow(script):
script_file = os.path.abspath(script)
script_dir = os.path.dirname(script_file)
print('Launching test run with the provided file:\n\n{}\n'.format(script_file))
process = subprocess.Popen(['sbatch', '{}'.format(script_file)], cwd = script_dir, stdout = subprocess.PIPE, shell=True, executable = '/bin/bash')
output, error = process.communicate()
print('Test run completed')
# Lists of output files to compare
## 01_clean_qc:
global _01_1_cleaned_reads, _01_2_qc, _01_3_taxonomic_affiliation_reads
_01_1_cleaned_reads = [
('01_1_cleaned_reads/logs/a.no_filter.flagstat', 'diff'),
('01_1_cleaned_reads/logs/c.no_filter.flagstat', 'diff'),
('01_1_cleaned_reads/logs/a_cutadapt.log', 'cut_diff'),
('01_1_cleaned_reads/logs/c_cutadapt.log', 'cut_diff'),
('01_1_cleaned_reads/logs/a_sickle.log', 'diff'),
('01_1_cleaned_reads/logs/c_sickle.log', 'diff'),
('01_1_cleaned_reads/logs/host_filter_flagstat/a.host_filter.flagstat', 'diff'),
('01_1_cleaned_reads/logs/host_filter_flagstat/c.host_filter.flagstat', 'diff')
]
_01_2_qc = [
('01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R1_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R1_fastqc.zip', 'not_empty'),
('01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R2_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_cleaned/cleaned_a/cleaned_a_R1_fastqc.zip', 'not_empty'),
('01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R1_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R1_fastqc.zip', 'not_empty'),
('01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R2_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_cleaned/cleaned_c/cleaned_c_R1_fastqc.zip', 'not_empty'),
('01_2_qc/fastqc_raw/a/a_R1_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_raw/a/a_R1_fastqc.zip', 'not_empty'),
('01_2_qc/fastqc_raw/a/a_R2_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_raw/a/a_R2_fastqc.zip', 'not_empty'),
('01_2_qc/fastqc_raw/c/c_R1_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_raw/c/c_R1_fastqc.zip', 'not_empty'),
('01_2_qc/fastqc_raw/c/c_R2_fastqc.html', 'not_empty'),
('01_2_qc/fastqc_raw/c/c_R2_fastqc.zip', 'not_empty')
]
_01_3_taxonomic_affiliation_reads = [
('01_3_taxonomic_affiliation_reads/a.krona.html', 'not_empty'),
('01_3_taxonomic_affiliation_reads/c.krona.html', 'not_empty'),
('01_3_taxonomic_affiliation_reads/taxo_affi_reads_class.tsv', 'diff'),
('01_3_taxonomic_affiliation_reads/taxo_affi_reads_family.tsv', 'diff'),
('01_3_taxonomic_affiliation_reads/taxo_affi_reads_genus.tsv', 'diff'),
('01_3_taxonomic_affiliation_reads/taxo_affi_reads_order.tsv', 'diff'),
('01_3_taxonomic_affiliation_reads/taxo_affi_reads_phylum.tsv', 'diff'),
('01_3_taxonomic_affiliation_reads/taxo_affi_reads_species.tsv', 'diff')
]
## 02_assembly:
global _02_assembly
_02_assembly = [
('logs/a.count_reads_on_contigs.flagstat', 'diff'),
('logs/c.count_reads_on_contigs.flagstat', 'diff'),
('logs/a.count_reads_on_contigs.idxstats', 'diff'),
('logs/c.count_reads_on_contigs.idxstats', 'diff'),
('a_metaspades/a_scaffolds.fasta', 'diff'),
('c_metaspades/c_scaffolds.fasta', 'diff'),
('a_all_contigs_QC/report.tsv', 'diff'),
('c_all_contigs_QC/report.tsv', 'diff'),
('a_all_contigs_QC/report.html', 'not_empty'),
('a_all_contigs_QC/report.html', 'not_empty')
]
## 03_filtering
global _03_filtering
_03_filtering = [
('a_select_contigs_cpm10.fasta', 'diff'),
('c_select_contigs_cpm10.fasta', 'diff'),
('a_discard_contigs_cpm10.fasta', 'diff'),
('c_discard_contigs_cpm10.fasta', 'diff'),
('a_select_contigs_QC/report.tsv', 'diff'),
('c_select_contigs_QC/report.tsv', 'diff'),
('a_select_contigs_QC/report.html', 'not_empty'),
('c_select_contigs_QC/report.html', 'not_empty')
]
## 04_structural_annot
global _04_structural_annot
_04_structural_annot = [
('a.annotated.faa', 'diff'),
('c.annotated.faa', 'diff'),
('a.annotated.ffn', 'diff'),
('c.annotated.ffn', 'diff'),
('a.annotated.fna', 'diff'),
('c.annotated.fna', 'diff'),
('a.annotated.gff', 'diff'),
('c.annotated.gff', 'diff'),
('a_prot.len', 'diff'),
('c_prot.len', 'diff')
]
## 05_alignment
global _05_1_reads_alignment_on_contigs, _05_2_database_alignment
_05_1_reads_alignment_on_contigs = [
('05_1_reads_alignment_on_contigs/a/a_contig.bed', 'diff'),
('05_1_reads_alignment_on_contigs/c/c_contig.bed', 'diff'),
('05_1_reads_alignment_on_contigs/a/a.sort.bam.idxstats', 'diff'),
('05_1_reads_alignment_on_contigs/c/c.sort.bam.idxstats', 'diff'),
('05_1_reads_alignment_on_contigs/a/a.regions.bed.gz', 'gz_diff'),
('05_1_reads_alignment_on_contigs/c/c.regions.bed.gz', 'gz_diff')
]
_05_2_database_alignment = [
('05_2_database_alignment/a/a_aln_diamond.m8', 'diff'),
('05_2_database_alignment/c/c_aln_diamond.m8', 'diff')
]
# Do file comparisons for given step and write output
def check_files(exp_dir, obs_dir, step):
if step == '01_clean_qc':
files_list = _01_1_cleaned_reads + _01_2_qc + _01_3_taxonomic_affiliation_reads
elif step == '02_assembly':
files_list = _02_assembly
elif step == '03_filtering':
files_list = _03_filtering
elif step == '04_structural_annot':
files_list = _04_structural_annot
elif step == '05_alignment':
files_list = _05_1_reads_alignment_on_contigs + _05_2_database_alignment
# elif step == '06_func_annot':
# files_list = _06_func_annot
# elif step == '07_taxo_affi':
# files_list = _07_taxo_affi
# elif step == '08_binning':
# files_list = _08_binning
f = open('ft_{}.log'.format(step), 'w+')
expected_prefix = '{}/{}/'.format(os.path.abspath(exp_dir), step)
observed_prefix = '{}/{}/'.format(os.path.abspath(obs_dir), step)
f.write('Expected directory: {}\nvs\nObserved directory: {}\n'.format(expected_prefix, observed_prefix))
max_cnt = len(files_list)
global true_cnt, false_cnt
true_cnt = 0
false_cnt = 0
print('\nLaunching {}...\n'.format(step))
for file in files_list:
file_path = file[0]
file_usage = file[1]
expected_path = '{}{}'.format(expected_prefix, file_path)
observed_path = '{}{}'.format(observed_prefix, file_path)
# Make log of non existing files
if not os.path.exists('{}'.format(expected_path)):
sys.exit(
'\n{} doesn\'t exist\n\nPlease check expected/ directory for any missing file\n'.format(expected_path)
)
elif not os.path.exists('{}'.format(observed_path)):
sys.exit(
'\n{} doesn\'t exist\n\nPlease check observed/ directory for any missing file\n'.format(observed_path)
)
else:
test, out = test_method(expected_path, observed_path, file_usage)
if test == False:
file_out = '''
------------------------------------------------------------------------------
File: {}
Test method: {}
'''.format(file_path, file_usage)
f.write(file_out)
print(file_out)
f.write(out)
print(out)
else: