30_08 changes (refactoring + f_tests)

functional_tests/functions.py

@@ -14,6 +14,7 @@ try:
     import re
     import os
     import os.path as path
+    from pathlib import Path
     import json
     import subprocess
 
@@ -75,20 +76,32 @@ def json_load(step):
 
     return files_list
 
+def files_load(exp_dir, step):
+
+    files_list = []
+    files_dir = path.join(path.abspath(exp_dir), step)
+    for file in Path(files_dir).rglob('*.*'):
+        files_list.append(path.relpath(file, start = files_dir))
+
+    return sorted(files_list)
+
 # Do file comparisons for given step and write output
-def check_files(exp_dir, obs_dir, step, sampleIds):
+def check_files(exp_dir, obs_dir, step):
 
     # Check existence of expected and observed directories
     if not path.exists(exp_dir) or not path.exists(obs_dir):
         sys.exit('{a} or {b} folder(s) do not exist, please check --exp_dir and --obs_dir parameters'.format(a = exp_dir, b = obs_dir))
 
     # Load files and methods list from JSON file
-    files_list = json_load(step)
-    # print files_list, len(files_list)
-    # sys.exit()
+    # files_list = json_load(step)
+
+    # Load files list to test from exp_dir folder
+    files_list = files_load(exp_dir, step)
+    print(files_list)
+    sys.exit()
 
     # Load list of sample ids given by user
-    sampleIds_list = sampleIds.split(',')
+    # sampleIds_list = sampleIds.split(',')
 
     # Initiate log for untested files (removes already existing logs)
     not_tested_log = 'ft_{}.not_tested'.format(step)

functional_tests/main.py

@@ -63,7 +63,7 @@ def main():
             if steps_list[step] <= steps_list[args.step]:
 
                 # Launch test on expected and observed files from this step
-                out = check_files(args.exp_dir, args.obs_dir, step, args.sampleIds)
+                out = check_files(args.exp_dir, args.obs_dir, step)
                 outs.append(out)
 
         print(''.join(outs))

main.nf

@@ -53,7 +53,7 @@
        You need to use --skip_removal_host or --host_fasta or --skip_01_clean_qc. If it is not the case, an error message will occur.
        --skip_kaiju                  Skip taxonomic affiliation of reads with kaiju.
        --kaiju_db_dir                Directory with kaiju database already built ("PATH/directory").
-       --kaiju_db                    Indicate kaiju database you want to build. Use: --kaiju_db ""http://kaiju.binf.ku.dk/database/CHOOSEN_DATABASE.tgz".
+       --kaiju_db                    Indicate kaiju database you want to build. Use: --kaiju_db "http://kaiju.binf.ku.dk/database/CHOOSEN_DATABASE.tgz".
        You need to use --kaiju_db_dir or --kaiju_db or --skip_kaiju. If it is not the case, an error message will occur.
      
      02_assembly options:
@@ -637,6 +637,11 @@ process reads_deduplication {
     samtools sort -n -o ${sampleId}.filtered.sort.bam ${sampleId}.filtered.bam
     bedtools bamtofastq -i ${sampleId}.filtered.sort.bam -fq ${sampleId}_R1_dedup.fastq -fq2 ${sampleId}_R2_dedup.fastq
     gzip ${sampleId}_R1_dedup.fastq ; gzip ${sampleId}_R2_dedup.fastq
+    rm ${sampleId}.sort.bam
+    rm ${sampleId}.fixmate.bam
+    rm ${sampleId}.fixmate.positionsort.bam
+    rm ${sampleId}.filtered.bam
+    rm ${sampleId}.filtered.sort.bam
     """
 }
 

nextflow.config

@@ -32,7 +32,7 @@ params {
 
     // Ressources.
     kaiju_db_dir = false
-    kaiju_db = false
+    kaiju_db = "https://kaiju.binf.ku.dk/database/kaiju_db_refseq_2021-02-26.tgz"
     diamond_bank = ""
     accession2taxid = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/accession2taxid/prot.accession2taxid.gz"
     taxdump = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz"