ng6workflow.py

#
# Copyright (C) 2015 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import inspect
import os
import re
import sys
import pickle
import datetime
import argparse
import xml.etree.ElementTree as ET
import logging
from jflow.workflow import Workflow
from jflow.utils import display_error_message
from jflow.parameter import *

from ng6.t3MySQLdb import t3MySQLdb
from ng6.project import Project
from ng6.run import Run
from ng6.sample import Sample
from ng6.utils import Utils
from ng6.config_reader import NG6ConfigReader

class BasicNG6Workflow (Workflow):

    def __init__(self, args={}, id=None, function= "process"):
        Workflow.__init__(self, args, id, function)
        self.add_parameter("admin_login", "Who is the project administrator", required = True, type = 'ng6userlogin', display_name="Admin login")

    def add_component(self, component_name, args=[], kwargs={}, component_prefix="default", parent=None, addto="run"):
        logging.getLogger("ng6").debug("addto, logging test 1")
        # first build and check if this component is OK
        if component_name in self.internal_components or component_name in self.external_components:

            if component_name in self.internal_components :
                my_pckge = __import__(self.internal_components[component_name], globals(), locals(), [component_name])
                # build the object and define required field
                cmpt_object = getattr(my_pckge, component_name)()
                cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
                cmpt_object.set_prefix(component_prefix)
                if kwargs: cmpt_object.define_parameters(**kwargs)
                else: cmpt_object.define_parameters(*args)
            # external components
            else :
                cmpt_object = self.external_components[component_name]()
                cmpt_object.output_directory = self.get_component_output_directory(component_name, component_prefix)
                cmpt_object.set_prefix(component_prefix)
                # can't use positional arguments with external components
                cmpt_object.define_parameters(**kwargs)
            # if the built object is an analysis
            for derived_class in cmpt_object.__class__.__bases__:
                if derived_class.__name__ == "Analysis":
                    # add parent and project/run information
                    cmpt_object.parent = parent
                    if addto == "project":
                        cmpt_object.project = self.project
                        cmpt_object.space_id = cmpt_object.project.space_id
                    elif addto == "run":
                        cmpt_object.run = self.runobj
                        #We replace the default space_id 
                        if cmpt_object.space_id == "default":
                            cmpt_object.space_id = cmpt_object.run.space_id
                    
                    # link analysis with ots create user
                    cmpt_object.admin_login = self.admin_login
            # there is a dynamic component
            if cmpt_object.is_dynamic():
                self.dynamic_component_present = True
                # if already init, add the component to the list and check if weaver should be executed
                if self.component_nameids_is_init:
                    # add the component
                    self.components_to_exec.append(cmpt_object)
                    self.components.append(cmpt_object)
                    self._execute_weaver()
                    # update outputs
                    for output in cmpt_object.get_dynamic_outputs():
                        output.update()
                else:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " +
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.component_nameids[cmpt_object.get_nameid()] = None
                    self.components_to_exec = []
                    self.components = []
            else:
                if self.component_nameids_is_init:
                    # add the component
                    self.components_to_exec.append(cmpt_object)
                    self.components.append(cmpt_object)
                elif not self.component_nameids_is_init and not self.dynamic_component_present:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " +
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.components_to_exec.append(cmpt_object)
                    self.components.append(cmpt_object)
                else:
                    if self._component_is_duplicated(cmpt_object):
                        raise ValueError("Component " + cmpt_object.__class__.__name__ + " with prefix " +
                                            cmpt_object.prefix + " already exist in this pipeline!")
                    self.component_nameids[cmpt_object.get_nameid()] = None

            return cmpt_object
        else:
            raise ImportError(component_name + " component cannot be loaded, available components are: {0}".format(
                                           ", ".join(list(self.internal_components.keys()) + list(self.external_components.keys()))))

    def pre_process(self):
        t3mysql = t3MySQLdb()
        infos = t3mysql.get_user_info(self.admin_login)
        if infos['email'] :
            self.set_to_address(infos['email'])


class DownloadWorkflow(Workflow):
    """
        Base class for download workflows classes
    """
    def __init__(self, args={}, id=None, function= "process"):
        Workflow.__init__(self, args, id, function)
        self.add_parameter("admin_login", "Who is the project administrator", type = 'ng6userlogin', display_name="Admin login")
        self.add_parameter_list('data_id', 'Ids of a run from which rawdata will be retrieved', type = 'existingrun')
        self.add_parameter_list('run_id', 'Ids of run from which all data will be retrieved', type = 'existingrun')
        self.add_parameter_list('analysis_id', 'Ids of analysis to retrieve', type = 'existinganalysis')

class NG6Workflow (BasicNG6Workflow):

    def __init__(self, args={}, id=None, function= "process"):
        BasicNG6Workflow.__init__(self, args, id, function)
        self._add_run_parameters()
        self.__add_sample_parameters__()

        self.samples = []
        self.reads1 = []
        self.reads2 = []
        self.readsi = []
        self.samples_names = []
        self.reads1_indexes = []
        self.reads2_indexes = []

    def _add_run_parameters(self):
        self.add_parameter("project_name", "The project name the run belongs to", required = True, type = 'existingproject', group="Run information")
        self.add_parameter("run_name", "Give a name to your run", flag = "--name", required = True, display_name="Name", group="Run information")
        self.add_parameter("run_description", "Give a description to your run", flag = "--description", required = True, display_name="Description", group="Run information")
        self.add_parameter("run_date", "When were the data produced", flag = "--date", required = True, type = 'date', display_name="Date", group="Run information")
        self.add_parameter("data_nature", "Are Sequences cDNA, genomique, RNA, ...", required = True, display_name="Data nature", group="Run information")
        self.add_parameter("sequencer", "Which sequencer produced the data", required = True, display_name="Sequencer", group="Run information")
        self.add_parameter("species", "Which species has been sequenced", required = True, display_name="Species", group="Run information")
        self.add_parameter("run_type", "What type of data is it (1 lane, 1 region)", flag = "--type", required = True, display_name="Type", group="Run information")
        self.add_parameter("align_subset_reads", "Align only on subset reads", type=bool, default = False)

    def __add_sample_parameters__(self):
        self.add_multiple_parameter_list("input_sample", "Definition of a sample", flag="--sample",  group="Sample description") # required = True, # TO CHECK casavaWorkflow required not if casava dir
        self.add_parameter("sample_id", "The uniq identifier of the sample", type="nospacestr", add_to = "input_sample")
        self.add_parameter("sample_name", "A descriptive name for the sample", type="nospacestr", add_to = "input_sample")
        self.add_parameter("sample_description", "A brief description of the sample", add_to = "input_sample")
        self.add_parameter("type", "Read orientation and type", choices = Sample.AVAILABLE_TYPES, default='se', add_to = "input_sample")
        self.add_parameter("insert_size", "Insert size for paired end reads", type = int, add_to = "input_sample")
        self.add_parameter("species", "Species related to this sample", add_to = "input_sample")
        self.add_parameter_list("metadata", "Add metadata to the sample", type='samplemetadata'  ,add_to = "input_sample")
        self.add_input_file_list("read1", "Read 1 data file path", required = True, add_to = "input_sample")
        self.add_input_file_list("read2", "Read 2 data file path", add_to = "input_sample")
        self.add_input_file_list("index",  "Index data file path", add_to = "input_sample")

    def __create_samples__(self):
        for sd in self.input_sample :
            sp_object = Sample( sd['sample_id'], sd['read1'], sd['read2'], sd['index'], name = sd['sample_name'], description = sd['sample_description'], type = sd['type'],
                               insert_size = sd['insert_size'], species = sd['species'] )

            for metadata in sd['metadata'] :
                k, v = metadata.split(':', 2)
                sp_object.add_metadata(k, v)
            self.samples.append(sp_object)

    def __preprocess_samples__(self):
        samples_ids = []
        pidx = 1
        nidx = 1
        for rang, sample in enumerate(self.samples) :
            print(sample)
            print("sample readsi")
            print(sample.readsi)
            if sample.name :
                self.samples_names.append(sample.name)
            else :
                sample.name = 'SAMPLE_' + str(nidx)
                nidx += 1

            if sample.sample_id :
                if sample.sample_id in samples_ids :
                    display_error_message( "Sample identifier %s must be uniq ( sample %s ).\n" % sample.sample_id, sample.name )
            # add an automatic id for samples
            else :
                sample.sample_id = 'sample_' + str(pidx)
                pidx += 1
            samples_ids.append(sample.sample_id)

            for rfile in sample.reads1 :
                self.reads1_indexes.append(sample.sample_id)
                self.reads1.append(rfile)

            for rfile in sample.reads2 :
                self.reads2_indexes.append(sample.sample_id)
                self.reads2.append(rfile)
            
            for rfile in sample.readsi :
                self.readsi.append(rfile)

        if len(self.samples_names) != 0 :
            if len(self.samples_names) != len (self.samples) :
                display_error_message( "All samples must have a defined sample name" )

    def get_all_reads(self, type = None):
        if type == 'read1' :
            return self.reads1
        elif type == 'read2' :
            return self.reads2
        elif type == 'index' : 
            return self.readsi
        return self.reads1 + self.reads2 + self.readsi

    def get_files_index(self, type = None):
        if type == 'read1' :
            return self.reads1_indexes
        elif type == 'read2' :
            return self.reads2_indexes
        return self.reads1_indexes + self.reads2_indexes

    def is_paired_end(self):
        return len(self.reads2) > 0

    def pre_process(self):
        BasicNG6Workflow.pre_process(self)
        # start from an existing project
        self.project = Project.get_from_name(self.project_name)
        self.metadata.append("project_id="+str(self.project.id))
        # if user is not allowed to add data on project (is not admin)
        if self.project is not None and not self.project.is_admin(self.admin_login):
            display_error_message( "The user login '" + self.admin_login + "' is not allowed to add data on project '" + self.project.name + "'.\n" )

        # build the run
        self.runobj = Run(self.run_name, self.run_date, self.species, self.data_nature,
                          self.run_type, self.run_description, self.sequencer, self.project.space_id)
        self.runobj.admin_login=self.admin_login
        # then add the run to the project
        self.project.add_run(self.runobj)
        self.metadata.append("run_id="+str(self.runobj.id))

        self.__create_samples__()
        self.__preprocess_samples__()
        # add samples to run
        self.runobj.add_samples(self.samples)

    def post_process(self):
        # once everything done, sync directories
        logging.getLogger("ng6").debug("post_process enter")
        if self.runobj:
            logging.getLogger("ng6").debug("post_process enter self.runobj.sync")
            self.runobj.sync()
        elif self.project:
            logging.getLogger("ng6").debug("post_process enter self.project.sync")
            self.project.sync()


def get_files_from_casava(casava_directory, project_name, lane_number):
    """
        Retrieve all fastq files of a specific project and lane number from a given casava directory
        @param casava_directory : path to CASAVA output directory
        @param project_name : project name
        @param lane_number : lane number
    """

    def bcl2fastq_18(directory, pname, lane):
        """bcl2fastq <= 1.8"""
        files = []
        with open(os.path.join(directory, "SampleSheet.mk")) as fh :
            subdirs_list = []
            for line in fh :
                if line.startswith("l" + str(lane) + "_SUBDIRS"):
                    parts = line.strip().split(":=")
                    subdirs_list = parts[1].split(" ")
            # parse samples
            for subdir in subdirs_list:
                # filter on project name
                if re.match("Project_" + pname + "/Sample_.+", subdir) or subdir.startswith("Undetermined_indices"):
                    for file in os.listdir(directory + "/" + subdir):
                        filepath = directory + "/" + subdir + "/" + file
                        if file.endswith(".fastq.gz") and re.search(".*_L00" + str(lane) + "_.*", file):
                            files.append(filepath);
        return files
        
    def bcl2fastq_10X(directory, pname, lane):
        """longranger"""
        files = []
        with open(os.path.join(directory, "SampleSheet_10X.mk")) as fh :
            subdirs_list = []
            for line in fh :
                if line.startswith("l" + str(lane) + "_SUBDIRS"):
                    parts = line.strip().split(":=")
                    subdirs_list = parts[1].split(" ")
            # parse samples
            for subdir in subdirs_list:
                # filter on project name
                if re.match("Project_" + pname + "/Sample_.+", subdir) or subdir.startswith("Undetermined_indices"):
                    for file in os.listdir(directory + "/" + subdir):
                        filepath = directory + "/" + subdir + "/" + file
                        if file.endswith(".fastq.gz") and re.search(".*_L00" + str(lane) + "_.*", file):
                            files.append(filepath);
        return files

    def bcl2fastq_216(directory, pname, lane):
        """bcl2fastq >= 1.9"""
        files = []
        tree = ET.parse( os.path.join( directory, 'Stats', 'DemultiplexingStats.xml'))
        root = tree.getroot()
        project = root.find(".//Project[@name='%s']"%pname)

        if project is not None :
            project_files = os.listdir(directory + "/" + pname)

            for sample in project.findall("./Sample") :
                if sample.get('name') != 'all' :
                    for barcode in sample.findall('./Barcode'):
                        if barcode.get('name') != 'all':
                            lnum = int(barcode.find('Lane').get('number'))
                            if lnum == lane :
                                fileregexp = '%s_S\d_L%03d_'%(sample.get('name'), lnum)
                                for pfile in project_files :
                                    if re.match(fileregexp,pfile) :
                                        files.append(os.path.join(directory,pname,pfile))
        return files

    if os.path.exists(os.path.join(casava_directory, "SampleSheet.mk")) :
        return bcl2fastq_18(casava_directory, project_name, lane_number)
    elif os.path.exists(os.path.join( casava_directory, 'Stats', 'DemultiplexingStats.xml')) :
        return bcl2fastq_216(casava_directory, project_name, lane_number)
    elif os.path.exists(os.path.join(casava_directory, "SampleSheet_10X.mk")) :
        return bcl2fastq_10X(casava_directory, project_name, lane_number)



class CasavaNG6Workflow(NG6Workflow):

    def __init__(self, args={}, id=None, function= "process"):
        NG6Workflow.__init__(self, args, id, function)

        self.group_prefix = None
        self.undetermined_reads1 = []
        self.undetermined_reads2 = []
        self.undetermined_index = []
        self.log_files = []
        self.is_casava = False
        self.is_10Xcasava = False

    def __add_sample_parameters__(self):
        self.add_multiple_parameter('casava', 'Provide the options to retrieve samples from a CASAVA directory', group="Sample description")
        self.add_input_directory("directory", "Path to the CASAVA directory to use", required=True, get_files_fn=get_files_from_casava, add_to="casava" )
        self.add_parameter("lane", "The lane number to be retrieved from the casava directory",  type='int', add_to="casava") #required=True,
        self.add_parameter('project', 'The name of the project to retrieve in casava directory. The default name is the name of the nG6  project',add_to="casava")
        self.add_parameter('mismatch_index', 'Set this value to true if the index sequence in the sample fastq files allows at least 1 mismatch',
                           type ='bool', add_to="casava")
        self.add_parameter_list('select_sample_id', 'The ids of the sample that will be selected in the SampleSheet.mk file. By default all samples are selected.', add_to="casava",
                           display_name = 'Selected samples')

        NG6Workflow.__add_sample_parameters__(self)

        #TODO exclude self.add_exclusion_rule("casava", "input_sample")

        self.add_parameter("compression", "How should the data be compressed once archived", choices= [ "none", "gz", "bz2"], default = "none")
        self.add_parameter("keep_reads", "Keep or discard reads which pass the illumina filter. 'all' option will keep all reads", flag = "--keep",
                           choices=[ "pass_illumina_filters", "not_pass_illumina_filters", "all"], default = "pass_illumina_filters")
        self.add_input_file_list("contamination_databank", "Which databank should be used to seek contamination (as to be phiX databank indexed for bwa)")
        self.add_parameter("no_group", "Disables grouping of bases for reads >50bp", type=bool, default = True)


    def __create_samples__(self):
        """
            Create samples object from a casava directory if provided
            @param casava_directory : path to CASAVA output directory
            @param lane_number : files in each sample are sequenced on this lane
        """
        logging.getLogger("ng6").debug("CasavaNG6Workflow.__create_samples__ entering")

        if self.casava and self.casava["directory"] and self.casava["lane"] :
            logging.getLogger("ng6").debug("CasavaNG6Workflow.__create_samples__ self.casava")
            self.is_casava = True
            casava_directory = self.casava["directory"]
            lane_number = self.casava["lane"]
            all_samples, all_samples_id = [], []

            # get the casava project_name
            if self.casava["project"] :
                project_name = self.casava["project"]
            else :
                project_name = self.project_name

            project_name = project_name.replace(" ", "_")
            input_files = casava_directory.get_files( project_name, lane_number)
            logging.getLogger("ng6").debug("CasavaNG6Workflow.__create_samples__ input_files = " + ",".join(input_files))
            if len(input_files) == 0 :
                raise Exception("Error while parsing casava directory %s, invalid project name '%s' for lane %s"% (casava_directory, project_name, lane_number))

            all_samples, all_samples_id = [], []
            if os.path.exists(os.path.join(casava_directory, "SampleSheet.mk")) :

                logging.getLogger("ng6").debug("CasavaNG6Workflow.__create_samples__ before self._process_casava_18")
                all_samples, all_samples_id = self._process_casava_18(casava_directory, project_name, lane_number, input_files)
                logging.getLogger("ng6").debug("CasavaNG6Workflow.__create_samples__ before self._process_casava_18")

            elif os.path.exists(os.path.join( casava_directory, 'Stats', 'DemultiplexingStats.xml')) :
                all_samples, all_samples_id = self._process_casava_216(casava_directory, project_name, lane_number, input_files)

            elif os.path.exists(os.path.join( casava_directory, "SampleSheet_10X.mk")):
                logging.getLogger("ng6").debug("CasavaNG6Workflow.__create_samples__ before self._process_casava_10X")
                all_samples, all_samples_id = self._process_casava_10X(casava_directory, project_name, lane_number, input_files)
                self.is_10Xcasava = True
                logging.getLogger("ng6").debug("CasavaNG6Workflow.__create_samples__ after self._process_casava_10X")

            selected_samples = self.casava['select_sample_id']
            logging.getLogger("CasavaNG6Workflow").debug("__create_samples__. all_samples_id = "+", ".join(all_samples_id))
            if selected_samples :
                for sid in selected_samples :
                    assert sid in all_samples_id , "The sample id %s is not in the SampleSheet.mk" % sid

            for sample in all_samples :
                if selected_samples :
                    if sample.name in selected_samples :
                        self.samples.append(sample)
                else :
                    self.samples.append(sample)
        else :
            NG6Workflow.__create_samples__(self)

    def __preprocess_samples__(self):
        NG6Workflow.__preprocess_samples__(self)

        if self.is_casava:
            self.group_prefix = list((Utils.get_group_basenames(self.get_all_reads(), "read")).keys())

    def _process_casava_18(self, casava_directory, project_name, lane_number, input_files):
        logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 enter")
        logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 casava_directory = " + casava_directory + ", project_name = " + str(project_name))
        """
            Creates samples from casavadir (<=1.8) using input files
            @param casava_directory:
            @param project_name:
            @param lane_number:
            @param input_files:
        """
        all_samples = []
        all_samples_id = []

        # open casava samplesheet again to associate our files with a sample
        with open(os.path.join(casava_directory, "SampleSheet.mk")) as fh :
            logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 SampleSheet.mk exists")
            barcodes_list = []
            sample_ids_list = []
            subdirs_list = []
            for line in fh :
                if line.startswith("l" + str(lane_number) + "_BARCODES"):
                    parts = line.strip().split(":=")
                    barcodes_list = [ re.sub( r"[-_\s]+", "", x) for x in  parts[1].split() ]
                elif line.startswith("l" + str(lane_number) + "_SAMPLEIDS" ):
                    parts = line.strip().split(":=")
                    sample_ids_list = parts[1].split(" ")
                elif line.startswith("l" + str(lane_number) + "_SUBDIRS"):
                    parts = line.strip().split(":=")
                    subdirs_list = parts[1].split(" ")

            assert  len(barcodes_list) == len(sample_ids_list) == len(subdirs_list), "Invalid lane {0} in SampleSheet.mk".format(lane_number)
            logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 SampleSheet.mk parsed")
            # parse samples
            for i in range(len(barcodes_list)):
                sample = {
                    'barcode'                   : barcodes_list[i],
                    'sample_id'                 : sample_ids_list[i],
                    'subdir'                    : subdirs_list[i],
                    'reads1'                    : [],
                    'reads2'                    : [],
                    'readsi'                    : []
                }

                # filter on project name
                if re.match("Project_" + project_name + "/Sample_.+", sample['subdir']) or sample['subdir'].startswith("Undetermined_indices"):
                    for file in os.listdir(casava_directory + "/" + sample['subdir']):
                        filepath = casava_directory + "/" + sample['subdir'] + "/" + file
                        if file.endswith(".fastq.gz") and re.search(".*_L00" + str(lane_number) + "_.*", file):
                            for idx, iofile in enumerate(input_files) :
                                if iofile == filepath :
                                    if re.search(".*_R1_.*", file):
                                        if not sample['subdir'].startswith("Undetermined_indices"):
                                            sample['reads1'].append(iofile)
                                        else:
                                            self.undetermined_reads1.append(iofile)
                                    if re.search(".*_R2_.*", file):
                                        if not sample['subdir'].startswith("Undetermined_indices"):
                                            sample['reads2'].append(iofile)
                                        else:
                                            self.undetermined_reads2.append(iofile)
                                    input_files.pop(idx)
                                    break

                    if not sample['subdir'].startswith("Undetermined_indices") :
                        logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 create sample " + sample['sample_id'])
                        sp_object = Sample(sample['barcode'], sample['reads1'], reads2 = sample['reads2'], readsi = [], name=sample['sample_id'])
                        sp_object.add_metadata('barcode', sample['barcode'])
                        sp_object.add_metadata('is_casava', True)

                        all_samples.append(sp_object)
                        all_samples_id.append(sample['sample_id'])
            for file in os.listdir(casava_directory):
                filepath = casava_directory + "/" + file
                if file.endswith(".log"):
                    self.log_files.append(filepath)
            logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 self.log_files = " + ",".join(self.log_files))
            logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 all_samples_id = " + ",".join(all_samples_id))
            logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_18 exiting")
            
        return all_samples, all_samples_id

    def _process_casava_10X(self,casava_directory, project_name, lane_number, input_files):
        logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_10X enter")
        logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_10X casava_directory = " + casava_directory + ", project_name = " + str(project_name))
        """
            Creates samples from casavadir from longranger demultiplexing
            @param casava_directory:
            @param project_name:
            @param lane_number:
            @param input_files:
        """
        all_samples = []
        all_samples_id = []

        # open casava samplesheet again to associate our files with a sample
        with open(os.path.join(casava_directory, "SampleSheet_10X.mk")) as fh :
            barcodes_list = []
            sample_ids_list = []
            subdirs_list = []
            for line in fh :
                if line.startswith("l" + str(lane_number) + "_BARCODES"):
                    parts = line.strip().split(":=")
                    barcodes_list = [ re.sub( r"[_\s]+", "", x) for x in  parts[1].split() ]
                elif line.startswith("l" + str(lane_number) + "_SAMPLEIDS" ):
                    parts = line.strip().split(":=")
                    sample_ids_list = parts[1].split(" ")
                elif line.startswith("l" + str(lane_number) + "_SUBDIRS"):
                    parts = line.strip().split(":=")
                    subdirs_list = parts[1].split(" ")

            assert  len(barcodes_list) == len(sample_ids_list) == len(subdirs_list), "Invalid lane {0} in SampleSheet_10X.mk".format(lane_number)

            cfg_reader = NG6ConfigReader()
            indexs = cfg_reader.get_10X_indexs()
            
            # parse samples
            for i in range(len(barcodes_list)):
                
                if barcodes_list[i] == 'Undetermined' :
                    barcode = 'Undetermined'
                else : 
                    barcode = indexs[barcodes_list[i]]

                sample = {
                    'barcode'                   : barcode,
                    'sample_id'                 : sample_ids_list[i],
                    'subdir'                    : subdirs_list[i],
                    'reads1'                    : [],
                    'reads2'                    : [],
                    'readsi'                    : []
                }
                
                # filter on project name

                if re.match("Project_" + project_name + "/Sample_.+", sample['subdir']) or sample['subdir'].startswith("Undetermined_indices"):

                    for file in os.listdir(casava_directory + "/" + sample['subdir']):

                        filepath = casava_directory + "/" + sample['subdir'] + "/" + file
                        
                        if file.endswith(".fastq.gz") and re.search(".*_L00" + str(lane_number) + "_.*", file):
                            for idx, iofile in enumerate(input_files) :
                                if iofile == filepath :

                                    if re.search(".*_R1_.*", file):

                                        if not sample['subdir'].startswith("Undetermined_indices"):
                                            sample['reads1'].append(iofile)
                                        else:
                                            self.undetermined_reads1.append(iofile)
                                    if re.search(".*_R2_.*", file):
                                        if not sample['subdir'].startswith("Undetermined_indices"):
                                            sample['reads2'].append(iofile)
                                        else:
                                            self.undetermined_reads2.append(iofile)
                                    if re.search(".*_I1_.*", file):
                                        if not sample['subdir'].startswith("Undetermined_indices"):
                                            sample['readsi'].append(iofile)
                                        else:
                                            self.undetermined_index.append(iofile)
                                    input_files.pop(idx)
                                    break

                    if not sample['subdir'].startswith("Undetermined_indices") :
                        sp_object = Sample(sample['barcode'], sample['reads1'], reads2 = sample['reads2'], readsi = sample['readsi'], name=sample['sample_id'])
                        sp_object.add_metadata('barcode', sample['barcode'])
                        sp_object.add_metadata('is_casava', True)

                        all_samples.append(sp_object)
                        all_samples_id.append(sample['sample_id'])
            for file in os.listdir(casava_directory):
                filepath = casava_directory + "/" + file

                if file.endswith(".log"):
                    self.log_files.append(filepath)
            logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_10X all_samples_id = " + ",".join(all_samples_id))
            logging.getLogger("ng6").debug("CasavaNG6Workflow._process_casava_10X exiting")
            
        return all_samples, all_samples_id


    def _process_casava_216(self,casava_directory, project_name, lane_number, input_files):
        """
            Creates samples from casavadir (>=1.9) using input files
            @param casava_directory:
            @param project_name:
            @param lane_number:
            @param input_files:
        """
        raise NotImplementedError

    def illumina_process(self):
        fastqilluminafilter = None
        concatenatefastq = None
        filtered_read1_files = []
        filtered_read2_files = []
        saved_files = []
        logging.getLogger("ng6").debug("illumina_process entering")
        if self.is_casava :
            logging.getLogger("ng6").debug("illumina_process self.is_casava")
            if len(self.log_files) > 0 :
                add_log = self.add_component("BasicAnalysis", [self.log_files,"Log Files","Log files generated during primary analysis","-","-","-","gz", "","log.gz"])
            
            if len(self.undetermined_reads1) > 0 :
                if self.casava['mismatch_index'] :
                    demultiplex_stats = self.add_component("DemultiplexStats", [self.get_all_reads("read1"), self.undetermined_reads1, self.get_files_index('read1')])
                elif self.is_10Xcasava :
                    demultiplex_stats = self.add_component("Demultiplex10XStats", [self.get_all_reads("read1"), self.undetermined_reads1, self.get_files_index("read1")])
                else :
                    demultiplex_stats = self.add_component("DemultiplexStats", [self.get_all_reads("read1"), self.undetermined_reads1])                 


            if self.keep_reads != "all" :
                logging.getLogger("ng6").debug("illumina_process self.keep_reads != all")
                logging.getLogger("ng6").debug("illumina_process BEFORE FASTQILLUMINAFILTER self.get_all_reads() = " + ",".join(self.get_all_reads()))
                logging.getLogger("ng6").debug("illumina_process self.group_prefix = " + ",".join(self.group_prefix))
                # fastq illumina filter
                fastqilluminafilter = self.add_component("FastqIlluminaFilter", [self.runobj,self.get_all_reads(), self.keep_reads, self.group_prefix])

                
                # list filtered files
                if self.is_paired_end() :
                    # split read 1 and read 2 from filtered files list
                    [filtered_read1_files, filtered_read2_files] = Utils.split_pair(fastqilluminafilter.fastq_files_filtered, (self.group_prefix is not None))
                else:
                    filtered_read1_files = fastqilluminafilter.fastq_files_filtered
                    filtered_read2_files = []
                filtered_read1_files = sorted(filtered_read1_files)
                filtered_read2_files = sorted(filtered_read2_files)
            else:
                fastqilluminafilter = None
                filtered_read1_files = self.get_all_reads("read1")
                filtered_read2_files = self.get_all_reads("read2")

            # archive the files
            #TODO : if self.group_prefix == None, the create the output of fastqilluminafilter in the run.get_work_directory()
            logging.getLogger("CasavaNG6Workflow").debug("illumina_process filter reads = " + ",".join(filtered_read1_files))
            logging.getLogger("CasavaNG6Workflow").debug("illumina_process readsi =" + ",".join(self.get_all_reads("index")))
            saved_files = filtered_read1_files + filtered_read2_files + self.get_all_reads("index")
            logging.getLogger("CasavaNG6Workflow").debug("illumina_process saved_files = " + ",".join(saved_files))
            reads_prefixes = None
            if self.group_prefix != None :
                # concatenate fastq
                reads_prefixes = list((Utils.get_group_basenames(saved_files, "read")).keys())
                logging.getLogger("CasavaNG6Workflow").debug("illumina_process saved_files = " + ",".join(saved_files))
                concatenatefastq = self.add_component("ConcatenateFilesGroups", [self.runobj,saved_files, reads_prefixes])
                saved_files = concatenatefastq.concat_files
                logging.getLogger("CasavaNG6Workflow").debug("illumina_process after concatenatefastq, saved_files = " + ",".join(saved_files))

        else :
            reads_prefixes = None
            fastqilluminafilter = None
            filtered_read1_files = self.get_all_reads("read1")
            filtered_read2_files = self.get_all_reads("read2")
            saved_files = self.get_all_reads()

        # add raw
        addrawfiles = self.add_component("AddRawFiles", [self.runobj, saved_files, self.compression])
        contam = []
        try :
            contam.append(self.get_resource("phix_bwa"))
            contam.append(self.get_resource("ecoli_bwa"))
            contam.append(self.get_resource("yeast_bwa"))
        except : pass

        # contamination_search
        if contam :
            if self.contamination_databank:  contam.extend(self.contamination_databank)
            contamination_search = self.add_component("ContaminationSearch", [filtered_read1_files+filtered_read2_files, contam, reads_prefixes], parent = fastqilluminafilter)

        # make some statistics on raw file
        fastqc = self.add_component("FastQC", [filtered_read1_files+filtered_read2_files, (self.group_prefix is not None), self.no_group, "fastqc.tar.gz"], parent = fastqilluminafilter)
        return fastqilluminafilter, filtered_read1_files, filtered_read2_files, saved_files, concatenatefastq