analysis.py

#
# Copyright (C) 2009 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

__author__ = 'Plateforme bioinformatique Midi Pyrenees'
__copyright__ = 'Copyright (C) 2009 INRA'
__license__ = 'GNU General Public License'
__version__ = '1.0'
__email__ = 'support.bioinfo.genotoul@inra.fr'
__status__ = 'beta'

import os
import re
import tempfile
import uuid
import sys
import logging
import pickle
import datetime
import time
import shutil
from subprocess import call
from shutil import copyfile, rmtree
from configparser import ConfigParser, RawConfigParser

from ng6.run import Run
from ng6.config_reader import NG6ConfigReader
from ng6.t3MySQLdb import t3MySQLdb
from ng6.utils import Utils

from jflow.component import Component
from weaver.function import PythonFunction


def add_analysis(parent_id, analysis_cfg, *input_files):
    from ng6.analysis import Analysis
    from ng6.project import Project
    from ng6.run import Run
    import pickle
    import logging
    logging.getLogger("Analysis").debug("Start. Imports went good.")
    logging.getLogger("Analysis").debug("Start. working for analysis " + analysis_cfg)

    # get inputs from parameters
    analysis_serialized_path = input_files[0]
    try: parent_analysis_cfg = input_files[1]
    except: parent_analysis_cfg = None
    logging.getLogger("Analysis").debug("Start. parent_analysis_cfg is initialised.")

    # load the analysis object
    analysis_dump = open(analysis_serialized_path, "rb")
    analysis = pickle.load(analysis_dump)
    analysis_dump.close()
    logging.getLogger("Analysis").debug("Dump. analysis_dump is closed and analysis is loaded.")

    # add the parent information
    logging.getLogger("Analysis").debug("parent_analysis_cfg. Starting")
    if parent_analysis_cfg:
        if os.path.isfile(parent_analysis_cfg):
            logging.getLogger("Analysis").debug("parent_analysis_cfg. Analysis.get_from_file(parent_analysis_cfg) is about to start")
            parent = Analysis.get_from_file(parent_analysis_cfg)
            logging.getLogger("Analysis").debug("parent_analysis_cfg. Analysis.get_from_file(parent_analysis_cfg) is done")
            analysis.parent = parent
    elif parent_id != 'none' :
        logging.getLogger("Analysis").debug("parent_analysis_cfg. Analysis.get_from_file(parent_analysis_cfg) is about to start")
        parent = Analysis.get_from_id(int(parent_id))
        analysis.parent = parent
        logging.getLogger("Analysis").debug("parent_analysis_cfg. Analysis.get_from_file(parent_analysis_cfg) is done")

    # process the parsing of the analysis
    logging.getLogger("Analysis").debug("analysis.post_process. Starting")
    analysis.post_process()
    logging.getLogger("Analysis").debug("analysis.post_process. Done")

    # add the analysis to the right run/project object
    logging.getLogger("Analysis").debug("analysis.project. Starting add_analysis(analysis)")
    if analysis.project:
        logging.getLogger("Analysis").debug("analysis.project. Starting analysis.project.add_analysis(analysis)")
        analysis.project.add_analysis(analysis)
        logging.getLogger("Analysis").debug("analysis.project. Finishing analysis.project.add_analysis(analysis)")
    elif analysis.run:
        logging.getLogger("Analysis").debug("analysis.run. Starting analysis.run.add_analysis(analysis)")
        analysis.run.add_analysis(analysis)
        logging.getLogger("Analysis").debug("analysis.run. Finishing analysis.run.add_analysis(analysis)")
    logging.getLogger("Analysis").debug("Config File. Starting the writing of config file")
    analysis.write_config_file()
    logging.getLogger("Analysis").debug("Config File. Is written")

class Analysis (Component):
    """
    Class Analysis: Define an nG6 Analysis
    """

    # The directories structure into ng6
    DIRECTORIES_STRUCTURE = "analyze"
    ANALYSIS_CONFIG_FILE_NAME = "analysis.cfg"

    def __init__(self, name="", description="", software="",
                 options="", version="", id=None, date=None, parent_analysis=None, space_id="default"):
        """
        Build an Analysis object
          @param name            : the analysis name
          @param description     : the analysis description
          @param admin_login     : the admin login
          @param software        : the software used to run the analysis
          @param options         : the options used to run the software
          @param id              : the analysis id if not defined
          @param parent_analysis : the parent analysis
        """
        Component.__init__(self)
        self.name = name
        self.description = description
        self.admin_login = None
        self.software = software
        self.options = options
        self.id = id
        self.version = version
        self.parent = parent_analysis
        self.results = {}
        self.run = None
        self.project = None
        self.is_editable = False
        self.space_id = space_id
        self.retention_date = None
        self.date = date
        
        # Set the temp folder to the ng6 temp folder
        ng6conf = NG6ConfigReader()
        logging.getLogger("Analysis.__init__").debug("ng6conf est chargé")
        tempfile.tempdir = ng6conf.get_tmp_directory()
        logging.getLogger("Analysis.__init__").debug("tempfile.tempdir = " + tempfile.tempdir)
        if not os.path.isdir(tempfile.tempdir):
            os.makedirs(tempfile.tempdir, 0o751)
        logging.getLogger("Analysis.__init__").debug("self.space_id = " + self.space_id)
        
        
        if id != None : # If not a new analysis
            logging.getLogger("Analysis.__init__").debug("Connexion à la BD")
            t3mysql = t3MySQLdb()
            logging.getLogger("Analysis.__init__").debug("Connexion effectuée")
            self.run = Run.get_from_id(self.__get_run_id())
            from ng6.project import Project
            if self.run != None :
                self.project = Project.get_from_run_id(self.__get_run_id())
            else :
                self.project = Project.get_from_id(self.__get_project_id())
            self.space_id=self.project.space_id
            self.directory = t3mysql.select_analysis_directory(id)
            logging.getLogger("Analysis.__init__").debug("Building analysis with id=" + str(id) + " [" + str(self) + "]")
        else :
            self.version = self.get_version()
            if isinstance(self.version, bytes):
                self.version = self.version.decode()
            logging.getLogger("Analysis.__init__").debug("Building brand new analysis [" + str(self) + "]")

    def define_analysis(self):
        """
        Define all analysis attributs, has to be implemented by subclasses
        """
        raise NotImplementedError

    def post_process(self):
        """
        Process the analysis, has to be implemented in the sub class
        """
        raise NotImplementedError

    def execute(self):
        ng6conf = NG6ConfigReader()
        directory_name = uuid.uuid4().hex[:9]
        while True:
            save_dir = os.path.join(ng6conf.get_save_directory(), ng6conf.get_space_directory(self.space_id), self.DIRECTORIES_STRUCTURE, directory_name)
            logging.getLogger("Analysis.__init__").debug("Building analysis save_dir=" + save_dir)
            work_dir = os.path.join(ng6conf.get_work_directory(), ng6conf.get_space_directory(self.space_id), self.DIRECTORIES_STRUCTURE, directory_name)
            logging.getLogger("Analysis.__init__").debug("Building analysis work_dir=" + save_dir)
            if not os.path.isdir(save_dir) and not os.path.isdir(work_dir):
                break
            directory_name = uuid.uuid4().hex[:9]
        self.directory = "/" + os.path.join(ng6conf.get_space_directory(self.space_id), self.DIRECTORIES_STRUCTURE, directory_name)
        self.retention_date = ng6conf.get_retention_date(self.space_id)

        # first create the output directory
        if not os.path.isdir(self.output_directory):
            os.makedirs(self.output_directory, 0o751)
        # and analysis output
        if not os.path.isdir(self.__get_work_directory()):
            os.makedirs(self.__get_work_directory(), 0o751)

        # then add analysis information
        self.define_analysis()

        inputs = []
        # serialized the object
        analysis_dump_path = self.get_temporary_file(".dump")
        analysis_dump = open(analysis_dump_path, "wb")
        pickle.dump(self, analysis_dump)
        analysis_dump.close()
        inputs.append(analysis_dump_path)

        parent_id = 'none'
        wait_for_files = []
        for attr in self.__dict__:
            # TODO if no Output object raise error
            if self.__getattribute__(attr).__class__.__name__ == "OutputFile" or \
                self.__getattribute__(attr).__class__.__name__ == "OutputDirectory":
                wait_for_files.append(self.__getattribute__(attr))
            elif self.__getattribute__(attr).__class__.__name__ == "OutputFileList":
                wait_for_files.extend(self.__getattribute__(attr))
        if self.parent:
            if self.parent.id:
                parent_id = self.parent.id
            else :
                inputs.append(self.parent.get_config_file_path())
        # then run the component
        self.process()
        # add the add_analysis command lines to the make
        add = PythonFunction(add_analysis, cmd_format="{EXE} {ARG} {OUT} {IN}")
        add(includes=wait_for_files, outputs=self.get_config_file_path(), inputs=inputs, arguments=parent_id)

    def __str__(self):
        """
        Return a full description of the analysis
        """
        return "id="+str(self.id)+";name="+str(self.name)+";description="+str(self.description)+";software="+\
            str(self.software)+";version="+str(self.version)+";options="+str(self.options)+\
            ";space_id="+str(self.space_id)+";project="+str(self.project)+";run="+str(self.run)

    def sync(self):
        """
        Synchronize data related to the Analysis between temporary folder and long term storage folder.
        """
        if os.path.isdir(self.__get_work_directory()):
            if len(os.listdir(self.__get_work_directory())):
                logging.getLogger("Analysis.sync").debug("Synchronizing analysis id=" + str(self.id) + " from " + self.__get_work_directory() + " to " + self.__get_save_directory())
                try :
                    if not os.path.isdir(self.__get_save_directory()) :
                        # First make the save directory
                        os.makedirs(self.__get_save_directory(), 0o751)
                    if os.path.exists(self.__get_work_directory()) and self.__get_work_directory() != self.__get_save_directory():
                        cmd = "rsync -avh --remove-sent-files "+ self.__get_work_directory() + "/ " + self.__get_save_directory()
                        logging.getLogger("Analysis.sync").debug(cmd)
                        retcode = call(["rsync", "-avh", "--remove-sent-files",self.__get_work_directory() + "/", self.__get_save_directory()], shell=False)
                        if retcode < 0:
                            logging.getLogger("Analysis.sync").error("Error when synchronizing analysis id=" + str(self.id) + "(retcode=" + str(retcode) + ")")
                        else:
                            try: os.rmdir(self.__get_work_directory())
                            except: pass
                            logging.getLogger("Analysis.sync").debug("Synchronization done for analysis id=" + str(self.id) + "(retcode=" + str(retcode) + ")")
                    else:
                        logging.getLogger("Analysis.sync").info("The synchronisation has not been performed, either because " + self.__get_work_directory() + " does not exists or because the source and destination folders are the same.")
                    # update the storage size
                    t3mysql = t3MySQLdb()
                    t3mysql.update_analysis_field(self.id, "storage_size", Utils.get_storage_size(self.__get_save_directory()))
                except Exception as err:
                    raise Exception(str(err))

    def list_or_copy_analysis_files(self, outputdir, pattern):
        """
        Return list of file corresponding to the pattern and copy unzipped files into output dir if it's zipped
          @param file  : the pattern of files to return
          @return      : list of file.
        """
        list = []
        if outputdir == "":
            outputdir = tempfile.mkdtemp()

        for file in os.listdir(self.__get_save_directory()) :
            # If it fits criteria
            if pattern != "":
                if file.endswith(pattern):
                    list.append(os.path.join(self.__get_save_directory(), file))
                elif file.endswith(pattern + ".gz") or file.endswith(pattern + ".zip"):
                    list.append(Utils.gunzip(os.path.join(self.__get_save_directory(), file), outputdir))
                elif file.endswith(pattern + ".bz2") or file.endswith(pattern + ".bz"):
                    list.append(Utils.uncompress_bz2(os.path.join(self.__get_save_directory(), file), outputdir))
                elif file.endswith("tar.gz") or file.endswith("tar.zip"):
                    list.extend(Utils.untar_files(os.path.join(self.__get_save_directory(), file), pattern, outputdir))
                elif file.endswith("tar.bz2") or file.endswith("tar.bz"):
                    list.extend(Utils.untar_files(os.path.join(self.__get_save_directory(), file), pattern, outputdir))
            else:
                list.append(file)
        return list


    def get_run_config_file(self):
        """
        Return the file path to the config file if the analysis if link to a run
          @return      : the file run file if it's a run analysis.
        """
        try:
            t3mysql = t3MySQLdb()
            run_id = t3mysql.select_analysis_run_id(self.id)
            my_run = Run.get_from_id(run_id)
            return my_run.get_config_file()
        except :
            return None

    def get_template(self):
        """
        Return the name of the template used for the visualisation.
        """
        return self.__class__.__name__

    def save(self):
        """
        update the database info
        """
        # First add the run into the database
        logging.getLogger("Analysis.save").debug("Connexion try")
        t3mysql = t3MySQLdb()
        logging.getLogger("Analysis.save").debug("Connexion done")
        if self.parent: parent_uid = self.parent.id
        else: parent_uid = 0
        logging.getLogger("Analysis.save").debug("Doing the add analysis")
        self.id = t3mysql.add_analysis(self.get_template(), self.name, self.description, self.admin_login,
                                      datetime.date.today(), self.directory, self.software, self.version,
                                      self.options, self.retention_date, self.retention_date, 
                                      self.is_editable, parent_uid=parent_uid)
        logging.getLogger("Analysis.save").debug("add_analysis done")
        # Then add all results link to this analysis
        for file in self.results:
            for result in self.results[file]:
                t3mysql.add_result(self.id, file, result[0], result[1], result[2])

        # Finaly return it's id
        return self.id

    def get_config_file_path(self):
        return os.path.join(self.output_directory, self.ANALYSIS_CONFIG_FILE_NAME)

    def write_config_file(self):
        """
        Write an analysis config file
          @return : path the the config file
        """
        t3mysql = t3MySQLdb()

        # First select run
        [name, date, description, software, options, version] = t3mysql.select_analysis(self.id)
        config = RawConfigParser()
        config.add_section("Analysis")
        config.set("Analysis", "analysis_id", self.id)
        config.set("Analysis", "name", name)
        config.set("Analysis", "date", date)
        config.set("Analysis", "description", description)
        config.set("Analysis", "software", software)
        config.set("Analysis", "options", options)
        config.set("Analysis", "version", version)

        config_path = self.get_config_file_path()
        config.write(open(config_path,'w'))
        return config_path


    def _add_result_element(self, file, result_key, result_value, result_group="default"):
        """
        add the result row
          @param file         : the file name the result is linked to
          @param result_key   : the result key
          @param result_value : the result value associated to the key
          @param result_group : the result group it belongs to
        """
        if file in self.results:
            self.results[file].append([result_key, result_value, result_group])
        else :
            self.results[file] = [[result_key, result_value, result_group]]


    def _create_and_archive(self, files, archive_name=None, prefix="dir"):
        logging.getLogger("Analysis").debug("_create_and_archive entering")
        """
        return the web path to the archive files
        If there are samples, datas will be organised by samples
          @param files        : table of file
          @param archive_name : the archive name to use, if not set will be
                                the same as link_value
          @param prefix       : in case identique file names put them
                                in directories prefixed by this value
          @return             : the web path to the archive
        """
        # If archive name not set uses the link_value instead
        if archive_name == None:
            archive_name = "archive"
        if archive_name.endswith(".gz"):
            archive_name = os.path.splitext(archive_name)[0]

        file_basenames = []
        for file in files:
            (fhead, ftail)= os.path.split(file)
            file_basenames.append(ftail)

        #If some files have the same name
        if len(file_basenames) != len(set(file_basenames)) :

            try : # If there are samples

                if self.run :
                    samples = self.run.get_samples()
                else :
                    run_id = self.__get_run_id()
                    my_run = Run.get_from_id(run_id)
                    samples = my_run.get_samples()
                logging.getLogger("Analysis").debug("_create_and_archive with samples")
                gfiles = {}
                ungrouped_files = []
                what_left = []
                for file in files :
                    ungrouped_files.append(file)
                    what_left.append(file)

                # First group files if they have the sample name in their path
                for file in files :

                    # In case of multiple description take the longuest
                    best_description = []
                    for sample in samples :
                        spname = sample.name
                        spname_regex = re.compile(".*" + spname + ".*")
                        spr = spname_regex.match(file)
                        if spr :
                            best_description.append(spname)

                    max_len = 0
                    final_description = None
                    if len(best_description) > 0:
                        for bdescription in best_description:
                            if len(bdescription) > max_len:
                                max_len = len(bdescription)
                                final_description = bdescription

                    if final_description != None :
                        if final_description in gfiles:
                            gfiles[final_description].append(file)
                        else :
                            gfiles[final_description] = [file]
                        if file in ungrouped_files:
                            ungrouped_files.remove(file)
                            what_left.remove(file)

                # For files which don't have the mid in their path, try to group them considering there path
                to_add = {}
                for file in ungrouped_files:
                    (fhead, ftail)= os.path.split(file)
                    for mid in gfiles.keys() : # For each group
                        for grouped_file in gfiles[mid]: # for each file of this group
                            (ghead, gtail)= os.path.split(grouped_file)
                            if fhead == ghead: # If this group file
                                if file in what_left :
                                    what_left.remove(file)
                                if mid in to_add :
                                    to_add[mid].append(file)
                                else :
                                    to_add[mid] = [file]

                for mid in to_add.keys() :
                     gfiles[mid].extend(to_add[mid])

                # Then create the archive
                tmp_dir = tempfile.NamedTemporaryFile().name
                os.makedirs(tmp_dir)
                for mid in gfiles.keys() :
                    mid_dir = os.path.join(tmp_dir, mid)
                    os.makedirs(mid_dir)
                    for file in gfiles[mid]:
                        copyfile(file, os.path.join(mid_dir, os.path.basename(file)))

                if len(what_left) > 0:
                    if len(gfiles.keys()) > 0 :
                        other_dir = os.path.join(tmp_dir, "others")
                        os.makedirs(other_dir)
                        for file in what_left:
                            copyfile(file, os.path.join(other_dir, os.path.basename(file)))
                    else :
                        for file in what_left:
                            copyfile(file, os.path.join(tmp_dir, os.path.basename(file)))

                logging.getLogger("Analysis").debug("_create_and_archive before tarf")
                tarf = Utils.tar_dir(tmp_dir, os.path.join(self.__get_work_directory(), archive_name))
                logging.getLogger("Analysis").debug("_create_and_archive before targzf")
                targzf = Utils.gzip(tarf, self.__get_work_directory(), delete=False)
                # Then delete temporary files
                logging.getLogger("Analysis").debug("_create_and_archive before os.remove(tarf)")
                os.remove(tarf)
                logging.getLogger("Analysis").debug("_create_and_archive before rmtree(tmp_dir)")
                rmtree(tmp_dir)
                logging.getLogger("Analysis").debug("_create_and_archive before return " + os.path.join(self.directory, os.path.basename(targzf)))
                return 'fileadmin' + os.path.join(self.directory, os.path.basename(targzf))

            except :
                logging.getLogger("Analysis").debug("_create_and_archive in execpt, without samples?")
                gfiles = {}
                # Group files by folders
                for file in files:
                    (fhead, ftail)= os.path.split(file)
                    if fhead in gfiles:
                        gfiles[fhead].append(file)
                    else :
                        gfiles[fhead] = [file]
                # Then create the archive
                tmp_dir = tempfile.NamedTemporaryFile().name
                os.makedirs(tmp_dir)
                for i, dir_name in enumerate(gfiles.keys()) :
                    dir_index = prefix
                    for j in range(3 - len(str(i))):
                        dir_index += "0"
                    dir_index += str(i+1)
                    dir = os.path.join(tmp_dir, dir_index)
                    os.makedirs(dir)
                    for file in gfiles[dir_name]:
                        copyfile(file, os.path.join(dir, os.path.basename(file)))
                logging.getLogger("Analysis").debug("_create_and_archive before tarf")

                tarf = Utils.tar_dir(tmp_dir, os.path.join(self.__get_work_directory(), archive_name))
                logging.getLogger("Analysis").debug("_create_and_archive before targzf")
                targzf = Utils.gzip(tarf, self.__get_work_directory(), delete=False)
                # Then delete temporary files
                logging.getLogger("Analysis").debug("_create_and_archive before os.remove(tarf)")
                os.remove(tarf)
                logging.getLogger("Analysis").debug("_create_and_archive before rmtree(tmp_dir)")
                rmtree(tmp_dir)
                logging.getLogger("Analysis").debug("_create_and_archive before return " + os.path.join(self.directory, os.path.basename(targzf)))
                return 'fileadmin' + os.path.join(self.directory, os.path.basename(targzf))

        else :
            logging.getLogger("Analysis").debug("_create_and_archive, length differs")
            logging.getLogger("Analysis").debug("_create_and_archive before tarf")
            tarf = Utils.tar_files(files, os.path.join(self.__get_work_directory(), archive_name))
            logging.getLogger("Analysis").debug("_create_and_archive before targzf")
            targzf = Utils.gzip(tarf, self.__get_work_directory(), delete=False)
            # Then delete temporary files
            logging.getLogger("Analysis").debug("_create_and_archive before os.remove(tarf)")
            os.remove(tarf)
            logging.getLogger("Analysis").debug("_create_and_archive before return " + os.path.join(self.directory, os.path.basename(targzf)))
            return 'fileadmin' + os.path.join(self.directory, os.path.basename(targzf))


    def _archive_files(self, files, mode, archive_name="ng6_archive.tar", delete=False):
        """
        Copy, archive or compress the files list to the current analysis. Files can then be downloaded and gave back by
        the script ng6run2ergatis.
          @param files: the files to archive
          @param mode: can be none, gz, bz2, tar.gz and tar.bz2
          @param archive_name: the archive name if tar is requested
          @param delete: delete files
        """

        # First handle if only one file
        if len(files) == 1 and mode == "tar.gz":
            mode = "gz"
        elif len(files) == 1 and mode == "tar.bz2":
            mode = "bz2"

        if mode == "none":
            for file in files:
                if os.path.isfile(file):
                    copyfile(file, os.path.join(self.__get_work_directory(), os.path.basename(file)))
                # Finaly try to delete the original file if asked to do so
                try:
                    if delete:
                        os.remove(file)
                except:
                    pass

        elif mode == "gz":
            for file in files:
                Utils.gzip(file, self.__get_work_directory(), delete)

        elif mode == "bz2":
            for file in files:
                Utils.bz2(file, self.__get_work_directory(), delete)

        elif mode == "tar.gz":
            Utils.tar_files(files, os.path.join(self.__get_work_directory(), archive_name), delete)
            Utils.gzip(os.path.join(self.__get_work_directory(), archive_name), self.__get_work_directory(), True)

        elif mode == "tar.bz2":
            Utils.tar_files(files, os.path.join(self.__get_work_directory(), archive_name), delete)
            Utils.bz2(os.path.join(self.__get_work_directory(), archive_name), self.__get_work_directory(), True)

    def _save_directory(self, directory, directory_name=None):
        """
        add a directory in the analysis directory
          @param directory      : the path to the directory to link
          @param directory_name : the directory name to use
        """
        # First copy the file into the analysis directory
        if os.path.isdir(directory) :
            if directory_name == None:
                directory_name = os.path.basename(directory)
            shutil.copytree(directory, os.path.join(self.__get_work_directory(), directory_name))

    def _save_file(self, file, file_name=None, gzip=False):
        """
        add a file in the analysis directory, and return its web path
          @param file         : the path to the file to link
          @param file_name    : the file name to use to store the image
          @param gzip         : if True the file will be gziped
          @return             : the web path to the file
        """

        if not os.path.isdir(self.__get_work_directory()):
            os.makedirs(self.__get_work_directory(), 0o751)
        # First copy the file into the analysis directory
        if os.path.isfile(file) :
            if file_name == None:
                file_name = os.path.basename(file)
            if gzip is True :
                file = Utils.gzip(file, self.__get_work_directory(), delete=False)
                file_name = os.path.basename(file)
            else :
                copyfile(file, os.path.join(self.__get_work_directory(), file_name))
            return self.get_web_filepath(file_name, 'fileadmin')

    def _save_files(self, files, gzip=False):
        """
        add a file in the analysis directory, and return its web path
          @param file         : the path to the file to link
          @param gzip         : if True the file will be gziped
          @return             : the web path to the file
        """
        file_path = []
        for file in files:
            file_path.append(self._save_file(file, gzip=gzip))
        return file_path

    def __get_run_id(self):
        """
        Return the run id the analysis belongs to
          @return run_id  : the run id the analysis belongs to
        """
        t3mysql = t3MySQLdb()
        return t3mysql.select_analysis_run_id(self.id)

    def __get_project_id(self):
        """
        Return the project id the analysis belongs to
          @return project_id  : the project id the analysis belongs to
        """
        t3mysql = t3MySQLdb()
        return t3mysql.select_analysis_project_id(self.id)

    def __get_save_directory(self):
        """
        Return the full path of the analysis directory into the save dir
        """
        ng6conf = NG6ConfigReader()
        return ng6conf.get_save_directory() + self.directory


    def __get_work_directory(self):
        """
        Return the full path of the analysis directory into the work dir
        """
        ng6conf = NG6ConfigReader()
        return ng6conf.get_work_directory() + self.directory


    @staticmethod
    def get_from_id (id):
        """
        Return an Analysis object specified by its id
          @param id : the analysis id
        """
        logging.getLogger("Analysis.get_from_id").debug("Loading the analysis from id " + str(id))
        #try:
        t3mysql = t3MySQLdb()
        [name, date, description, software, options, version] = t3mysql.select_analysis(id)
        my_analysis = Analysis(name, description, software, options, version, id, date)
        if my_analysis == None:
            logging.getLogger("Analysis.get_from_id").error("The analysis id=" + str(id) + " does not exists in the database.")
            raise Exception("The analysis id=" + str(id) + " does not exists in the database.\n")
        return my_analysis
        #except Exception as err:
        #    logging.getLogger("Analysis.qget_from_id").error("Error while loading the analysis from the database [" + str(err) + "]")
        #    raise Exception("Error while loading the analysis from the database\n" + str(err))


    @staticmethod
    def get_from_file (config_file):
        """
        Return an analysis object specified by its config file
          @param config_file : the analysis config file
        """
        try:
            reader = ConfigParser()
            reader.read(config_file)
            analysis_id = reader.get('Analysis', 'analysis_id')
            my_analysis = Analysis.get_from_id(analysis_id)
            return my_analysis
        except :
            pass


    def get_creation_date(self):
        """
        Returns an analysis's crdate value
        """
        if self.id != None :
            t3mysql = t3MySQLdb()
            result = t3mysql.get_analysis_creation_date(str(self.id))
            return next(iter(result))
    
    def get_directory(self):
        """
        Returns an analysis's directory
        """
        if self.id != None :
            t3mysql = t3MySQLdb()
            return t3mysql.select_analysis_directory(str(self.id))
    
    def change_space (self, space_id ):
        import os
        from ng6.project import Project
        ng6conf = NG6ConfigReader()
        
        old_path = ng6conf.get_save_directory() + self.directory
        directory_name = os.path.split(old_path)[-1]
        new_relative_path = os.path.join(ng6conf.get_space_directory(space_id), self.DIRECTORIES_STRUCTURE, directory_name)
        
        new_absolute_path =  os.path.join(ng6conf.get_save_directory(), new_relative_path )
        
        #We create the /analyze directory if it's missing
        path_to_analyze_dir = os.path.join(ng6conf.get_save_directory(),ng6conf.get_space_directory(space_id),self.DIRECTORIES_STRUCTURE)
        if not os.path.isdir(path_to_analyze_dir):
            os.mkdir(path_to_analyze_dir,0o755)
        
        str_cmd = ""
        retcode = -1
        if str(old_path) != str(new_absolute_path):
            [retcode, str_cmd] = Utils.rsync_getcmd(old_path,new_absolute_path)
            if retcode != 0 :               
                raise Exception("Error while trying to rsync " + old_path + " to " + new_absolute_path + "\n" +
                                "Command : " + str_cmd + "\n" + "Error code : " + str(retcode) + "\n")
        else:
            str_cmd = "Source and destination directories are the same : " + old_path + " , ignored.\n"
        
        new_retention = ng6conf.get_retention_date(space_id, self.date)
        
        t3mysql = t3MySQLdb()
        t3mysql.update_fields('tx_nG6_analyze', str(self.id), ['directory','retention_date'], [str("/"+new_relative_path),new_retention] )
        
        result_files = t3mysql.get_analysis_result_files(str(self.id), 'fileadmin')
        for result_file in result_files :
            filename = os.path.basename(result_file['rvalue'])
            new_filepath = self.get_web_filepath(filename)
            t3mysql.update_fields('tx_nG6_result', str(result_file['uid']), ['rvalue'], [str(new_filepath)] )
            
        return [retcode, str_cmd]
    
    def get_web_filepath(self, file_name, base_filepath='fileadmin' ):
        return base_filepath + os.path.join(self.directory, file_name)