Commit 9d36ac81 authored by Jerome Mariette's avatar Jerome Mariette
Browse files

improve code

parent cceb7b77
......@@ -22,8 +22,15 @@ __version__ = '1.0'
__email__ = 'support.genopole@toulouse.inra.fr'
__status__ = 'beta'
import os, re, tempfile, uuid, sys, logging
import datetime, time
import os
import re
import tempfile
import uuid
import sys
import logging
import pickle
import datetime
import time
from subprocess import call
from shutil import copyfile, rmtree
from ConfigParser import ConfigParser, RawConfigParser
......@@ -34,9 +41,10 @@ from ng6.t3MySQLdb import t3MySQLdb
from ng6.utils import Utils
from jflow.component import Component
from weaver.function import PythonFunction
def add_analysis(analysis_serialized_path, pr_serialized_path, *files):
def add_analysis(analysis_serialized_path, *files):
from ng6.analysis import Analysis
from ng6.project import Project
from ng6.run import Run
......@@ -45,21 +53,20 @@ def add_analysis(analysis_serialized_path, pr_serialized_path, *files):
analysis_dump = open(analysis_serialized_path, "rb")
analysis = pickle.load(analysis_dump)
analysis_dump.close()
# check if the parent analysis config file is in files
for file in files:
if file != analysis.get_config_file_path() and os.path.basename(file) == Analysis.ANALYSIS_CONFIG_FILE_NAME:
parent = Analysis.get_from_file(file)
analysis.set_parent_analysis(parent)
# load the project or the run
pr_dump = open(pr_serialized_path, "rb")
pr = pickle.load(pr_dump)
pr_dump.close()
analysis.parent = parent
# process the parsing of the analysis
analysis.post_process()
# add the analysis to the run or the project
pr.add_analysis(analysis)
# add the analysis to the right run/project object
if analysis.project: analysis.project.add_analysis(analysis)
elif analysis.run: analysis.run.add_analysis(analysis)
analysis.write_config_file()
class Analysis (Component):
"""
......@@ -88,7 +95,7 @@ class Analysis (Component):
self.options = options
self.id = id
self.version = version
self.set_parent_analysis(parent_analysis)
self.parent = parent_analysis
self.results = {}
self.run = None
self.project = None
......@@ -119,12 +126,6 @@ class Analysis (Component):
self.directory = os.path.join(self.DIRECTORIES_STRUCTURE, directory_name)
logging.getLogger("Analysis.__init__").debug("Building brand new analysis [" + str(self) + "]")
def set_parent_analysis(self, parent_analysis):
if parent_analysis != None:
self.parent_uid = parent_analysis.id
else :
self.parent_uid = 0
def get_version(self):
"""
Return the tool version, has to be implemented by subclasses
......@@ -143,6 +144,37 @@ class Analysis (Component):
"""
raise NotImplementedError
def execute(self):
# first create the output directory
if not os.path.isdir(self.output_directory):
os.makedirs(self.output_directory, 0751)
# first add analysis information
self.define_analysis()
# serialized the object
analysis_dump_path = self.get_temporary_file(".dump")
analysis_dump = open(analysis_dump_path, "wb")
pickle.dump(self, analysis_dump)
analysis_dump.close()
wait_for_files = []
for attr in self.__dict__:
# TODO if no Output object raise error
if self.__getattribute__(attr).__class__.__name__ == "OutputFile" or \
self.__getattribute__(attr).__class__.__name__ == "OutputDirectory":
wait_for_files.append(self.__getattribute__(attr))
elif self.__getattribute__(attr).__class__.__name__ == "OutputFileList":
wait_for_files.extend(self.__getattribute__(attr))
if self.parent:
wait_for_files.append(self.parent.get_config_file_path())
# then run the component
self.process()
# add the add_analysis command lines to the make
add = PythonFunction(add_analysis, cmd_format="{EXE} {ARG} {IN} {OUT}")
add(inputs=wait_for_files, outputs=self.get_config_file_path(), arguments=analysis_dump_path)
def __str__(self):
"""
Return a full description of the analysis
......@@ -217,9 +249,11 @@ class Analysis (Component):
"""
# First add the run into the database
t3mysql = t3MySQLdb()
if self.parent: parent_uid = self.parent.id
else: parent_uid = 0
self.id = t3mysql.add_analysis(self.__class__.__name__, self.name, self.description,
datetime.date.today(), self.directory, self.software, self.version,
self.options, parent_uid=self.parent_uid)
self.options, parent_uid=parent_uid)
# Then add all results link to this analysis
for file in self.results:
......
......@@ -21,8 +21,6 @@ import pickle
import datetime
from jflow.workflow import Workflow
from weaver.function import PythonFunction
from ng6.analysis import add_analysis
from ng6.project import Project
from ng6.run import Run
......@@ -66,14 +64,6 @@ class NG6Workflow (Workflow):
raise ValueError('Mids description incorrect: should be MID:MIDDESC;... or MID1,MID2:MIDSDESC;...')
# then add the run to the project
self.project.add_run(self.runobj)
def __add_analysis_behaviour(self, fn, input_files, analysis_config_file, arguments):
def new_run(*args, **kwargs):
fn(*args, **kwargs)
add = PythonFunction(add_analysis, cmd_format="{EXE} {ARG} {IN} {OUT}")
add(inputs=input_files, outputs=[analysis_config_file], arguments=arguments)
return new_run
def add_component(self, component_name, args=[], kwargs={}, component_prefix="default", parent=None, addto="run"):
# first build and check if this component is OK
......@@ -91,35 +81,10 @@ class NG6Workflow (Workflow):
# if the built object is an analysis
for derived_class in cmpt_object.__class__.__bases__:
if derived_class.__name__ == "Analysis":
# first add analysis information to the analysis object
cmpt_object.define_analysis()
# serialized the object
analysis_dump_path = self.get_temporary_file(".dump")
analysis_dump = open(analysis_dump_path, "wb")
pickle.dump(cmpt_object, analysis_dump)
analysis_dump.close()
pr_dump_path = self.get_temporary_file(".dump")
pr_dump = open(pr_dump_path, "wb")
if addto == "project":
pickle.dump(self.project, pr_dump)
elif addto == "run":
pickle.dump(self.runobj, pr_dump)
pr_dump.close()
args = [analysis_dump_path, pr_dump_path]
wait_for_files = []
for attr in cmpt_object.__dict__:
# TODO if no Output object raise error
if cmpt_object.__getattribute__(attr).__class__.__name__ == "OutputFile" or \
cmpt_object.__getattribute__(attr).__class__.__name__ == "OutputDirectory":
wait_for_files.append(cmpt_object.__getattribute__(attr))
elif cmpt_object.__getattribute__(attr).__class__.__name__ == "OutputFileList":
wait_for_files.extend(cmpt_object.__getattribute__(attr))
parent_config_file = None
if parent:
wait_for_files.append(parent.get_config_file_path())
cmpt_config_file = cmpt_object.get_config_file_path()
cmpt_object.process = self.__add_analysis_behaviour(cmpt_object.process, wait_for_files, cmpt_config_file, args)
break
# add parent and project/run information
cmpt_object.parent = parent
if addto == "project": cmpt_object.project = self.project
elif addto == "run": cmpt_object.run = self.runobj
# add the component
self.components.append(cmpt_object)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment