Commit 9ec14ddb authored by Jerome Mariette's avatar Jerome Mariette

add temporary directory handling

parent 9d96d282
......@@ -37,8 +37,8 @@ dbname = typo3
log_file = /home/jmariett/scratch/ng6/ng6.log
[storage]
save_directory = /home/jmariett/scratch/ngspipelines/
working_directory = /home/jmariett/scratch/weaver
tmp_directory = /home/jmariett/scratch/tmp
[softwares]
blastall = /usr/bin/blastall
......
......@@ -17,6 +17,7 @@
import os
import inspect
import tempfile
from jflow.workflows_manager import WorkflowsManager
from jflow.config_reader import JFlowConfigReader
......@@ -36,6 +37,7 @@ class Component(object):
self.is_dynamic = is_dynamic
self.prefix = "default"
self.output_directory = None
self.config_reader = JFlowConfigReader()
def get_outputs(self, output_list=None, input_list=None):
"""
......@@ -79,6 +81,13 @@ class Component(object):
"""
raise NotImplementedError
def get_temporary_file(self, suffix=".txt"):
# first check if tmp directory exists
if not os.path.isdir(self.config_reader.get_tmp_directory()):
os.makedirs(self.config_reader.get_tmp_directory(), 0751)
tempfile_name = os.path.basename(tempfile.NamedTemporaryFile(suffix=suffix).name)
return os.path.join(self.config_reader.get_tmp_directory(), tempfile_name)
def define_parameters(self, *args):
"""
Define the component parameters, has to be implemented by subclasses
......@@ -86,8 +95,7 @@ class Component(object):
raise NotImplementedError
def get_exec_path(self, software):
config_reader = JFlowConfigReader()
exec_path = config_reader.get_exec(software)
exec_path = self.config_reader.get_exec(software)
if exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), software)):
exec_path = os.path.join(os.path.dirname(inspect.getfile(self.__class__)), software)
elif exec_path is None and os.path.isfile(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), "../bin", software)):
......
......@@ -93,6 +93,9 @@ class JFlowConfigReader(object):
"""
self.reader = ConfigParser()
self.reader.read(os.path.join(os.path.dirname(inspect.getfile(self.__class__)), self.CONFIG_FILE_PATH))
def get_tmp_directory(self):
return self.reader.get("storage", "tmp_directory")
def get_working_directory(self):
return self.reader.get("storage", "working_directory")
......
......@@ -25,7 +25,7 @@ import tempfile
import jflow.components
from jflow.config_reader import WorkflowConfigReader
from jflow.parameter import Parameter
from jflow.workflows_manager import WorkflowsManager
from jflow.workflows_manager import WorkflowsManager, JFlowConfigReader
from weaver.script import ABSTRACTIONS
from weaver.script import DATASETS
......@@ -47,6 +47,7 @@ class Workflow(object):
"""
"""
self.config_reader = WorkflowConfigReader(self._get_property_path())
self.jflow_config_reader = JFlowConfigReader()
self.manager = WorkflowsManager()
self.components = []
self.globals = {}
......@@ -90,10 +91,10 @@ class Workflow(object):
def get_temporary_file(self, suffix=".txt"):
# first check if tmp directory exists
if not os.path.isdir(os.path.join(self.directory, ".tmp")):
os.makedirs(os.path.join(self.directory, ".tmp"), 0751)
if not os.path.isdir(self.jflow_config_reader.get_tmp_directory()):
os.makedirs(self.jflow_config_reader.get_tmp_directory(), 0751)
tempfile_name = os.path.basename(tempfile.NamedTemporaryFile(suffix=suffix).name)
return os.path.join(self.directory, ".tmp", tempfile_name)
return os.path.join(self.jflow_config_reader.get_tmp_directory(), tempfile_name)
def get_component_output_directory(self, component_name, component_prefix):
return os.path.join(self.directory, component_name + "_" + component_prefix)
......
......@@ -46,4 +46,4 @@ class AddRun (Workflow):
# Then add the run to the project
my_project.add_run(my_run)
# archive the files
my_run.add_raw_files(files_to_save, self.args["archive_mode"])
\ No newline at end of file
addrawfiles = self.add_component("AddRawFiles", [my_run, files_to_save, self.args["archive_mode"]])
\ No newline at end of file
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
\ No newline at end of file
#
# Copyright (C) 2012 INRA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import pickle
from jflow.component import Component
from weaver.function import PythonFunction, ShellFunction
from weaver.abstraction import Merge, Map
from ng6.run import Run
def add_raw_files (run_dump_path, archive_mode, *files_to_save):
import pickle
# load the analysis object
run_dump = open(run_dump_path, "rb")
my_run = pickle.load(run_dump)
run_dump.close()
my_run.add_raw_files(files_to_save, archive_mode)
class AddRawFiles (Component):
def define_parameters(self, runobj, files_to_save, archive_mode):
self.runobj = runobj
self.files_to_save = files_to_save
self.archive_mode = archive_mode
self.stdout = os.path.join(self.output_directory, "addRawFiles.stdout")
def run(self):
run_dump_path = self.get_temporary_file(".dump")
run_dump = open(run_dump_path, "wb")
pickle.dump(self.runobj, run_dump)
run_dump.close()
addraw = PythonFunction(add_raw_files)
addraw(outputs=self.stdout, arguments=[run_dump_path, self.archive_mode, self.files_to_save])
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment