Commit dc242d4e authored by Jerome Mariette's avatar Jerome Mariette
Browse files

handle version 4.2.2

parent ff940d3d
......@@ -15,7 +15,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
[global]
# uncomment and set if not in the PATH
# uncomment and set if not in the PATH, should be version >= 4.2.2
#makeflow = /usr/bin/makeflow
# batch system type: local, condor, sge, moab, cluster, wq, hadoop, mpi-queue
batch_system_type = local
......
......@@ -85,7 +85,8 @@ class Workflow(threading.Thread):
self.function = function
self.parameters_section = parameters_section
self.comp_pckg = self._import_components()
self.engine_arguments = ''
# intruduce --log-verbose to ba able to monitor the new version of makeflow >=4.2.2
self.engine_arguments = ' --log-verbose '
self.component_nameids_is_init = False
self.component_nameids = {}
# try to parse engine arguments
......
......@@ -127,12 +127,12 @@ class AllPairs(Abstraction):
if self.native:
# Store inputs A and B lists as required by allpairs_master
inputs_a_file = next(self.nest.stash)
with open(inputs_a_file, 'w+') as fs:
with open(inputs_a_file, 'w') as fs:
for input_file in map(str, inputs_a):
fs.write(input_file + '\n')
inputs_b_file = next(self.nest.stash)
with open(inputs_b_file, 'w+') as fs:
with open(inputs_b_file, 'w') as fs:
for input_file in map(str, inputs_b):
fs.write(input_file + '\n')
......
......@@ -5,7 +5,7 @@
""" Weaver data module """
from weaver.logger import D_DATA, debug
from weaver.stack import CurrentNest
from weaver.stack import CurrentNest, CurrentScript
from weaver.util import iterable, normalize_path, parse_object_list, \
parse_string_list, WeaverError
......
......@@ -105,7 +105,7 @@ def cache_generation(method):
"""
@functools.wraps(method)
def wrapper(self):
with open(self.cache_path, 'w+') as fs:
with open(self.cache_path, 'w') as fs:
for i in flatten(method(self)):
fs.write(i.path + '\n')
yield i
......
......@@ -91,7 +91,7 @@ class Makeflow(Engine):
# if a symbol is provided
if symbol:
self.dag_file.write('\t'.join(['', '# SYMBOL\t' + symbol]) + '\n')
self.dag_file.write('@SYMBOL="' + symbol+'"\n')
# Write environmental variables
if options.local:
......
......@@ -81,8 +81,10 @@ class Function(object):
if local:
options.local = True
if environment:
options.environment.update(environment)
if nest.batch:
options.batch = nest.batch
......@@ -147,7 +149,7 @@ class ScriptFunction(Function):
if executable is None:
executable = next(CurrentNest().stash)
with open(executable, 'w+') as fs:
with open(executable, 'w') as fs:
fs.write(source)
os.chmod(executable, 0o755)
......@@ -224,6 +226,7 @@ if __name__ == '__main__':
imports.extend(function.func_imports)
except AttributeError:
pass
if add_path:
add_path = add_path.extend(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
else:
......@@ -232,6 +235,7 @@ if __name__ == '__main__':
path = ""
for apath in add_path:
path += "sys.path.insert(0, '" + apath + "')\n"
source = self.PYTHON_TEMPLATE.format(', '.join(imports), path, body, name)
ScriptFunction.__init__(self, source, executable, cmd_format)
......
......@@ -23,6 +23,7 @@ D_NEST = 'nest'
D_OPTIONS = 'options'
D_SCRIPT = 'script'
D_UTIL = 'utility'
D_USER = 'user'
D_ALL = 'all'
......@@ -119,7 +120,7 @@ class Logger(object):
self.log('W', system, message)
def set_log_path(self, path):
self.stream = open(path, 'w+')
self.stream = open(path, 'w')
# Internal Weaver Logger instance
......
......@@ -47,6 +47,8 @@ class Script(object):
self.set_options(args.popleft().split(',')),
'-I': lambda self, args:
setattr(self, 'import_builtins', False),
'-N': lambda self, args:
setattr(self, 'normalize_paths', False),
'-d': lambda self, args:
weaver.logger.enable(args.popleft().split(',')),
'-f': lambda self, args:
......@@ -58,7 +60,7 @@ class Script(object):
'-x': lambda self, args:
setattr(self, 'execute_dag', True),
'-o': lambda self, args:
setattr(self, 'output_directory', os.path.abspath(str(args.popleft()))),
setattr(self, 'output_directory', str(args.popleft())),
'-a': lambda self, args:
setattr(self, 'nested_abstractions', True),
'-t': lambda self, args:
......@@ -85,6 +87,7 @@ class Script(object):
self.engine_wrapper = None
self.engine_arguments = None
self.include_symbols = False
self.normalize_paths = True
args = collections.deque(args)
while args:
......@@ -99,6 +102,9 @@ class Script(object):
except (IndexError, KeyError):
fatal(D_SCRIPT, 'invalid command line option: {0}'.format(arg))
if self.normalize_paths:
self.output_directory = os.path.abspath(self.output_directory)
debug(D_SCRIPT, 'path = {0}'.format(self.path))
debug(D_SCRIPT, 'force = {0}'.format(self.force))
debug(D_SCRIPT, 'import_builtins = {0}'.format(self.import_builtins))
......@@ -110,6 +116,7 @@ class Script(object):
debug(D_SCRIPT, 'execute_dag = {0}'.format(self.execute_dag))
debug(D_SCRIPT, 'engine_wrapper = {0}'.format(self.engine_wrapper))
debug(D_SCRIPT, 'engine_arguments = {0}'.format(self.engine_arguments))
debug(D_SCRIPT, 'normalize_paths = {0}'.format(self.normalize_paths))
if self.path is None:
self.show_usage()
......@@ -150,6 +157,7 @@ General Options:
-f Ignore warnings.
-g Include debugging symbols in DAG.
-I Do not automatically import built-ins.
-N Do not normalize paths.
-b <options> Set batch job options (cpu, memory, disk, batch, local, collect).
-d <subsystem> Enable debugging for subsystem.
-l <log_path> Set log path (default: stderr).
......
......@@ -200,7 +200,7 @@ def find_executable(executable, find_dirs=None):
them in `find_dirs`.
"""
if os.path.isfile(executable):
return os.path.abspath(executable)
return normalize_path(executable)
find_dirs = parse_string_list(find_dirs)
find_dirs = itertools.chain(find_dirs,
......@@ -220,6 +220,11 @@ def normalize_path(path, ref_path=None):
If path is absolute or no `ref_path` is specified, then return absolute
path. Otherwise, return relative path.
"""
from weaver.stack import CurrentScript
if not CurrentScript().normalize_paths:
return path
if os.path.isabs(path):
return path
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment