handle version 4.2.2

application.properties

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 [global]
-# uncomment and set if not in the PATH
+# uncomment and set if not in the PATH, should be version >= 4.2.2
 #makeflow = /usr/bin/makeflow
 # batch system type: local, condor, sge, moab, cluster, wq, hadoop, mpi-queue
 batch_system_type = local

src/jflow/workflow.py

@@ -85,7 +85,8 @@ class Workflow(threading.Thread):
         self.function = function
         self.parameters_section = parameters_section
         self.comp_pckg = self._import_components()
-        self.engine_arguments = ''
+        # intruduce --log-verbose to ba able to monitor the new version of makeflow >=4.2.2
+        self.engine_arguments = ' --log-verbose '
         self.component_nameids_is_init = False
         self.component_nameids = {}
         # try to parse engine arguments

src/weaver/__init__.py

@@ -17,4 +17,4 @@ from . import script
 from . import stack
 from . import util
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/abstraction.py

@@ -61,7 +61,7 @@ class Abstraction(Dataset):
         Dataset.__init__(self)
         self.symbol     = self.nest.symbol
         self.batch      = self.nest.batch
-
+        
         if collect:
             self.collect = parse_input_list(self.inputs)
         else:
@@ -127,12 +127,12 @@ class AllPairs(Abstraction):
             if self.native:
                 # Store inputs A and B lists as required by allpairs_master
                 inputs_a_file = next(self.nest.stash)
-                with open(inputs_a_file, 'w+') as fs:
+                with open(inputs_a_file, 'w') as fs:
                     for input_file in map(str, inputs_a):
                         fs.write(input_file + '\n')
 
                 inputs_b_file = next(self.nest.stash)
-                with open(inputs_b_file, 'w+') as fs:
+                with open(inputs_b_file, 'w') as fs:
                     for input_file in map(str, inputs_b):
                         fs.write(input_file + '\n')
 
@@ -352,4 +352,4 @@ class Merge(Abstraction):
 
 SENTINEL = Container(group=0)
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/compat.py

@@ -68,4 +68,4 @@ def compat_next(original_class):
     original_class.next = original_class.__next__
     return original_class
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/data.py

@@ -5,7 +5,7 @@
 """ Weaver data module """
 
 from weaver.logger  import D_DATA, debug
-from weaver.stack   import CurrentNest
+from weaver.stack   import CurrentNest, CurrentScript
 from weaver.util    import iterable, normalize_path, parse_object_list, \
                            parse_string_list, WeaverError
 
@@ -140,4 +140,4 @@ def parse_output_list(output_list=None, input_list=None):
     raise WeaverError(D_DATA,
         'Could not parse output argument: {0}'.format(output_list))
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/dataset.py

@@ -105,7 +105,7 @@ def cache_generation(method):
     """
     @functools.wraps(method)
     def wrapper(self):
-        with open(self.cache_path, 'w+') as fs:
+        with open(self.cache_path, 'w') as fs:
             for i in flatten(method(self)):
                 fs.write(i.path + '\n')
                 yield i
@@ -384,4 +384,4 @@ class Query(Dataset):
     def _generate(self):
         return self._dataset._query(self._filters, **self._parameters)
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/engine.py

@@ -88,11 +88,11 @@ class Makeflow(Engine):
                 self.dag_file.write('\t'.join(['', '# SYMBOL', str(function)]) + '\n')
             else:
                 self.dag_file.write('\t'.join(['', '# SYMBOL', str(abstraction)]) + '\n')
-        
+
         # if a symbol is provided
         if symbol:
-            self.dag_file.write('\t'.join(['', '# SYMBOL\t' + symbol]) + '\n')
-        
+            self.dag_file.write('@SYMBOL="' + symbol+'"\n')
+
         # Write environmental variables
         if options.local:
             self.dag_file.write('\t@BATCH_LOCAL=1\n')
@@ -155,4 +155,4 @@ class Makeflow(Engine):
             """
             raise RuntimeError('Failed to execute DAG {0} using {1}:\n{2}'.format(self.dag_path, self.path, e))
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/function.py

@@ -81,11 +81,13 @@ class Function(object):
 
         if local:
             options.local = True
+
         if environment:
             options.environment.update(environment)
+
         if nest.batch:
             options.batch = nest.batch
-        
+
         nest.schedule(abstraction, self, command,
             list(inputs) + list(includes), outputs, options, nest.symbol)
 
@@ -147,7 +149,7 @@ class ScriptFunction(Function):
         if executable is None:
             executable = next(CurrentNest().stash)
 
-        with open(executable, 'w+') as fs:
+        with open(executable, 'w') as fs:
             fs.write(source)
         os.chmod(executable, 0o755)
 
@@ -224,6 +226,7 @@ if __name__ == '__main__':
             imports.extend(function.func_imports)
         except AttributeError:
             pass
+        
         if add_path:
             add_path = add_path.extend(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
         else: 
@@ -232,6 +235,7 @@ if __name__ == '__main__':
         path = ""
         for apath in add_path:
             path += "sys.path.insert(0, '" + apath + "')\n"
+        
         source = self.PYTHON_TEMPLATE.format(', '.join(imports), path, body, name)
         ScriptFunction.__init__(self, source, executable, cmd_format)
 
@@ -301,4 +305,4 @@ class Pipeline(Function):
     def __str__(self):
         return self.cmd_format
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/logger.py

@@ -23,6 +23,7 @@ D_NEST        = 'nest'
 D_OPTIONS     = 'options'
 D_SCRIPT      = 'script'
 D_UTIL        = 'utility'
+D_USER        = 'user'
 D_ALL         = 'all'
 
 
@@ -119,7 +120,7 @@ class Logger(object):
         self.log('W', system, message)
 
     def set_log_path(self, path):
-        self.stream = open(path, 'w+')
+        self.stream = open(path, 'w')
 
 
 # Internal Weaver Logger instance
@@ -134,4 +135,4 @@ fatal         = _WeaverLogger.fatal
 warn          = _WeaverLogger.warn
 set_log_path  = _WeaverLogger.set_log_path
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/nest.py

@@ -196,7 +196,7 @@ class Nest(Makeflow):
             options, self))
 
         if abstraction is None:
-            abstraction = SENTINEL            
+            abstraction = SENTINEL
         self.tasks.append(
             [abstraction, function, command, inputs, outputs, options, symbol])
 
@@ -229,4 +229,4 @@ def Define(key, value, export=False):
 def Export(variables):
     CurrentNest().exports.update(parse_string_list(variables))
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/options.py

@@ -59,4 +59,4 @@ class Options(object):
         return 'Options(cpu={0}, memory={1}, disk={2}, batch={3}, local={4}, collect={5}, environment={6})'.format(
             self.cpu, self.memory, self.disk, self.batch, self.local, self.collect, self.environment)
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/script.py

@@ -47,6 +47,8 @@ class Script(object):
                 self.set_options(args.popleft().split(',')),
         '-I': lambda self, args:
                 setattr(self, 'import_builtins', False),
+        '-N': lambda self, args:
+                setattr(self, 'normalize_paths', False),
         '-d': lambda self, args:
                 weaver.logger.enable(args.popleft().split(',')),
         '-f': lambda self, args:
@@ -58,7 +60,7 @@ class Script(object):
         '-x': lambda self, args:
                 setattr(self, 'execute_dag', True),
         '-o': lambda self, args:
-                setattr(self, 'output_directory', os.path.abspath(str(args.popleft()))),
+                setattr(self, 'output_directory', str(args.popleft())),
         '-a': lambda self, args:
                 setattr(self, 'nested_abstractions', True),
         '-t': lambda self, args:
@@ -85,6 +87,7 @@ class Script(object):
         self.engine_wrapper      = None
         self.engine_arguments    = None
         self.include_symbols     = False
+        self.normalize_paths     = True
 
         args = collections.deque(args)
         while args:
@@ -99,6 +102,9 @@ class Script(object):
             except (IndexError, KeyError):
                 fatal(D_SCRIPT, 'invalid command line option: {0}'.format(arg))
 
+        if self.normalize_paths:
+            self.output_directory = os.path.abspath(self.output_directory)
+
         debug(D_SCRIPT, 'path                = {0}'.format(self.path))
         debug(D_SCRIPT, 'force               = {0}'.format(self.force))
         debug(D_SCRIPT, 'import_builtins     = {0}'.format(self.import_builtins))
@@ -110,6 +116,7 @@ class Script(object):
         debug(D_SCRIPT, 'execute_dag         = {0}'.format(self.execute_dag))
         debug(D_SCRIPT, 'engine_wrapper      = {0}'.format(self.engine_wrapper))
         debug(D_SCRIPT, 'engine_arguments    = {0}'.format(self.engine_arguments))
+        debug(D_SCRIPT, 'normalize_paths     = {0}'.format(self.normalize_paths))
 
         if self.path is None:
             self.show_usage()
@@ -150,6 +157,7 @@ General Options:
   -f              Ignore warnings.
   -g              Include debugging symbols in DAG.
   -I              Do not automatically import built-ins.
+  -N              Do not normalize paths.
   -b <options>    Set batch job options (cpu, memory, disk, batch, local, collect).
   -d <subsystem>  Enable debugging for subsystem.
   -l <log_path>   Set log path (default: stderr).
@@ -206,4 +214,4 @@ Subsystems:
                             nest.dag_path, nest.path))
                         nest.execute(self.engine_arguments, exit_on_failure=True)
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/stack.py

@@ -102,4 +102,4 @@ def CurrentScript():
         WeaverScripts.push(Script(sys.argv[0]))
     return WeaverScripts.top()
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file

src/weaver/test/__init__.py

@@ -4,4 +4,4 @@
 
 """ Weaver test sub-package """
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
\ No newline at end of file

src/weaver/test/logger.py

@@ -64,4 +64,4 @@ if __name__ == '__main__':
     test_suite  = TestLoader().loadTestsFromTestCase(LoggerTestCase)
     test_runner.run(test_suite)
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
\ No newline at end of file

src/weaver/test/util.py

@@ -89,4 +89,4 @@ if __name__ == '__main__':
     test_suite  = TestSuite(map(TestLoader().loadTestsFromTestCase, TestCases))
     test_runner.run(test_suite)
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
\ No newline at end of file

src/weaver/util.py

@@ -200,7 +200,7 @@ def find_executable(executable, find_dirs=None):
     them in `find_dirs`.
     """
     if os.path.isfile(executable):
-        return os.path.abspath(executable)
+        return normalize_path(executable)
 
     find_dirs = parse_string_list(find_dirs)
     find_dirs = itertools.chain(find_dirs,
@@ -220,6 +220,11 @@ def normalize_path(path, ref_path=None):
     If path is absolute or no `ref_path` is specified, then return absolute
     path.  Otherwise, return relative path.
     """
+    from weaver.stack import CurrentScript
+
+    if not CurrentScript().normalize_paths:
+        return path
+
     if os.path.isabs(path):
         return path
 
@@ -255,4 +260,4 @@ def type_str(obj, full=False):
     else:
         return obj_type.split("'")[1].split(".")[-1]
 
-# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
+# vim: set sts=4 sw=4 ts=8 expandtab ft=python:
\ No newline at end of file