handle version 4.2.2

application.properties

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 [global]
-# uncomment and set if not in the PATH
+# uncomment and set if not in the PATH, should be version >= 4.2.2
 #makeflow = /usr/bin/makeflow
 # batch system type: local, condor, sge, moab, cluster, wq, hadoop, mpi-queue
 batch_system_type = local

src/jflow/workflow.py

@@ -85,7 +85,8 @@ class Workflow(threading.Thread):
         self.function = function
         self.parameters_section = parameters_section
         self.comp_pckg = self._import_components()
-        self.engine_arguments = ''
+        # intruduce --log-verbose to ba able to monitor the new version of makeflow >=4.2.2
+        self.engine_arguments = ' --log-verbose '
         self.component_nameids_is_init = False
         self.component_nameids = {}
         # try to parse engine arguments

src/weaver/abstraction.py

@@ -127,12 +127,12 @@ class AllPairs(Abstraction):
             if self.native:
                 # Store inputs A and B lists as required by allpairs_master
                 inputs_a_file = next(self.nest.stash)
-                with open(inputs_a_file, 'w+') as fs:
+                with open(inputs_a_file, 'w') as fs:
                     for input_file in map(str, inputs_a):
                         fs.write(input_file + '\n')
 
                 inputs_b_file = next(self.nest.stash)
-                with open(inputs_b_file, 'w+') as fs:
+                with open(inputs_b_file, 'w') as fs:
                     for input_file in map(str, inputs_b):
                         fs.write(input_file + '\n')
 

src/weaver/data.py

@@ -5,7 +5,7 @@
 """ Weaver data module """
 
 from weaver.logger  import D_DATA, debug
-from weaver.stack   import CurrentNest
+from weaver.stack   import CurrentNest, CurrentScript
 from weaver.util    import iterable, normalize_path, parse_object_list, \
                            parse_string_list, WeaverError
 

src/weaver/dataset.py

@@ -105,7 +105,7 @@ def cache_generation(method):
     """
     @functools.wraps(method)
     def wrapper(self):
-        with open(self.cache_path, 'w+') as fs:
+        with open(self.cache_path, 'w') as fs:
             for i in flatten(method(self)):
                 fs.write(i.path + '\n')
                 yield i

src/weaver/engine.py

@@ -91,7 +91,7 @@ class Makeflow(Engine):
 
         # if a symbol is provided
         if symbol:
-            self.dag_file.write('\t'.join(['', '# SYMBOL\t' + symbol]) + '\n')
+            self.dag_file.write('@SYMBOL="' + symbol+'"\n')
 
         # Write environmental variables
         if options.local:

src/weaver/function.py

@@ -81,8 +81,10 @@ class Function(object):
 
         if local:
             options.local = True
+
         if environment:
             options.environment.update(environment)
+
         if nest.batch:
             options.batch = nest.batch
 
@@ -147,7 +149,7 @@ class ScriptFunction(Function):
         if executable is None:
             executable = next(CurrentNest().stash)
 
-        with open(executable, 'w+') as fs:
+        with open(executable, 'w') as fs:
             fs.write(source)
         os.chmod(executable, 0o755)
 
@@ -224,6 +226,7 @@ if __name__ == '__main__':
             imports.extend(function.func_imports)
         except AttributeError:
             pass
+        
         if add_path:
             add_path = add_path.extend(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
         else: 
@@ -232,6 +235,7 @@ if __name__ == '__main__':
         path = ""
         for apath in add_path:
             path += "sys.path.insert(0, '" + apath + "')\n"
+        
         source = self.PYTHON_TEMPLATE.format(', '.join(imports), path, body, name)
         ScriptFunction.__init__(self, source, executable, cmd_format)
 

src/weaver/logger.py

@@ -23,6 +23,7 @@ D_NEST        = 'nest'
 D_OPTIONS     = 'options'
 D_SCRIPT      = 'script'
 D_UTIL        = 'utility'
+D_USER        = 'user'
 D_ALL         = 'all'
 
 
@@ -119,7 +120,7 @@ class Logger(object):
         self.log('W', system, message)
 
     def set_log_path(self, path):
-        self.stream = open(path, 'w+')
+        self.stream = open(path, 'w')
 
 
 # Internal Weaver Logger instance

src/weaver/script.py

@@ -47,6 +47,8 @@ class Script(object):
                 self.set_options(args.popleft().split(',')),
         '-I': lambda self, args:
                 setattr(self, 'import_builtins', False),
+        '-N': lambda self, args:
+                setattr(self, 'normalize_paths', False),
         '-d': lambda self, args:
                 weaver.logger.enable(args.popleft().split(',')),
         '-f': lambda self, args:
@@ -58,7 +60,7 @@ class Script(object):
         '-x': lambda self, args:
                 setattr(self, 'execute_dag', True),
         '-o': lambda self, args:
-                setattr(self, 'output_directory', os.path.abspath(str(args.popleft()))),
+                setattr(self, 'output_directory', str(args.popleft())),
         '-a': lambda self, args:
                 setattr(self, 'nested_abstractions', True),
         '-t': lambda self, args:
@@ -85,6 +87,7 @@ class Script(object):
         self.engine_wrapper      = None
         self.engine_arguments    = None
         self.include_symbols     = False
+        self.normalize_paths     = True
 
         args = collections.deque(args)
         while args:
@@ -99,6 +102,9 @@ class Script(object):
             except (IndexError, KeyError):
                 fatal(D_SCRIPT, 'invalid command line option: {0}'.format(arg))
 
+        if self.normalize_paths:
+            self.output_directory = os.path.abspath(self.output_directory)
+
         debug(D_SCRIPT, 'path                = {0}'.format(self.path))
         debug(D_SCRIPT, 'force               = {0}'.format(self.force))
         debug(D_SCRIPT, 'import_builtins     = {0}'.format(self.import_builtins))
@@ -110,6 +116,7 @@ class Script(object):
         debug(D_SCRIPT, 'execute_dag         = {0}'.format(self.execute_dag))
         debug(D_SCRIPT, 'engine_wrapper      = {0}'.format(self.engine_wrapper))
         debug(D_SCRIPT, 'engine_arguments    = {0}'.format(self.engine_arguments))
+        debug(D_SCRIPT, 'normalize_paths     = {0}'.format(self.normalize_paths))
 
         if self.path is None:
             self.show_usage()
@@ -150,6 +157,7 @@ General Options:
   -f              Ignore warnings.
   -g              Include debugging symbols in DAG.
   -I              Do not automatically import built-ins.
+  -N              Do not normalize paths.
   -b <options>    Set batch job options (cpu, memory, disk, batch, local, collect).
   -d <subsystem>  Enable debugging for subsystem.
   -l <log_path>   Set log path (default: stderr).

src/weaver/util.py

@@ -200,7 +200,7 @@ def find_executable(executable, find_dirs=None):
     them in `find_dirs`.
     """
     if os.path.isfile(executable):
-        return os.path.abspath(executable)
+        return normalize_path(executable)
 
     find_dirs = parse_string_list(find_dirs)
     find_dirs = itertools.chain(find_dirs,
@@ -220,6 +220,11 @@ def normalize_path(path, ref_path=None):
     If path is absolute or no `ref_path` is specified, then return absolute
     path.  Otherwise, return relative path.
     """
+    from weaver.stack import CurrentScript
+
+    if not CurrentScript().normalize_paths:
+        return path
+
     if os.path.isabs(path):
         return path