make the config_parse able to parse a whole config file

bin/jflow_cli.py

@@ -29,6 +29,29 @@ from jflow.workflow import Workflow
 import jflow.utils as utils
 from cctools.util import time_format
 
+class JflowArgumentParser (argparse.ArgumentParser):
+    def _read_args_from_files(self, arg_strings):
+        # expand arguments referencing files
+        new_arg_strings = []
+        for arg_string in arg_strings:
+            # for regular arguments, just add them back into the list
+            if not arg_string or arg_string[0] not in self.fromfile_prefix_chars:
+                new_arg_strings.append(arg_string)
+            # replace arguments referencing files with the file content
+            else:
+                try:
+                    with open(arg_string[1:]) as args_file:
+                        arg_strings = []
+                        # give to the convert_arg_line_to_args a table of lines instead of line per line
+                        for arg in self.convert_arg_line_to_args(args_file.read().splitlines()):
+                            arg_strings.append(arg)
+                        arg_strings = self._read_args_from_files(arg_strings)
+                        new_arg_strings.extend(arg_strings)
+                except OSError:
+                    err = _sys.exc_info()[1]
+                    self.error(str(err))
+        # return the modified argument list
+        return new_arg_strings
 
 def display_workflow_status(workflow, detailed=False):
     if workflow.start_time: start_time = time.asctime(time.localtime(workflow.start_time))
@@ -81,7 +104,7 @@ if __name__ == '__main__':
     wfmanager = WorkflowsManager()
     
     # Create the top-level parser
-    parser = argparse.ArgumentParser(fromfile_prefix_chars='@')
+    parser = JflowArgumentParser(fromfile_prefix_chars='@')
     subparsers = parser.add_subparsers(title='Available sub commands')
     
     # Add rerun workflow availability

src/jflow/workflow.py

@@ -135,11 +135,12 @@ class Workflow(threading.Thread):
                 self.component_nameids[component_name+"."+component_prefix] = None
     
     @staticmethod
-    def config_parser(arg_line):
-        for arg in arg_line.split():
-            if not arg.strip():
-                continue
-            yield arg
+    def config_parser(arg_lines):
+        for arg_line in arg_lines:
+            for arg in arg_line.split():
+                if not arg.strip():
+                    continue
+                yield arg
     
     def __setstate__(self, state):
         self.__dict__ = state.copy()