Implemented live results in a subfolder of the output directory

bin/nextflow.py

 from string import Template
 import os.path
 import os
+from copy import deepcopy
 
 analysis_template = Template ('''
 process ${id} {
@@ -16,6 +17,37 @@ process ${id} {
     """
 }
 ''')
+live_results_template = Template('''
+process generate_${id}_live_results {
+
+    publishDir "${output}/live", mode: 'copy', pattern: '*????????-????-????-????-????????????.json'
+
+    input:
+    file result from ${id}_json_live
+
+    output:
+    file "*.json" into ${id}_live_results
+
+    script:
+    """
+    split_json_into_separate_files.py --json $$result --output . --uuid
+    """
+}
+''')
+convert_live_template = Template ('''
+process convert_${id}_to_json {
+    input:
+    file result from ${id}_results
+
+    output:
+    file "$${result}.json" into ${id}_json, ${id}_json_live
+
+    script:
+    """
+    ${converter_script} --result $$result --output $${result}.json
+    """
+}
+''')
 convert_template = Template ('''
 process convert_${id}_to_json {
     input:
@@ -95,7 +127,13 @@ Channel.fromPath(params.fasta).set{fasta}''')
 
     for m in modules:
         fragments.append(analysis_template.substitute(m))
-        fragments.append(convert_template.substitute(m))
+        if execution['mode'] == 'live':
+            fragments.append(convert_live_template.substitute(m))
+            copy = deepcopy(m)
+            copy['output'] = execution['output']
+            fragments.append(live_results_template.substitute(copy))
+        else:
+            fragments.append(convert_template.substitute(m))
 
     json_inputs = []
     for m in modules:

bin/psot

@@ -20,7 +20,7 @@ def main():
     analyze_parser.add_argument('--fasta', '-f', required=True, help='A fasta file with aminoacid sequences')
     analyze_parser.add_argument('--output', '-o', required=True, help='The output directory for the json documents')
     analyze_parser.add_argument('--profile', '-p', default='fast', help='The profile to use')
-    analyze_parser.add_argument('--live', '-l', action='store_true', help='Report results as they are computed, not only at the end of the computation')
+    analyze_parser.add_argument('--live', '-l', action='store_true', help='Report results as they are computed, not only at the end of the computation. The live results will be available in the $output/live.')
     analyze_parser.add_argument('--config', '-c', help='The config to use')
     analyze_parser.add_argument('--debug', '-d', action='store_true', help='Debug mode, computation directory will not be removed after computation')
     analyze_parser.add_argument('--execution_dir', '-e', help='Use the specified execution directory and do not delete it after the computation')

bin/split_json_into_separate_files.py

@@ -4,9 +4,11 @@ import json
 import argparse
 import os
 import os.path
+from uuid import uuid4
 
 parser = argparse.ArgumentParser(description='Split a json file with multiple documents into single files per document')
 parser.add_argument('--json', '-j', required=True, help='json documents')
+parser.add_argument('--uuid', '-u', action='store_true', help='Include a uuid in the filename.')
 parser.add_argument('--output', '-o', required=True, help='The name of the output directory')
 args = parser.parse_args()
 
@@ -22,6 +24,9 @@ file = args.json
 with open(file) as f:
     doc = json.load(f)
     for k in doc:
-        output_filename = k + '.json'
+        if args.uuid:
+            output_filename = k + "_" + str(uuid4()) + '.json'
+        else:
+            output_filename = k + '.json'
         with open(output_dir + '/' + output_filename, 'w') as o:
             json.dump(doc[k], o)