Modified module manifest structure to allow parameters for converter scripts

CHANGELOG

+[0.2]
+* Changed module manifest format to allow parameters for converters
+* Hidden files in modules directory are ignored
+
 [0.1]
 * Implemented psot runner script
 * Implemented psot configuration

bin/config.py

@@ -46,8 +46,14 @@ def load_modules():
         with (open(manifest)) as f:
             module = yaml.load(f)
             modules[module['name']] = module
-            if 'parameters' not in module:
-                module['parameters'] = {}
+            if 'parameters' not in module['analysis']:
+                module['analysis']['parameters'] = {}
+            if 'parameters' not in module['converter']:
+                module['converter']['parameters'] = {}
+            if module['analysis']['parameters'] is None:
+                module['analysis']['parameters'] = {}
+            if module['converter']['parameters'] is None:
+                module['converter']['parameters'] = {}
     return modules
 
 def normalize_profiles(profiles):
@@ -70,7 +76,7 @@ def merge_modules_in_profiles(config):
             resolved_module = modules[module_name]
             copied_module = copy.deepcopy(resolved_module)
             for c in profile_module_config:
-                copied_module['parameters'][c] = profile_module_config[c]
+                copied_module['analysis']['parameters'][c] = profile_module_config[c]
             resolved_modules.append(copied_module)
         profile['modules'] = resolved_modules
 

bin/nextflow.py

@@ -2,6 +2,18 @@ from string import Template
 import os.path
 import os
 from copy import deepcopy
+import collections
+
+# taken from https://stackoverflow.com/questions/6027558/flatten-nested-python-dictionaries-compressing-keys
+def flatten(d, parent_key='', sep='_'):
+    items = []
+    for k, v in d.items():
+        new_key = parent_key + sep + k if parent_key else k
+        if isinstance(v, collections.MutableMapping):
+            items.extend(flatten(v, new_key, sep=sep).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
 
 analysis_template = Template ('''
 process ${id} {
@@ -13,7 +25,7 @@ process ${id} {
 
     script:
     """
-    ${analysis_script} --fasta $$fasta --output $${fasta}.${id}.results ${params}
+    ${analysis_script} --fasta $$fasta --output $${fasta}.${id}.results ${analysis_params}
     """
 }
 ''')
@@ -44,7 +56,7 @@ process convert_${id}_to_json {
 
     script:
     """
-    ${converter_script} --result $$result --output $${result}.json
+    ${converter_script} --result $$result --output $${result}.json ${converter_params}
     """
 }
 ''')
@@ -58,7 +70,7 @@ process convert_${id}_to_json {
 
     script:
     """
-    ${converter_script} --result $$result --output $${result}.json
+    ${converter_script} --result $$result --output $${result}.json ${converter_params}
     """
 }
 ''')
@@ -126,14 +138,14 @@ Channel.fromPath(params.fasta).set{fasta}''')
     fragments.append('fasta.into{'+';'.join(target_channels)+';}')
 
     for m in modules:
-        fragments.append(analysis_template.substitute(m))
+        fragments.append(analysis_template.substitute(flatten(m)))
         if execution['mode'] == 'live':
-            fragments.append(convert_live_template.substitute(m))
+            fragments.append(convert_live_template.substitute(flatten(m)))
             copy = deepcopy(m)
             copy['output'] = execution['output']
-            fragments.append(live_results_template.substitute(copy))
+            fragments.append(live_results_template.substitute(flatten(copy)))
         else:
-            fragments.append(convert_template.substitute(m))
+            fragments.append(convert_template.substitute(flatten(m)))
 
     json_inputs = []
     for m in modules:

bin/psot

@@ -66,7 +66,8 @@ def generate_execution_modules_for_profile(config, profile):
     # generate unique ids for each module
     for module in modules:
         module['id'] = module['name']
-        module['params'] = generate_params_string(module['parameters'])
+        module['analysis']['params'] = generate_params_string(module['analysis']['parameters'])
+        module['converter']['params'] = generate_params_string(module['converter']['parameters'])
     return modules
 
 def generate_params_string(options):

modules/blast_vs_swissprot.yaml

@@ -8,12 +8,14 @@ name: 'blastp_swissprot'
 info: 'blastp analysis against swissprot'
 
 # The name of the script for the analysis step. Must take a --fasta parameter
-analysis_script: 'run_blastp.py'
+analysis:
+    script: 'run_blastp.py'
+    parameters:
+        database: '/vol/biodb/uniprot/uniprot_sprot.fasta'
+        evalue: 1e-10
 
 # The name of the result to json converter script. Must take one parameter, the
 # result file from the analysis_script
-converter_script: 'convert_blastp.py'
-
-parameters:
-    database: '/vol/biodb/uniprot/uniprot_sprot.fasta'
-    evalue: 1e-10
+converter:
+    script: 'convert_blastp.py'
+    parameters:

modules/ghostx_vs_swissprot.yaml

-# Module manifest for the blastp against swissprot analysis
+# Module manifest for the ghostx against swissprot analysis
 
 # The name of the module. Is needed for the list-analyses option, for custom
 # configurations and custom profiles
@@ -8,11 +8,13 @@ name: 'ghostx_swissprot'
 info: 'ghostx analysis against swissprot'
 
 # The name of the script for the analysis step. Must take a --fasta parameter
-analysis_script: 'run_ghostx.py'
+analysis:
+    script: 'run_ghostx.py'
+    parameters:
+        database: '/vol/biodb/ghostx/uniprot_sprot'
 
 # The name of the result to json converter script. Must take one parameter, the
 # result file from the analysis_script
-converter_script: 'convert_ghostx.py'
+converter:
+    script: 'convert_ghostx.py'
 
-parameters:
-    database: '/vol/biodb/ghostx/uniprot_sprot'

modules/hmmer_vs_PFAM-A.yaml

@@ -8,12 +8,13 @@ name: 'hmmer_pfam_a'
 info: 'hmmscan analysis against PFAM-A'
 
 # The name of the script for the analysis step. Must take a --fasta parameter
-analysis_script: 'run_hmmer.py'
+analysis:
+    script: 'run_hmmer.py'
+    parameters:
+        database: '/vol/biodb/pfam/Pfam-A.hmm'
+        evalue: 1e-10
 
 # The name of the result to json converter script. Must take one parameter, the
 # result file from the analysis_script
-converter_script: 'convert_hmmer.py'
-
-parameters:
-    database: '/vol/biodb/pfam/Pfam-A.hmm'
-    evalue: 1e-10
+converter:
+    script: 'convert_hmmer.py'

modules/signalp.yaml

@@ -7,9 +7,16 @@ name: 'signalp'
 # Short description of the analysis
 info: 'predict signal peptides'
 
-# The name of the script for the analysis step. Must take a --fasta parameter
-analysis_script: 'run_signalp.py'
+# The configuration of the script for the analysis step. 
+analysis:
+    # script must take a --fasta parameter
+    script: 'run_signalp.py'
+    # specify additional default configuration here
+    parameters:
 
-# The name of the result to json converter script. Must take one parameter, the
-# result file from the analysis_script
-converter_script: 'convert_signalp.py'
+# The configuration of the script for the json conversion step. 
+converter:
+    # script must take a --result parameter, which is the result from the analysis step
+    script: 'convert_signalp.py'
+    # specify additional default configuration here
+    parameters: