Refactor config into repository and application config.

psot/config.py

@@ -3,124 +3,132 @@ import yaml
 import copy
 import pkg_resources
 
-def get_install_location():
-    """Finds the location directory of the tool"""
-    script_path = os.path.realpath(__file__)
-    script_dir = os.path.dirname(script_path)
-    install_dir = os.path.dirname(script_dir)
-    return install_dir
-
-def get_script_location():
-    return os.path.join(get_install_location(),'bin')
-
-def get_scripts_location():
-    return pkg_resources.resource_filename('psot', 'scripts')
-
-def get_template_location():
-    return pkg_resources.resource_filename('psot', 'templates')
-
-def get_modules_location():
-    return pkg_resources.resource_filename('psot', 'modules')
-
-def get_profiles_location():
-    return pkg_resources.resource_filename('psot', 'profiles')
-
-def get_config_file():
-    return pkg_resources.resource_filename('psot', 'config.yaml')
-
-def load_config_file():
-    config = {}
-    with (open(get_config_file())) as f:
-        config = yaml.load(f)
-    return config
-
-def get_module_manifests():
-    mypath = get_modules_location()
-    return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
-
-def get_profile_manifests():
-    mypath = get_profiles_location()
-    return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
-
-def create_app_config():
-    config = {}
-    config['app_path'] = get_install_location()
-    config['bin_path'] = get_script_location()
-    config['script_path'] = get_scripts_location()
-    return config
-
-def load_profiles():
-    profiles = []
-    for manifest in get_profile_manifests():
-        with (open(manifest)) as f:
-            profile = yaml.load(f)
-            profiles.append(profile)
-
-    return profiles
-
-        
-def load_modules():
-    modules = {}
-    for manifest in get_module_manifests():
-        with (open(manifest)) as f:
-            module = yaml.load(f)
-            modules[module['name']] = module
-            if 'parameters' not in module['analysis']:
-                module['analysis']['parameters'] = {}
-            if 'parameters' not in module['converter']:
-                module['converter']['parameters'] = {}
-            if module['analysis']['parameters'] is None:
-                module['analysis']['parameters'] = {}
-            if module['converter']['parameters'] is None:
-                module['converter']['parameters'] = {}
-    return modules
-
-def normalize_profiles(profiles):
-    """take care that all keys of the profile map exist"""
-    for profile in profiles:
-        if 'modules' not in profile:
-            profile['modules'] = []
-        for module in profile['modules']:
-            if profile['modules'][module] is None:
-                profile['modules'][module] = {}
-
-def merge_modules_in_profiles(config):
-    """Merge the module configuration and the profile configuration. Profile has precedence over module."""
-    profiles = config['profiles']
-    modules = config['modules']
-    for profile in profiles:
-        resolved_modules = []
-        for module_name in profile['modules']:
-            profile_module_config = profile['modules'][module_name]
-            resolved_module = modules[module_name]
-            copied_module = copy.deepcopy(resolved_module)
-            for c in profile_module_config:
-                copied_module['analysis']['parameters'][c] = profile_module_config[c]
-            resolved_modules.append(copied_module)
-        profile['modules'] = resolved_modules
+class Repository:
+
+  def __init__(self, path):
+    self.path = path
+
+  def load_config(self):
+      '''return a dictionary with the whole configuration for this repository'''
+      config = self.load_config_file()
+      print(config)
+      config['profiles'] = self.load_profiles()
+      config['modules'] = self.load_modules()
+      self.normalize_profiles(config['profiles'])
+      self.merge_modules_in_profiles(config)
+      return config
+
+  def get_module_manifests(self):
+      mypath = self.get_modules_location()
+      return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
+
+  def get_profile_manifests(self):
+      mypath = self.get_profiles_location()
+      return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
+
+  def get_modules_location(self):
+      return os.path.join(self.path, 'modules')
+
+  def get_profiles_location(self):
+      return os.path.join(self.path, 'profiles')
+
+  def get_config_file(self):
+      return os.path.join(self.path, 'config.yaml')
+
+  def get_scripts_location(self):
+      return pkg_resources.resource_filename('psot', 'scripts')
+
+  def load_config_file(self):
+      config = {}
+      with (open(self.get_config_file())) as f:
+          config = yaml.load(f)
+      return config
+
+  def load_profiles(self):
+      profiles = []
+      for manifest in self.get_profile_manifests():
+          with (open(manifest)) as f:
+              profile = yaml.load(f)
+              profiles.append(profile)
+
+      return profiles
+          
+  def load_modules(self):
+      modules = {}
+      for manifest in self.get_module_manifests():
+          with (open(manifest)) as f:
+              module = yaml.load(f)
+              modules[module['name']] = module
+              if 'parameters' not in module['analysis']:
+                  module['analysis']['parameters'] = {}
+              if 'parameters' not in module['converter']:
+                  module['converter']['parameters'] = {}
+              if module['analysis']['parameters'] is None:
+                  module['analysis']['parameters'] = {}
+              if module['converter']['parameters'] is None:
+                  module['converter']['parameters'] = {}
+      return modules
+
+  def normalize_profiles(self, profiles):
+      """take care that all keys of the profile map exist"""
+      for profile in profiles:
+          if 'modules' not in profile:
+              profile['modules'] = []
+          for module in profile['modules']:
+              if profile['modules'][module] is None:
+                  profile['modules'][module] = {}
+
+  def merge_modules_in_profiles(self, config):
+      """Merge the module configuration and the profile configuration. Profile has precedence over module."""
+      profiles = config['profiles']
+      modules = config['modules']
+      for profile in profiles:
+          resolved_modules = []
+          for module_name in profile['modules']:
+              profile_module_config = profile['modules'][module_name]
+              resolved_module = modules[module_name]
+              copied_module = copy.deepcopy(resolved_module)
+              for c in profile_module_config:
+                  copied_module['analysis']['parameters'][c] = profile_module_config[c]
+              resolved_modules.append(copied_module)
+          profile['modules'] = resolved_modules
+
+class Config:
+
+  def __init__(self, repository_paths):
+    self.repo_paths = repository_paths
+
+  def merge_config(self, source, target):
+    # Quick hack: replacement instead of merge
+    # TODO do an actual merge instead of just a replacement
+    for k in source:
+      target[k] = source[k]
+    return
+
+  def load_config(self):
+      config = {}
+      config['app'] = {}
+      for repo_path in self.repo_paths:
+        repo = Repository(repo_path)
+        repo_config = repo.load_config()
+        self.merge_config(repo_config, config)
+        # quick hack
+        # use the last script directory
+        # TODO replace with list
+        config['app']['script_path'] = repo.get_scripts_location()
+
+      config['install_path'] = pkg_resources.resource_filename('psot', '')
+      config['repository_paths'] = self.repo_paths
+      return config
 
 def load_config():
-    config = load_config_file()
-    config['profiles'] = load_profiles()
-    config['modules'] = load_modules()
-    config['app'] = create_app_config()
-    normalize_profiles(config['profiles'])
-    merge_modules_in_profiles(config)
-    return config
+    return Config([pkg_resources.resource_filename('psot', '')]).load_config()
 
 if __name__ == "__main__":
     """prints the configuration"""
-    import config
     from pprint import pformat
+    config = Config([pkg_resources.resource_filename('psot', '')])
 
-    print('Install location: ' + config.get_install_location())
-    print('Script lcoation: ' + config.get_script_location())
-    print('Config location: ' + config.get_config_file())
-    print('Available profile manifests:\n\t' + '\n\t'.join(config.get_profile_manifests()) + '\n')
-    print('Available module manifests:\n\t' + '\n\t'.join(config.get_module_manifests()) + '\n')
-    print('Profile manifests content:\n' + pformat(config.load_profiles()) + '\n')
-    print('Module manifests content:\n' + pformat(config.load_modules()) + '\n')
-    print('Config content:\n' + pformat(config.load_config_file()) + '\n')
-
+    print('Install location: ' + str(config.repo_paths))
     print('Aggregated config:\n' + pformat(config.load_config()) + '\n')
 

psot/main.py

@@ -62,9 +62,7 @@ def generate_execution(config, args):
     execution['use_cluster'] = args.use_cluster
     execution['fetch_informations'] = args.fetch_informations 
     execution['mode'] = 'live' if args.live else 'complete'
-    execution['bin_path'] = config['app']['bin_path']
     execution['script_path'] = config['app']['script_path']
-    execution['psot_path'] = os.path.abspath(os.path.join(os.path.dirname(config['app']['bin_path']), 'psot'))
     execution['fasta'] = os.path.abspath(args.fasta)
     execution['output'] = os.path.abspath(args.output)
     if args.execution_dir:

psot/nextflow.py

@@ -167,8 +167,8 @@ def setup_execution_directory(execution):
 
     if not os.path.exists(directory + '/bin'):
         os.symlink(execution['script_path'], directory + '/bin')
-    if not os.path.exists(directory + '/psot'):
-        os.symlink(execution['psot_path'], directory + '/psot')
+    #if not os.path.exists(directory + '/psot'):
+    #    os.symlink(execution['psot_path'], directory + '/psot')
 
 def execute_analysis(execution):
     old_cwd = os.getcwd()

tests/data/example_repo/config.yaml

+tools:

tests/data/example_repo/modules/example.yaml

+name: 'example'
+info: 'example module for testing'
+analysis:
+    script: 'run_example.sh'
+    parameters:
+converter:
+    script: 'convert_example.sh'
+    parameters:

tests/data/example_repo/profiles/test.yaml

+name: 'test'
+info: 'Profile for testing purposes'
+modules:
+  example:

tests/test_repository.py

+import unittest
+import pkg_resources
+from psot import config
+
+class TestExampleRepositoryIsLoadedCorrectly(unittest.TestCase):
+  def setUp(self):
+    repo_path = pkg_resources.resource_filename('tests', 'data/example_repo/')
+    self.repository = config.Repository(repo_path)
+    self.config = self.repository.load_config()
+
+  def test_that_profile_exists(self):
+    # check that a profile with the name 'test' exists in the list of profiles
+    for profile in self.config['profiles']:
+      if 'test' == profile['name']:
+        return
+    self.fail('Profile test not found')
+
+  def test_that_module_exits(self):
+    # check that a module with the name 'example' exists in the list of modules
+    self.assertTrue('example' in self.config['modules'], 'Module example not found')
+
+  def test_that_scripts_are_resolved_correctly(self):
+    self.fail('not implemented yet')
+
+  def test_that_config_exists(self):
+    self.fail('not implemented yet')