Implement support for accessing modules from other repositories in profiles

psot/config.py

@@ -14,33 +14,31 @@ class Repository:
       config['profiles'] = self.load_profiles()
       config['modules'] = self.load_modules()
       self.normalize_profiles(config['profiles'])
-      self.merge_modules_in_profiles(config)
       return config
 
   def get_module_manifests(self):
-      mypath = self.get_modules_location()
-      return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
+      mypath = os.path.join(self.path, 'modules')
+      if os.path.exists(mypath):
+        return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
+      else:
+        return []
 
   def get_profile_manifests(self):
-      mypath = self.get_profiles_location()
-      return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
-
-  def get_modules_location(self):
-      return os.path.join(self.path, 'modules')
-
-  def get_profiles_location(self):
-      return os.path.join(self.path, 'profiles')
-
-  def get_config_file(self):
-      return os.path.join(self.path, 'config.yaml')
+      mypath = os.path.join(self.path, 'profiles')
+      if os.path.exists(mypath):
+        return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
+      else:
+        return []
 
   def get_scripts_location(self):
       return os.path.join(self.path, 'scripts')
 
   def load_config_file(self):
       config = {}
-      with (open(self.get_config_file())) as f:
-          config = yaml.load(f)
+      config_file = os.path.join(self.path, 'config.yaml')
+      if os.path.exists(config_file):
+        with (open(config_file)) as f:
+            config = yaml.load(f)
       return config
 
   def load_profiles(self):
@@ -79,21 +77,6 @@ class Repository:
               if profile['modules'][module] is None:
                   profile['modules'][module] = {}
 
-  def merge_modules_in_profiles(self, config):
-      """Merge the module configuration and the profile configuration. Profile has precedence over module."""
-      profiles = config['profiles']
-      modules = config['modules']
-      for profile in profiles:
-          resolved_modules = []
-          for module_name in profile['modules']:
-              profile_module_config = profile['modules'][module_name]
-              resolved_module = modules[module_name]
-              copied_module = copy.deepcopy(resolved_module)
-              for c in profile_module_config:
-                  copied_module['analysis']['parameters'][c] = profile_module_config[c]
-              resolved_modules.append(copied_module)
-          profile['modules'] = resolved_modules
-
 class Config:
 
   def __init__(self, repository_paths):
@@ -122,8 +105,25 @@ class Config:
       config['install_path'] = pkg_resources.resource_filename('psot', '')
       config['helpers_path'] = pkg_resources.resource_filename('psot', 'helpers')
       config['repository_paths'] = self.repo_paths
+      self.merge_modules_in_profiles(config)
       return config
 
+  def merge_modules_in_profiles(self, config):
+      """Merge the module configuration and the profile configuration. Profile has precedence over module."""
+      profiles = config['profiles']
+      modules = config['modules']
+      for profile in profiles:
+          resolved_modules = []
+          for module_name in profile['modules']:
+              profile_module_config = profile['modules'][module_name]
+              resolved_module = modules[module_name]
+              copied_module = copy.deepcopy(resolved_module)
+              for c in profile_module_config:
+                  copied_module['analysis']['parameters'][c] = profile_module_config[c]
+              resolved_modules.append(copied_module)
+          profile['modules'] = resolved_modules
+
+
 def load_config(repositories=None):
     # always use the installed repository as the first repo
     all_repositories = [pkg_resources.resource_filename('psot', '')]

tests/data/repo_that_references_other_repo/profiles/test.yaml

+name: 'test3'
+info: 'Profile for testing purposes'
+modules:
+  example2:

tests/test_repository.py

@@ -58,3 +58,16 @@ class TestMergingRepositoriesWorksCorrectly(unittest.TestCase):
   def test_config_value_was_overriden(self):
     value = self.config['tools']['override']
     self.assertEqual(value, 2)
+
+class TestProfileUsesModuleFromOtherRepository(unittest.TestCase):
+
+  def setUp(self):
+    repo_path1 = pkg_resources.resource_filename('tests', 'data/example_repo2/')
+    repo_path2 = pkg_resources.resource_filename('tests', 'data/repo_that_references_other_repo/')
+    self.repository = config.Config([repo_path1, repo_path2])
+    self.config = self.repository.load_config()
+
+  def test_profile_correctly_populated(self):
+    profiles = self.config['profiles']
+    print(profiles)
+