Commit 6cfc2cf4 authored by Lukas Jelonek's avatar Lukas Jelonek
Browse files

Refactor config into repository and application config.

parent c5e97033
......@@ -3,124 +3,132 @@ import yaml
import copy
import pkg_resources
def get_install_location():
"""Finds the location directory of the tool"""
script_path = os.path.realpath(__file__)
script_dir = os.path.dirname(script_path)
install_dir = os.path.dirname(script_dir)
return install_dir
def get_script_location():
return os.path.join(get_install_location(),'bin')
def get_scripts_location():
return pkg_resources.resource_filename('psot', 'scripts')
def get_template_location():
return pkg_resources.resource_filename('psot', 'templates')
def get_modules_location():
return pkg_resources.resource_filename('psot', 'modules')
def get_profiles_location():
return pkg_resources.resource_filename('psot', 'profiles')
def get_config_file():
return pkg_resources.resource_filename('psot', 'config.yaml')
def load_config_file():
config = {}
with (open(get_config_file())) as f:
config = yaml.load(f)
return config
def get_module_manifests():
mypath = get_modules_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def get_profile_manifests():
mypath = get_profiles_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def create_app_config():
config = {}
config['app_path'] = get_install_location()
config['bin_path'] = get_script_location()
config['script_path'] = get_scripts_location()
return config
def load_profiles():
profiles = []
for manifest in get_profile_manifests():
with (open(manifest)) as f:
profile = yaml.load(f)
profiles.append(profile)
return profiles
def load_modules():
modules = {}
for manifest in get_module_manifests():
with (open(manifest)) as f:
module = yaml.load(f)
modules[module['name']] = module
if 'parameters' not in module['analysis']:
module['analysis']['parameters'] = {}
if 'parameters' not in module['converter']:
module['converter']['parameters'] = {}
if module['analysis']['parameters'] is None:
module['analysis']['parameters'] = {}
if module['converter']['parameters'] is None:
module['converter']['parameters'] = {}
return modules
def normalize_profiles(profiles):
"""take care that all keys of the profile map exist"""
for profile in profiles:
if 'modules' not in profile:
profile['modules'] = []
for module in profile['modules']:
if profile['modules'][module] is None:
profile['modules'][module] = {}
def merge_modules_in_profiles(config):
"""Merge the module configuration and the profile configuration. Profile has precedence over module."""
profiles = config['profiles']
modules = config['modules']
for profile in profiles:
resolved_modules = []
for module_name in profile['modules']:
profile_module_config = profile['modules'][module_name]
resolved_module = modules[module_name]
copied_module = copy.deepcopy(resolved_module)
for c in profile_module_config:
copied_module['analysis']['parameters'][c] = profile_module_config[c]
resolved_modules.append(copied_module)
profile['modules'] = resolved_modules
class Repository:
def __init__(self, path):
self.path = path
def load_config(self):
'''return a dictionary with the whole configuration for this repository'''
config = self.load_config_file()
print(config)
config['profiles'] = self.load_profiles()
config['modules'] = self.load_modules()
self.normalize_profiles(config['profiles'])
self.merge_modules_in_profiles(config)
return config
def get_module_manifests(self):
mypath = self.get_modules_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def get_profile_manifests(self):
mypath = self.get_profiles_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def get_modules_location(self):
return os.path.join(self.path, 'modules')
def get_profiles_location(self):
return os.path.join(self.path, 'profiles')
def get_config_file(self):
return os.path.join(self.path, 'config.yaml')
def get_scripts_location(self):
return pkg_resources.resource_filename('psot', 'scripts')
def load_config_file(self):
config = {}
with (open(self.get_config_file())) as f:
config = yaml.load(f)
return config
def load_profiles(self):
profiles = []
for manifest in self.get_profile_manifests():
with (open(manifest)) as f:
profile = yaml.load(f)
profiles.append(profile)
return profiles
def load_modules(self):
modules = {}
for manifest in self.get_module_manifests():
with (open(manifest)) as f:
module = yaml.load(f)
modules[module['name']] = module
if 'parameters' not in module['analysis']:
module['analysis']['parameters'] = {}
if 'parameters' not in module['converter']:
module['converter']['parameters'] = {}
if module['analysis']['parameters'] is None:
module['analysis']['parameters'] = {}
if module['converter']['parameters'] is None:
module['converter']['parameters'] = {}
return modules
def normalize_profiles(self, profiles):
"""take care that all keys of the profile map exist"""
for profile in profiles:
if 'modules' not in profile:
profile['modules'] = []
for module in profile['modules']:
if profile['modules'][module] is None:
profile['modules'][module] = {}
def merge_modules_in_profiles(self, config):
"""Merge the module configuration and the profile configuration. Profile has precedence over module."""
profiles = config['profiles']
modules = config['modules']
for profile in profiles:
resolved_modules = []
for module_name in profile['modules']:
profile_module_config = profile['modules'][module_name]
resolved_module = modules[module_name]
copied_module = copy.deepcopy(resolved_module)
for c in profile_module_config:
copied_module['analysis']['parameters'][c] = profile_module_config[c]
resolved_modules.append(copied_module)
profile['modules'] = resolved_modules
class Config:
def __init__(self, repository_paths):
self.repo_paths = repository_paths
def merge_config(self, source, target):
# Quick hack: replacement instead of merge
# TODO do an actual merge instead of just a replacement
for k in source:
target[k] = source[k]
return
def load_config(self):
config = {}
config['app'] = {}
for repo_path in self.repo_paths:
repo = Repository(repo_path)
repo_config = repo.load_config()
self.merge_config(repo_config, config)
# quick hack
# use the last script directory
# TODO replace with list
config['app']['script_path'] = repo.get_scripts_location()
config['install_path'] = pkg_resources.resource_filename('psot', '')
config['repository_paths'] = self.repo_paths
return config
def load_config():
config = load_config_file()
config['profiles'] = load_profiles()
config['modules'] = load_modules()
config['app'] = create_app_config()
normalize_profiles(config['profiles'])
merge_modules_in_profiles(config)
return config
return Config([pkg_resources.resource_filename('psot', '')]).load_config()
if __name__ == "__main__":
"""prints the configuration"""
import config
from pprint import pformat
config = Config([pkg_resources.resource_filename('psot', '')])
print('Install location: ' + config.get_install_location())
print('Script lcoation: ' + config.get_script_location())
print('Config location: ' + config.get_config_file())
print('Available profile manifests:\n\t' + '\n\t'.join(config.get_profile_manifests()) + '\n')
print('Available module manifests:\n\t' + '\n\t'.join(config.get_module_manifests()) + '\n')
print('Profile manifests content:\n' + pformat(config.load_profiles()) + '\n')
print('Module manifests content:\n' + pformat(config.load_modules()) + '\n')
print('Config content:\n' + pformat(config.load_config_file()) + '\n')
print('Install location: ' + str(config.repo_paths))
print('Aggregated config:\n' + pformat(config.load_config()) + '\n')
......@@ -62,9 +62,7 @@ def generate_execution(config, args):
execution['use_cluster'] = args.use_cluster
execution['fetch_informations'] = args.fetch_informations
execution['mode'] = 'live' if args.live else 'complete'
execution['bin_path'] = config['app']['bin_path']
execution['script_path'] = config['app']['script_path']
execution['psot_path'] = os.path.abspath(os.path.join(os.path.dirname(config['app']['bin_path']), 'psot'))
execution['fasta'] = os.path.abspath(args.fasta)
execution['output'] = os.path.abspath(args.output)
if args.execution_dir:
......
......@@ -167,8 +167,8 @@ def setup_execution_directory(execution):
if not os.path.exists(directory + '/bin'):
os.symlink(execution['script_path'], directory + '/bin')
if not os.path.exists(directory + '/psot'):
os.symlink(execution['psot_path'], directory + '/psot')
#if not os.path.exists(directory + '/psot'):
# os.symlink(execution['psot_path'], directory + '/psot')
def execute_analysis(execution):
old_cwd = os.getcwd()
......
name: 'example'
info: 'example module for testing'
analysis:
script: 'run_example.sh'
parameters:
converter:
script: 'convert_example.sh'
parameters:
name: 'test'
info: 'Profile for testing purposes'
modules:
example:
import unittest
import pkg_resources
from psot import config
class TestExampleRepositoryIsLoadedCorrectly(unittest.TestCase):
def setUp(self):
repo_path = pkg_resources.resource_filename('tests', 'data/example_repo/')
self.repository = config.Repository(repo_path)
self.config = self.repository.load_config()
def test_that_profile_exists(self):
# check that a profile with the name 'test' exists in the list of profiles
for profile in self.config['profiles']:
if 'test' == profile['name']:
return
self.fail('Profile test not found')
def test_that_module_exits(self):
# check that a module with the name 'example' exists in the list of modules
self.assertTrue('example' in self.config['modules'], 'Module example not found')
def test_that_scripts_are_resolved_correctly(self):
self.fail('not implemented yet')
def test_that_config_exists(self):
self.fail('not implemented yet')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment