Commit 048e6207 authored by hmueller's avatar hmueller
Browse files

Merge remote-tracking branch 'origin/develop' into tool-name-in-live-results

parents 044a1df9 278b28bc
......@@ -3,124 +3,134 @@ import yaml
import copy
import pkg_resources
def get_install_location():
"""Finds the location directory of the tool"""
script_path = os.path.realpath(__file__)
script_dir = os.path.dirname(script_path)
install_dir = os.path.dirname(script_dir)
return install_dir
def get_script_location():
return os.path.join(get_install_location(),'bin')
def get_scripts_location():
return pkg_resources.resource_filename('psot', 'scripts')
def get_template_location():
return pkg_resources.resource_filename('psot', 'templates')
def get_modules_location():
return pkg_resources.resource_filename('psot', 'modules')
def get_profiles_location():
return pkg_resources.resource_filename('psot', 'profiles')
def get_config_file():
return pkg_resources.resource_filename('psot', 'config.yaml')
def load_config_file():
config = {}
with (open(get_config_file())) as f:
config = yaml.load(f)
return config
def get_module_manifests():
mypath = get_modules_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def get_profile_manifests():
mypath = get_profiles_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def create_app_config():
config = {}
config['app_path'] = get_install_location()
config['bin_path'] = get_script_location()
config['script_path'] = get_scripts_location()
return config
def load_profiles():
profiles = []
for manifest in get_profile_manifests():
with (open(manifest)) as f:
profile = yaml.load(f)
profiles.append(profile)
return profiles
def load_modules():
modules = {}
for manifest in get_module_manifests():
with (open(manifest)) as f:
module = yaml.load(f)
modules[module['name']] = module
if 'parameters' not in module['analysis']:
module['analysis']['parameters'] = {}
if 'parameters' not in module['converter']:
module['converter']['parameters'] = {}
if module['analysis']['parameters'] is None:
module['analysis']['parameters'] = {}
if module['converter']['parameters'] is None:
module['converter']['parameters'] = {}
return modules
def normalize_profiles(profiles):
"""take care that all keys of the profile map exist"""
for profile in profiles:
if 'modules' not in profile:
profile['modules'] = []
for module in profile['modules']:
if profile['modules'][module] is None:
profile['modules'][module] = {}
def merge_modules_in_profiles(config):
"""Merge the module configuration and the profile configuration. Profile has precedence over module."""
profiles = config['profiles']
modules = config['modules']
for profile in profiles:
resolved_modules = []
for module_name in profile['modules']:
profile_module_config = profile['modules'][module_name]
resolved_module = modules[module_name]
copied_module = copy.deepcopy(resolved_module)
for c in profile_module_config:
copied_module['analysis']['parameters'][c] = profile_module_config[c]
resolved_modules.append(copied_module)
profile['modules'] = resolved_modules
class Repository:
def __init__(self, path):
self.path = path
def load_config(self):
'''return a dictionary with the whole configuration for this repository'''
config = self.load_config_file()
config['profiles'] = self.load_profiles()
config['modules'] = self.load_modules()
self.normalize_profiles(config['profiles'])
self.merge_modules_in_profiles(config)
return config
def get_module_manifests(self):
mypath = self.get_modules_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def get_profile_manifests(self):
mypath = self.get_profiles_location()
return [os.path.join(mypath, f) for f in os.listdir(mypath) if os.path.isfile(os.path.join(mypath, f)) and not f.startswith('.')]
def get_modules_location(self):
return os.path.join(self.path, 'modules')
def get_profiles_location(self):
return os.path.join(self.path, 'profiles')
def get_config_file(self):
return os.path.join(self.path, 'config.yaml')
def get_scripts_location(self):
return os.path.join(self.path, 'scripts')
def load_config_file(self):
config = {}
with (open(self.get_config_file())) as f:
config = yaml.load(f)
return config
def load_profiles(self):
profiles = []
for manifest in self.get_profile_manifests():
with (open(manifest)) as f:
profile = yaml.load(f)
profiles.append(profile)
return profiles
def load_modules(self):
modules = {}
for manifest in self.get_module_manifests():
with (open(manifest)) as f:
module = yaml.load(f)
modules[module['name']] = module
if 'parameters' not in module['analysis']:
module['analysis']['parameters'] = {}
if 'parameters' not in module['converter']:
module['converter']['parameters'] = {}
if module['analysis']['parameters'] is None:
module['analysis']['parameters'] = {}
if module['converter']['parameters'] is None:
module['converter']['parameters'] = {}
module['analysis']['script'] = os.path.join(self.get_scripts_location(), module['analysis']['script'])
module['converter']['script'] = os.path.join(self.get_scripts_location(), module['converter']['script'])
return modules
def normalize_profiles(self, profiles):
"""take care that all keys of the profile map exist"""
for profile in profiles:
if 'modules' not in profile:
profile['modules'] = []
for module in profile['modules']:
if profile['modules'][module] is None:
profile['modules'][module] = {}
def merge_modules_in_profiles(self, config):
"""Merge the module configuration and the profile configuration. Profile has precedence over module."""
profiles = config['profiles']
modules = config['modules']
for profile in profiles:
resolved_modules = []
for module_name in profile['modules']:
profile_module_config = profile['modules'][module_name]
resolved_module = modules[module_name]
copied_module = copy.deepcopy(resolved_module)
for c in profile_module_config:
copied_module['analysis']['parameters'][c] = profile_module_config[c]
resolved_modules.append(copied_module)
profile['modules'] = resolved_modules
class Config:
def __init__(self, repository_paths):
self.repo_paths = repository_paths
def merge_config(self, source, target):
import jsonmerge
# Quick hack: replacement instead of merge
# TODO do an actual merge instead of just a replacement
schema = { 'properties': {'profiles':{'mergeStrategy':'append'}} }
return jsonmerge.merge(target, source, schema)
def load_config(self):
config = {}
config['app'] = {}
for repo_path in self.repo_paths:
repo = Repository(repo_path)
repo_config = repo.load_config()
config = self.merge_config(repo_config, config)
# quick hack
# use the last script directory
# TODO replace with list
config['app']['script_path'] = repo.get_scripts_location()
config['install_path'] = pkg_resources.resource_filename('psot', '')
config['repository_paths'] = self.repo_paths
return config
def load_config():
config = load_config_file()
config['profiles'] = load_profiles()
config['modules'] = load_modules()
config['app'] = create_app_config()
normalize_profiles(config['profiles'])
merge_modules_in_profiles(config)
return config
return Config([pkg_resources.resource_filename('psot', '')]).load_config()
if __name__ == "__main__":
"""prints the configuration"""
import config
from pprint import pformat
config = Config([pkg_resources.resource_filename('psot', '')])
print('Install location: ' + config.get_install_location())
print('Script lcoation: ' + config.get_script_location())
print('Config location: ' + config.get_config_file())
print('Available profile manifests:\n\t' + '\n\t'.join(config.get_profile_manifests()) + '\n')
print('Available module manifests:\n\t' + '\n\t'.join(config.get_module_manifests()) + '\n')
print('Profile manifests content:\n' + pformat(config.load_profiles()) + '\n')
print('Module manifests content:\n' + pformat(config.load_modules()) + '\n')
print('Config content:\n' + pformat(config.load_config_file()) + '\n')
print('Install location: ' + str(config.repo_paths))
print('Aggregated config:\n' + pformat(config.load_config()) + '\n')
......@@ -62,9 +62,7 @@ def generate_execution(config, args):
execution['use_cluster'] = args.use_cluster
execution['fetch_informations'] = args.fetch_informations
execution['mode'] = 'live' if args.live else 'complete'
execution['bin_path'] = config['app']['bin_path']
execution['script_path'] = config['app']['script_path']
execution['psot_path'] = os.path.abspath(os.path.join(os.path.dirname(config['app']['bin_path']), 'psot'))
execution['fasta'] = os.path.abspath(args.fasta)
execution['output'] = os.path.abspath(args.output)
if args.execution_dir:
......
......@@ -167,8 +167,8 @@ def setup_execution_directory(execution):
if not os.path.exists(directory + '/bin'):
os.symlink(execution['script_path'], directory + '/bin')
if not os.path.exists(directory + '/psot'):
os.symlink(execution['psot_path'], directory + '/psot')
#if not os.path.exists(directory + '/psot'):
# os.symlink(execution['psot_path'], directory + '/psot')
def execute_analysis(execution):
old_cwd = os.getcwd()
......
name: 'example'
info: 'example module for testing'
analysis:
script: 'run_example.sh'
parameters:
converter:
script: 'convert_example.sh'
parameters:
name: 'test'
info: 'Profile for testing purposes'
modules:
example:
tools:
test: bla
override: 2
name: 'example2'
info: 'example module for testing'
analysis:
script: 'run_example.sh'
parameters:
converter:
script: 'convert_example.sh'
parameters:
name: 'test2'
info: 'Profile for testing purposes'
modules:
example2:
import unittest
import pkg_resources
import os.path
from psot import config
class TestExampleRepositoryIsLoadedCorrectly(unittest.TestCase):
def setUp(self):
repo_path = pkg_resources.resource_filename('tests', 'data/example_repo/')
self.repository = config.Repository(repo_path)
self.config = self.repository.load_config()
def test_that_profile_exists(self):
# check that a profile with the name 'test' exists in the list of profiles
for profile in self.config['profiles']:
if 'test' == profile['name']:
return
self.fail('Profile test not found')
def test_that_module_exits(self):
# check that a module with the name 'example' exists in the list of modules
self.assertTrue('example' in self.config['modules'], 'Module example not found')
def test_that_run_script_is_resolved_correctly(self):
# all paths to scripts should be stored as absolute paths in the configuration. This tests checks if the scripts are reachable via the path specified
script_path = self.config['modules']['example']['analysis']['script']
self.assertTrue(os.path.isfile(script_path), "Script is not a file " + script_path)
def test_that_convert_script_is_resolved_correctly(self):
script_path = self.config['modules']['example']['converter']['script']
self.assertTrue(os.path.isfile(script_path), "Script is not a file " + script_path)
def test_that_config_exists(self):
self.assertTrue('tools' in self.config, 'config.yml not loaded')
class TestMergingRepositoriesWorksCorrectly(unittest.TestCase):
def setUp(self):
repo_path1 = pkg_resources.resource_filename('tests', 'data/example_repo/')
repo_path2 = pkg_resources.resource_filename('tests', 'data/example_repo2/')
self.repository = config.Config([repo_path1, repo_path2])
self.config = self.repository.load_config()
# output config for debugging purposes
#from pprint import pformat
#print(pformat(self.config))
#print()
def test_profiles_merged(self):
profiles = self.config['profiles']
self.assertTrue(len(profiles) is 2, 'Only {} profile(s) found, expected 2'.format(str(len(profiles))))
def test_modules_merged(self):
modules = self.config['modules']
self.assertTrue(len(modules) is 2, 'Only {} module(s) found, expected 2'.format(str(len(modules))))
def test_config_value_was_overriden(self):
value = self.config['tools']['override']
self.assertEqual(value, 2)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment