Commit dd9ddf79 authored by Rudel Fankep's avatar Rudel Fankep
Browse files

upload begin

parent e160add2
#!/bin/bash
TOOL_DIR=$1
TOOL_FILE=$2
cd $TOOL_DIR
makeblastdb -dbtype prot -in $TOOL_FILE
rm $TOOL_FILE
cd -
\ No newline at end of file
#!/bin/bash
TOOL_DIR=$1
TOOL_FILE=$2
cd $TOOL_DIR
ghostx db -i $TOOL_FILE -o ghostx_db
rm $TOOL_FILE
cd -
\ No newline at end of file
#!/bin/bash
DOWNLOAD_DIR=$1
cd $DOWNLOAD_DIR
ghostx db -i uniprot_sprot.fasta -o sp_ghost
tar -czf sp_gx.tar.gz sp_ghost*
rm uniprot_sprot*
rm sp_ghost*
cd -
\ No newline at end of file
#!/home/theia/conda/bin/python
import argparse
import subprocess
import os
data = {'swissprot': {'prepare': './prepare_swissprot.sh',
'tool': {'blast': './blast_swissprot.sh', 'ghostx': './ghostx_swissprot.sh'},
'filename': 'uniprot_sprot.fasta'
},
'pfam': {'prepare': './prepare_pfam.sh',
'tool': {'hmmer': './hmmer_pfam.sh'},
'filename': 'Pfam-A.hmm'
},
'card': {
}
}
def myparser():
parser = argparse.ArgumentParser(description='Download, convert and share the data to working computer', prog='PSOT_database_maker')
subparsers = parser.add_subparsers(title='Subcommands', description='Authorized Subcommands', help='Additional help')
prepare_parser = subparsers.add_parser('prepare', help='Databank download from the Website, extraction and transformation for another Programm')
prepare_parser.add_argument('database', help='Database which have to be prepared', type=str)
prepare_parser.add_argument('tool',help='Programm/Tool for the post analysis',type=str,)
prepare_parser.add_argument('-d', '--directory', dest='dir', help='target directory for the output.Default will be a created("output") folder in the working directory', type=str)
prepare_parser.set_defaults(func=prepare)
upload_parser = subparsers.add_parser('upload', help='Databank copy from the local directory to the web storage')
upload_parser.add_argument('file', help='File to be transfered from the local directory', type=str)
upload_parser.add_argument('-d', '--directory', dest='dir', help='Target directory for the file transfer into "S3". Default will be "s3://db_storage"', type=str)
upload_parser.set_defaults(func=upload)
download_parser = subparsers.add_parser('download', help='Datenbank copy from the web storage to the working computer')
download_parser.add_argument('file', help='File to be transfered from "S3"', type=str)
download_parser.add_argument('-d', '--directory', dest='dir', help='Target directory for the file transfer. Default will be a created folder("output) in the current working directory',type=str)
download_parser.set_defaults(func=download)
delete_parser = subparsers.add_parser('delete', help='delete existing files from local directory or from "S3"')
delete_parser.add_argument('database', help = 'database which have to be delete')
delete_parser.add_argument('place', choices = ['local','S3'], help = 'defined the place where the database have to be delete')
delete_parser.add_argument('-d', '--directory', dest='dir', help='directory where the database have to be delete')
delete_parser.set_defaults(func=delete)
return parser.parse_args()
def targetdir_maker(args):
databasedir = "./output"
if args.dir:
databasedir = args.output
elif "dbman_dbdir" in os.environ:
databasedir = os.environ["dbman_dbdir"]
return databasedir
# paste the target directory with the filename.(will be important to test if the file is already there. "/" or not)
def filedir_maker(directory, file):
filedir = ''
if directory[-1] == '/':
filedir = directory+file
else:
filedir = directory+'/'+file
return filedir
def prepare(args):
dbman_dir = targetdir_maker(args)
testfile = filedir_maker(dbman_dir, data[args.database]['filename'])
print(testfile)
if not os.path.isfile(testfile):
subprocess.run([data[args.database]['prepare'], dbman_dir])
subprocess.run([data[args.database]['tool'][args.tool], dbman_dir])
print("The {} file is in: ".format(args.database) + dbman_dir)
else:
print('The {} file already exists in :'.format(args.database) + dbman_dir)
def upload(args):
dbman_dir = targetdir_maker(args)
subprocess.run(['./upload_db.sh', args.file, dbman_dir])
def download(args):
dbman_dir = targetdir_maker(args)
subprocess.run(['./download_db.sh', args.file,dbman_dir])
def delete(args):
if args.place == 'local':
dbman_dir = targetdir_maker(args)
testfile = filedir_maker(dbman_dir, data[args.database]['filename'])
if os.path.isfile(testfile):
subprocess.run(['./delete_db.sh',data[args.database]['filename'],dbman_dir])
print("The {} files were successfully delete in: ".format(args.database) + dbman_dir)
else:
print("The {} files aren't existing in: ".format(args.database) + dbman_dir)
else:
print('Skript für S3 noch nicht fertig')
def main():
if not os.path.isdir('output'):
os.mkdir('output')
args=myparser()
args.func(args)
if __name__=='__main__':
#!/home/theia/conda/bin/python
import argparse
import subprocess
import os
import shutil
data = {'swissprot': {'prepare': './prepare_swissprot.sh',
'tool': {'blast': './blast_db.sh', 'ghostx': './ghostx_db.sh'},
'filename':'uniprot_sprot.fasta'
},
'pfam': {'prepare': './prepare_pfam.sh',
'tool': {'hmmer': './hmmer_pfam.sh'},
'filename':''
},
'card': {'prepare':'./prepare_card.sh',
'tool':{'blast':'./blast_db.sh','ghostx':'./ghostx_db.sh'},
'filename':'protein_fasta_protein_homolog_model.fasta'
}
}
def myparser():
parser = argparse.ArgumentParser(description='Download, convert and share the data to working computer', prog='PSOT_database_maker')
subparsers = parser.add_subparsers(title='Subcommands', description='Authorized Subcommands', help='Additional help')
prepare_parser = subparsers.add_parser('prepare', help='Databank download from the Website, extraction and transformation for another Programm')
prepare_parser.add_argument('database', help='Database which have to be prepared', type=str)
prepare_parser.add_argument('tool',help='Programm/Tool for the post analysis',type=str,)
prepare_parser.add_argument('-d', '--directory', dest='dir', help='target directory for the output.Default will be a created("output") folder in the working directory', type=str)
prepare_parser.set_defaults(func=prepare)
upload_parser = subparsers.add_parser('upload', help='Databank copy from the local directory to the web storage')
upload_parser.add_argument('database', help='database to be transfered from the local directory', type=str)
upload_parser.add_argument('tool',type=str,help='database type which have to be transfered')
upload_parser.add_argument('-d', '--directory', dest='dir', help='Target directory for the file transfer into "S3". Default will be "s3://db_storage"', type=str)
upload_parser.set_defaults(func=upload)
download_parser = subparsers.add_parser('download', help='Datenbank copy from the web storage to the working computer')
download_parser.add_argument('file', help='File to be transfered from "S3"', type=str)
download_parser.add_argument('-d', '--directory', dest='dir', help='Target directory for the file transfer. Default will be a created folder("output) in the current working directory',type=str)
download_parser.set_defaults(func=download)
delete_parser = subparsers.add_parser('delete', help='delete existing files from local directory or from "S3"')
delete_parser.add_argument('database', help = 'database which have to be delete')
delete_parser.add_argument('place', choices = ['local','S3'], help = 'defined the place where the database have to be delete')
delete_parser.add_argument('-d', '--directory', dest='dir', help='directory where the database have to be delete')
delete_parser.set_defaults(func=delete)
return parser.parse_args()
def targetdir_maker(args):
databasedir = "./output"
if args.dir:
databasedir = args.dir
elif "DBMAN_DBDIR" in os.environ:
databasedir = os.environ["DBMAN_DBDIR"]
return databasedir
def s3dir_maker(args):
web_dir='s3://db_storage'
if args.dir:
web_dir=args.dir
elif 'S3_DBDIR' in os.environ:
web_dir=os.environ['S3_DBDIR']
return web_dir
# paste the target directory with the filename.(will be important to test if the file is already there. "/" or not)
def path_maker(directory, file):
filedir = ''
if directory[-1] == '/':
filedir = directory+file
else:
filedir = directory+'/'+file
return filedir
def prepare(args):
if args.database in data.keys():
dbman_dir = targetdir_maker(args)
os.environ['PREPARE_DATA'] = dbman_dir
raw_dir = path_maker(dbman_dir, args.database+'_raw')
if not os.path.isdir(raw_dir):
os.mkdir(raw_dir)
subprocess.run([data[args.database]['prepare'], raw_dir])
print("The {} file is in: ".format(args.database) + raw_dir)
else:
print('The {} file already exists in: '.format(args.database) + raw_dir)
if args.tool in data[args.database]['tool'].keys():
tool_dir=path_maker(dbman_dir,args.database+'_'+args.tool)
tool_file=path_maker(raw_dir,data[args.database]['filename'])
if not os.path.isdir(tool_dir):
os.mkdir(tool_dir)
shutil.copy(tool_file,tool_dir)
subprocess.run([data[args.database]['tool'][args.tool], tool_dir, data[args.database]['filename']])
print('The {} files are in: '.format(args.tool) + tool_dir)
else:
print('The {} files are already exists in: '.format(args.tool) + tool_dir)
else:
print('Tool error. Verify your input(tool) or/and check the possibility in the help')
else:
print('Database error. Verify your input(database) or/and check the possibility in the help')
return os.environ['PREPARE_DATA']
def upload(args):
if args.database in data.keys() and args.tool in data[args.database]['tool'].keys():
dbman_dir = s3dir_maker(args)
dbdata = os.environ['PREPARE_DATA']
upload_data = args.database+'_'+args.tool
upload_data_path = path_maker(dbdata, upload_data)
upload_data_name = '{}'.format(upload_data+'.tar.gz')
if os.path.isdir(upload_data_path):
subprocess.run(['./upload_db.sh', dbdata, upload_data, dbman_dir, upload_data_name])
else:
print('There is no data for upload in {}. Prepare the database first'.format(dbdata))
else:
print('Verify your database and/or Tool input ')
def download(args):
dbman_dir = targetdir_maker(args)
subprocess.run(['./download_db.sh', args.file,dbman_dir])
def delete(args):
if args.place == 'local':
dbman_dir = targetdir_maker(args)
testfile = filedir_maker(dbman_dir, data[args.database]['filename'])
if os.path.isfile(testfile):
subprocess.run(['./delete_db.sh',data[args.database]['filename'],dbman_dir])
print("The {} files were successfully delete in: ".format(args.database) + dbman_dir)
else:
print("The {} files aren't existing in: ".format(args.database) + dbman_dir)
else:
print('Skript für S3 noch nicht fertig')
def main():
if not os.path.isdir('output'):
os.mkdir('output')
args=myparser()
args.func(args)
if __name__=='__main__':
main()
\ No newline at end of file
This diff is collapsed.
#!/bin/bash
DOWNLOAD_DIR=$1
cd $DOWNLOAD_DIR
makeblastdb -dbtype prot -in uniprot_sprot.fasta
tar -czf sp_blast.tar.gz uniprot_sprot.*
wget https://card.mcmaster.ca/download/0/broadstreet-v3.0.3.tar.gz
tar xjf broadstreet-v3.0.3.tar.gz
cd -
\ No newline at end of file
......@@ -87,3 +87,9 @@ default='s3://db_storage'
ordner mit entsprechende namen!!!!
Database repository---> https://github.com/MGX-metagenomics/databases/blob/master/card.build
anzeigen lassen---> echo $VARIABLE
löschen---> unset VARIABLE
setzen---> export VARIABLE = pfad
\ No newline at end of file
#!/bin/bash
Target_file=$1
Target_dir=$2
s3cmd put $Target_file $Target_dir
DATA_DB=$1
TARGET_FILE=$2
TARGET_DIR=$3
TARGET_FILE_NAME=$4
cd $DATA_DB
tar -czvf $TARGET_FILE_NAME $TARGET_FILE
s3cmd put $TARGET_FILE_NAME $TARGET_DIR
rm $TARGET_FILE_NAME
cd -
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment