Add vfdb download

dbman/main.py

@@ -164,6 +164,12 @@ def run_in_tempdir(func=None, success=None, fail=None):
 
 def _recipes(args):
    recipes = {
+     'vfdb': {
+       'download': {'script': _pkgres('recipes/download_vfdb.py')},
+       'blast': {'script': _pkgres('recipes/create_blast_dbs.py')},
+       'diamond': {'script': _pkgres('recipes/create_diamond_dbs.py')},
+       'ghostx': {'script': _pkgres('recipes/create_ghostx_dbs.py')},
+     },
      'resfinder': {
        'download': {'script': _pkgres('recipes/download_resfinder.py')},
        'blast': {'script': _pkgres('recipes/create_blast_dbs.py')},

dbman/recipes/download_vfdb.py

+#!/usr/bin/env python3
+import dbman.helper as h
+import json
+import os
+import gzip
+import urllib.request
+from bs4 import BeautifulSoup
+import datetime
+
+import logging
+logging.basicConfig(level=logging.INFO)
+
+dbname = "vfdb"
+dbdescription = "The virulence factor database"
+url = "http://www.mgc.ac.cn/VFs/"
+
+# get version
+data = urllib.request.urlopen(url + "download.htm").read()
+bs = BeautifulSoup(data)
+version = [s.getText() for s in bs.find_all('i') if 'Last update' in s.getText()][0]
+version = version.replace('Last update: ', "")
+version = datetime.datetime.strptime(version, "%a %b %d  %H:%M:%S %Y").date().isoformat()
+
+# download archives
+files = [
+  {'file': 'Down/VFs.xls.gz', 'tags':['excel', 'gzip', 'compressed']},
+  {'file': 'Down/Comparative_tables_from_VFDB.tar.gz', 'tags':['excel', 'tar', 'gzip', 'compressed']},
+  {'file': 'Down/VFDB_setA_nt.fas.gz', 'tags':['core', 'fasta', 'nucleotide', 'gzip', 'compressed']},
+  {'file': 'Down/VFDB_setA_pro.fas.gz', 'tags':['core', 'fasta', 'protein', 'gzip', 'compressed']},
+  {'file': 'Down/VFDB_setB_nt.fas.gz', 'tags':['full', 'fasta', 'nucleotide', 'gzip', 'compressed']},
+  {'file': 'Down/VFDB_setB_pro.fas.gz', 'tags':['full', 'fasta', 'protein', 'gzip', 'compressed']},
+]
+
+parts = []
+for file in files:
+  fn = h.download(url + file['file'])
+  part = {'files': [fn], 'tags': file['tags']}
+  parts.append(part)
+
+# write metadata file
+h.create_metadata(dbname, "download", dbdescription, version=version, other={'parts':parts})

requirements.txt

@@ -2,3 +2,4 @@ wget
 sphinx
 s3cmd
 prettytable
+beautifulsoup4