encode_submission/encode3.py

   1 """Create a track hub
   2 """
   3
   4 #!/usr/bin/env python
   5 from ConfigParser import SafeConfigParser
   6 import fnmatch
   7 from glob import glob
   8 import json
   9 import logging
  10 import netrc
  11 from optparse import OptionParser, OptionGroup
  12 import os
  13 from pprint import pprint, pformat
  14 import shlex
  15 from StringIO import StringIO
  16 import stat
  17 import sys
  18 import time
  19 import types
  20 import urllib
  21 import urllib2
  22 import urlparse
  23 from zipfile import ZipFile
  24
  25 import RDF
  26
  27 from htsworkflow.util import api
  28 from htsworkflow.util.rdfhelp import \
  29     dafTermOntology, \
  30     fromTypedNode, \
  31     get_model, \
  32     get_serializer, \
  33     load_into_model, \
  34     sparql_query, \
  35     submissionOntology
  36 from htsworkflow.submission.daf import get_submission_uri
  37 from htsworkflow.submission.submission import list_submissions
  38 from htsworkflow.submission.results import ResultMap
  39 from htsworkflow.submission.trackhub_submission import TrackHubSubmission
  40 from htsworkflow.submission.condorfastq import CondorFastqExtract
  41
  42 logger = logging.getLogger(__name__)
  43
  44 INDENTED = "  " + os.linesep
  45
  46 import django
  47 if not 'DJANGO_SETTINGS_MODULE' in os.environ:
  48     os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings.local'
  49
  50 def main(cmdline=None):
  51     parser = make_parser()
  52     opts, args = parser.parse_args(cmdline)
  53     submission_uri = None
  54
  55     from django.conf import settings
  56
  57     if opts.debug:
  58         settings.LOGGING['loggers']['level'] = 'DEBUG'
  59     elif opts.verbose:
  60         settings.LOGGING['loggers']['level'] = 'INFO'
  61
  62     model = get_model(opts.model, opts.db_path)
  63
  64     submission_names = list(list_submissions(model))
  65     name = opts.name
  66     if len(submission_names) == 0 and opts.name is None:
  67         parser.error("Please name this submission")
  68     elif opts.name and submission_names and opts.name not in submission_names:
  69         parser.error("{} is not in this model. Choose from: {}{}".format(
  70             opts.name,
  71             os.linesep,
  72             INDENTED.join(submission_names)))
  73     elif opts.name is None and len(submission_names) > 1:
  74         parser.error("Please choose submission name from: {}{}".format(
  75             os.linesep,
  76             INDENTED.join(submission_names)))
  77     elif len(submission_names) == 1:
  78         name = submission_names[0]
  79
  80     if name:
  81         submission_uri = get_submission_uri(name)
  82         logger.info('Submission URI: %s', name)
  83     else:
  84         logger.debug('No name, unable to create submission ur')
  85
  86     mapper = None
  87     if opts.make_track_hub:
  88         mapper = TrackHubSubmission(name,
  89                                     model,
  90                                     baseurl=opts.make_track_hub,
  91                                     baseupload=opts.track_hub_upload,
  92                                     host=opts.host)
  93
  94     if opts.load_rdf is not None:
  95         if submission_uri is None:
  96             parser.error("Please specify the submission name")
  97         load_into_model(model, 'turtle', opts.load_rdf, submission_uri)
  98
  99     results = ResultMap()
 100     for a in args:
 101         if os.path.exists(a):
 102             results.add_results_from_file(a)
 103         else:
 104             logger.warn("File %s doesn't exist.", a)
 105
 106     if opts.make_link_tree_from is not None:
 107         results.make_tree_from(opts.make_link_tree_from, link=True)
 108
 109     if opts.copy_tree_from is not None:
 110         results.make_tree_from(opts.copy_tree_from, link=False)
 111
 112     if opts.fastq:
 113         logger.info("Building fastq extraction scripts")
 114         flowcells = os.path.join(opts.sequence, 'flowcells')
 115         extractor = CondorFastqExtract(opts.host, flowcells,
 116                                        model=opts.model,
 117                                        compression=opts.compression,
 118                                        force=opts.force)
 119         extractor.create_scripts(results)
 120
 121     if opts.scan_submission:
 122         if name is None:
 123             parser.error("Please define a submission name")
 124         if mapper is None:
 125             parser.error("Scan submission needs --make-track-hub=public-url")
 126         mapper.scan_submission_dirs(results)
 127
 128     if opts.make_track_hub:
 129         trackdb = mapper.make_hub(results)
 130
 131     if opts.make_manifest:
 132         make_manifest(mapper, results, opts.make_manifest)
 133
 134     if opts.sparql:
 135         sparql_query(model, opts.sparql)
 136
 137     if opts.print_rdf:
 138         writer = get_serializer()
 139         print writer.serialize_model_to_string(model)
 140
 141
 142 def make_manifest(mapper, results, filename=None):
 143     manifest = mapper.make_manifest(results)
 144
 145     if filename is None or filename == '-':
 146         sys.stdout.write(manifest)
 147     else:
 148         with open(filename, 'w') as mainifeststream:
 149             mainifeststream.write(manifest)
 150
 151
 152 def make_parser():
 153     parser = OptionParser()
 154
 155     model = OptionGroup(parser, 'model')
 156     model.add_option('--name', help="Set submission name")
 157     model.add_option('--db-path', default=None,
 158                      help="set rdf database path")
 159     model.add_option('--model', default=None,
 160                      help="Load model database")
 161     model.add_option('--load-rdf', default=None,
 162                      help="load rdf statements into model")
 163     model.add_option('--sparql', default=None, help="execute sparql query")
 164     model.add_option('--print-rdf', action="store_true", default=False,
 165                      help="print ending model state")
 166     parser.add_option_group(model)
 167     # commands
 168     commands = OptionGroup(parser, 'commands')
 169     commands.add_option('--make-link-tree-from',
 170                         help="create directories & link data files",
 171                         default=None)
 172     commands.add_option('--copy-tree-from',
 173                         help="create directories & copy data files",
 174                         default=None)
 175     commands.add_option('--fastq', default=False, action="store_true",
 176                         help="generate scripts for making fastq files")
 177     commands.add_option('--scan-submission', default=False, action="store_true",
 178                         help="Import metadata for submission into our model")
 179     commands.add_option('--make-track-hub', default=None,
 180                         help='web root that will host the trackhub.')
 181     commands.add_option('--track-hub-upload', default=None,
 182                         help='where to upload track hub <host>:<path>')
 183     commands.add_option('--make-manifest',
 184                         help='name the manifest file name or - for stdout to create it',
 185                         default=None)
 186
 187     parser.add_option_group(commands)
 188
 189     parser.add_option('--force', default=False, action="store_true",
 190                       help="Force regenerating fastqs")
 191     parser.add_option('--compression', default=None, type='choice',
 192                       choices=['gzip'],
 193                       help='select compression type for fastq files')
 194     parser.add_option('--daf', default=None, help='specify daf name')
 195     parser.add_option('--library-url', default=None,
 196                       help="specify an alternate source for library information")
 197     # debugging
 198     parser.add_option('--verbose', default=False, action="store_true",
 199                       help='verbose logging')
 200     parser.add_option('--debug', default=False, action="store_true",
 201                       help='debug logging')
 202
 203     api.add_auth_options(parser)
 204
 205     return parser
 206
 207 if __name__ == "__main__":
 208     django.setup()
 209
 210     main()