encode_submission/encode3.py

   1 """Create a track hub
   2 """
   3
   4 #!/usr/bin/env python
   5 from ConfigParser import SafeConfigParser
   6 import fnmatch
   7 from glob import glob
   8 import json
   9 import logging
  10 import netrc
  11 from optparse import OptionParser, OptionGroup
  12 import os
  13 from pprint import pprint, pformat
  14 import shlex
  15 from StringIO import StringIO
  16 import stat
  17 import sys
  18 import time
  19 import types
  20 import urllib
  21 import urllib2
  22 import urlparse
  23 from zipfile import ZipFile
  24
  25 import RDF
  26
  27 if not 'DJANGO_SETTINGS_MODULE' in os.environ:
  28     os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings'
  29
  30 from htsworkflow.util import api
  31 from htsworkflow.util.rdfhelp import \
  32      dafTermOntology, \
  33      fromTypedNode, \
  34      get_model, \
  35      get_serializer, \
  36      load_into_model, \
  37      sparql_query, \
  38      submissionOntology
  39 from htsworkflow.submission.daf import get_submission_uri
  40 from htsworkflow.submission.submission import list_submissions
  41 from htsworkflow.submission.results import ResultMap
  42 from htsworkflow.submission.trackhub_submission import TrackHubSubmission
  43 from htsworkflow.submission.condorfastq import CondorFastqExtract
  44
  45 logger = logging.getLogger(__name__)
  46
  47 INDENTED = "  " + os.linesep
  48
  49 def main(cmdline=None):
  50     parser = make_parser()
  51     opts, args = parser.parse_args(cmdline)
  52     submission_uri = None
  53
  54     if opts.debug:
  55         logging.basicConfig(level = logging.DEBUG )
  56     elif opts.verbose:
  57         logging.basicConfig(level = logging.INFO )
  58     else:
  59         logging.basicConfig(level = logging.WARNING )
  60
  61     apidata = api.make_auth_from_opts(opts, parser)
  62
  63     model = get_model(opts.model, opts.db_path)
  64
  65     submission_names = list(list_submissions(model))
  66     name = opts.name
  67     if len(submission_names) == 0 and opts.name is None:
  68         parser.error("Please name this submission")
  69     elif opts.name and submission_names and opts.name not in submission_names:
  70         parser.error("{} is not in this model. Choose from: {}{}".format(
  71             opts.name,
  72             os.linesep,
  73             INDENTED.join(submission_names)))
  74     elif opts.name is None and len(submission_names) > 1:
  75         parser.error("Please choose submission name from: {}{}".format(
  76             os.linesep,
  77             INDENTED.join(submission_names)))
  78     elif len(submission_names) == 1:
  79         name = submission_names[0]
  80
  81     if name:
  82         submission_uri = get_submission_uri(name)
  83         logger.info('Submission URI: %s', name)
  84     else:
  85         logger.debug('No name, unable to create submission ur')
  86
  87     mapper = None
  88     if opts.make_track_hub:
  89         mapper = TrackHubSubmission(name,
  90                                     model,
  91                                     baseurl=opts.make_track_hub,
  92                                     baseupload=opts.track_hub_upload,
  93                                     host=opts.host)
  94
  95     if opts.load_rdf is not None:
  96         if submission_uri is None:
  97             parser.error("Please specify the submission name")
  98         load_into_model(model, 'turtle', opts.load_rdf, submission_uri)
  99
 100     results = ResultMap()
 101     for a in args:
 102         if os.path.exists(a):
 103             results.add_results_from_file(a)
 104         else:
 105             logger.warn("File %s doesn't exist.", a)
 106
 107     if opts.make_link_tree_from is not None:
 108         results.make_tree_from(opts.make_link_tree_from, link=True)
 109
 110     if opts.copy_tree_from is not None:
 111         results.make_tree_from(opts.copy_tree_from, link=False)
 112
 113     if opts.fastq:
 114         logger.info("Building fastq extraction scripts")
 115         flowcells = os.path.join(opts.sequence, 'flowcells')
 116         extractor = CondorFastqExtract(opts.host, flowcells,
 117                                        model=opts.model,
 118                                        force=opts.force)
 119         extractor.create_scripts(results)
 120
 121     if opts.scan_submission:
 122         if name is None:
 123             parser.error("Please define a submission name")
 124         if mapper is None:
 125             parser.error("Scan submission needs --make-track-hub=public-url")
 126         mapper.scan_submission_dirs(results)
 127
 128     if opts.make_track_hub:
 129         trackdb = mapper.make_hub(results)
 130
 131     if opts.make_manifest:
 132         make_manifest(mapper, results, opts.make_manifest)
 133
 134     if opts.sparql:
 135         sparql_query(model, opts.sparql)
 136
 137     if opts.print_rdf:
 138         writer = get_serializer()
 139         print writer.serialize_model_to_string(model)
 140
 141
 142 def make_manifest(mapper, results, filename=None):
 143     manifest = mapper.make_manifest(results)
 144
 145     if filename is None or filename == '-':
 146         sys.stdout.write(manifest)
 147     else:
 148         with open(filename, 'w') as mainifeststream:
 149             mainifeststream.write(manifest)
 150
 151 def make_parser():
 152     parser = OptionParser()
 153
 154     model = OptionGroup(parser, 'model')
 155     model.add_option('--name', help="Set submission name")
 156     model.add_option('--db-path', default=None,
 157                      help="set rdf database path")
 158     model.add_option('--model', default=None,
 159       help="Load model database")
 160     model.add_option('--load-rdf', default=None,
 161       help="load rdf statements into model")
 162     model.add_option('--sparql', default=None, help="execute sparql query")
 163     model.add_option('--print-rdf', action="store_true", default=False,
 164       help="print ending model state")
 165     parser.add_option_group(model)
 166     # commands
 167     commands = OptionGroup(parser, 'commands')
 168     commands.add_option('--make-link-tree-from',
 169                       help="create directories & link data files",
 170                       default=None)
 171     commands.add_option('--copy-tree-from',
 172                       help="create directories & copy data files",
 173                       default=None)
 174     commands.add_option('--fastq', default=False, action="store_true",
 175                         help="generate scripts for making fastq files")
 176     commands.add_option('--scan-submission', default=False, action="store_true",
 177                       help="Import metadata for submission into our model")
 178     commands.add_option('--make-track-hub', default=None,
 179                         help='web root that will host the trackhub.')
 180     commands.add_option('--track-hub-upload', default=None,
 181                         help='where to upload track hub <host>:<path>')
 182     commands.add_option('--make-manifest',
 183                         help='name the manifest file name or - for stdout to create it',
 184                         default=None)
 185
 186
 187     parser.add_option_group(commands)
 188
 189     parser.add_option('--force', default=False, action="store_true",
 190                       help="Force regenerating fastqs")
 191     parser.add_option('--daf', default=None, help='specify daf name')
 192     parser.add_option('--library-url', default=None,
 193                       help="specify an alternate source for library information")
 194     # debugging
 195     parser.add_option('--verbose', default=False, action="store_true",
 196                       help='verbose logging')
 197     parser.add_option('--debug', default=False, action="store_true",
 198                       help='debug logging')
 199
 200     api.add_auth_options(parser)
 201
 202     return parser
 203
 204 if __name__ == "__main__":
 205     main()