encode_submission/encode3.py

   1 """Create a track hub
   2 """
   3
   4 #!/usr/bin/env python
   5 from configparser import SafeConfigParser
   6 import fnmatch
   7 from glob import glob
   8 import json
   9 import logging
  10 import netrc
  11 from optparse import OptionParser, OptionGroup
  12 import os
  13 from pprint import pprint, pformat
  14 import shlex
  15 from io import StringIO
  16 import stat
  17 import sys
  18 import time
  19 import types
  20 import urllib.request, urllib.parse, urllib.error
  21 import urllib.request, urllib.error, urllib.parse
  22 import urllib.parse
  23 from zipfile import ZipFile
  24
  25 import RDF
  26
  27 if not 'DJANGO_SETTINGS_MODULE' in os.environ:
  28     os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings'
  29
  30 from htsworkflow.util import api
  31 from htsworkflow.util.rdfhelp import \
  32      dafTermOntology, \
  33      fromTypedNode, \
  34      get_model, \
  35      get_serializer, \
  36      load_into_model, \
  37      sparql_query, \
  38      submissionOntology
  39 from htsworkflow.submission.daf import get_submission_uri
  40 from htsworkflow.submission.submission import list_submissions
  41 from htsworkflow.submission.results import ResultMap
  42 from htsworkflow.submission.trackhub_submission import TrackHubSubmission
  43 from htsworkflow.submission.condorfastq import CondorFastqExtract
  44
  45 logger = logging.getLogger(__name__)
  46
  47 INDENTED = "  " + os.linesep
  48
  49 def main(cmdline=None):
  50     parser = make_parser()
  51     opts, args = parser.parse_args(cmdline)
  52     submission_uri = None
  53
  54     if opts.debug:
  55         logging.basicConfig(level = logging.DEBUG )
  56     elif opts.verbose:
  57         logging.basicConfig(level = logging.INFO )
  58     else:
  59         logging.basicConfig(level = logging.WARNING )
  60
  61     apidata = api.make_auth_from_opts(opts, parser)
  62
  63     model = get_model(opts.model, opts.db_path)
  64
  65     submission_names = list(list_submissions(model))
  66     name = opts.name
  67     if len(submission_names) == 0 and opts.name is None:
  68         parser.error("Please name this submission")
  69     elif opts.name and submission_names and opts.name not in submission_names:
  70         parser.error("{} is not in this model. Choose from: {}{}".format(
  71             opts.name,
  72             os.linesep,
  73             INDENTED.join(submission_names)))
  74     elif opts.name is None and len(submission_names) > 1:
  75         parser.error("Please choose submission name from: {}{}".format(
  76             os.linesep,
  77             INDENTED.join(submission_names)))
  78     elif len(submission_names) == 1:
  79         name = submission_names[0]
  80
  81     mapper = None
  82     if opts.make_track_hub:
  83         mapper = TrackHubSubmission(name,
  84                                     model,
  85                                     baseurl=opts.make_track_hub,
  86                                     baseupload=opts.track_hub_upload,
  87                                     host=opts.host)
  88         submission_uri = get_submission_uri(name)
  89
  90
  91     if opts.load_rdf is not None:
  92         if submission_uri is None:
  93             parser.error("Please specify the submission name")
  94         load_into_model(model, 'turtle', opts.load_rdf, submission_uri)
  95
  96     results = ResultMap()
  97     for a in args:
  98         if os.path.exists(a):
  99             results.add_results_from_file(a)
 100         else:
 101             logger.warn("File %s doesn't exist.", a)
 102
 103     if opts.make_link_tree_from is not None:
 104         results.make_tree_from(opts.make_link_tree_from, link=True)
 105
 106     if opts.copy_tree_from is not None:
 107         results.make_tree_from(opts.copy_tree_from, link=False)
 108
 109     if opts.fastq:
 110         logger.info("Building fastq extraction scripts")
 111         flowcells = os.path.join(opts.sequence, 'flowcells')
 112         extractor = CondorFastqExtract(opts.host, flowcells,
 113                                        model=opts.model,
 114                                        force=opts.force)
 115         extractor.create_scripts(results)
 116
 117     if opts.scan_submission:
 118         if name is None:
 119             parser.error("Please define a submission name")
 120         mapper.scan_submission_dirs(results)
 121
 122     if opts.make_track_hub:
 123         trackdb = mapper.make_hub(results)
 124
 125     if opts.make_manifest:
 126         make_manifest(mapper, results, opts.make_manifest)
 127
 128     if opts.sparql:
 129         sparql_query(model, opts.sparql)
 130
 131     if opts.print_rdf:
 132         writer = get_serializer()
 133         print(writer.serialize_model_to_string(model))
 134
 135
 136 def make_manifest(mapper, results, filename=None):
 137     manifest = mapper.make_manifest(results)
 138
 139     if filename is None or filename == '-':
 140         sys.stdout.write(manifest)
 141     else:
 142         with open(filename, 'w') as mainifeststream:
 143             mainifeststream.write(manifest)
 144
 145 def make_parser():
 146     parser = OptionParser()
 147
 148     model = OptionGroup(parser, 'model')
 149     model.add_option('--name', help="Set submission name")
 150     model.add_option('--db-path', default=None,
 151                      help="set rdf database path")
 152     model.add_option('--model', default=None,
 153       help="Load model database")
 154     model.add_option('--load-rdf', default=None,
 155       help="load rdf statements into model")
 156     model.add_option('--sparql', default=None, help="execute sparql query")
 157     model.add_option('--print-rdf', action="store_true", default=False,
 158       help="print ending model state")
 159     parser.add_option_group(model)
 160     # commands
 161     commands = OptionGroup(parser, 'commands')
 162     commands.add_option('--make-link-tree-from',
 163                       help="create directories & link data files",
 164                       default=None)
 165     commands.add_option('--copy-tree-from',
 166                       help="create directories & copy data files",
 167                       default=None)
 168     commands.add_option('--fastq', default=False, action="store_true",
 169                         help="generate scripts for making fastq files")
 170     commands.add_option('--scan-submission', default=False, action="store_true",
 171                       help="Import metadata for submission into our model")
 172     commands.add_option('--make-track-hub', default=None,
 173                         help='web root that will host the trackhub.')
 174     commands.add_option('--track-hub-upload', default=None,
 175                         help='where to upload track hub <host>:<path>')
 176     commands.add_option('--make-manifest',
 177                         help='name the manifest file name or - for stdout to create it',
 178                         default=None)
 179
 180
 181     parser.add_option_group(commands)
 182
 183     parser.add_option('--force', default=False, action="store_true",
 184                       help="Force regenerating fastqs")
 185     parser.add_option('--daf', default=None, help='specify daf name')
 186     parser.add_option('--library-url', default=None,
 187                       help="specify an alternate source for library information")
 188     # debugging
 189     parser.add_option('--verbose', default=False, action="store_true",
 190                       help='verbose logging')
 191     parser.add_option('--debug', default=False, action="store_true",
 192                       help='debug logging')
 193
 194     api.add_auth_options(parser)
 195
 196     return parser
 197
 198 if __name__ == "__main__":
 199     main()