encode_submission/encode3.py

   1 #!/usr/bin/env python
   2 """Create a track hub
   3 """
   4 from __future__ import print_function, unicode_literals
   5
   6 from ConfigParser import SafeConfigParser
   7 import fnmatch
   8 from glob import glob
   9 import json
  10 import logging
  11 import netrc
  12 from optparse import OptionParser, OptionGroup
  13 import os
  14 from pprint import pprint, pformat
  15 import shlex
  16 from six.moves import StringIO
  17 import stat
  18 import sys
  19 import time
  20 import types
  21 from zipfile import ZipFile
  22
  23 import RDF
  24
  25 from htsworkflow.util import api
  26 from htsworkflow.util.rdfhelp import \
  27     dafTermOntology, \
  28     fromTypedNode, \
  29     get_model, \
  30     get_serializer, \
  31     load_into_model, \
  32     sparql_query, \
  33     submissionOntology
  34 from htsworkflow.submission.daf import get_submission_uri
  35 from htsworkflow.submission.submission import list_submissions
  36 from htsworkflow.submission.results import ResultMap
  37 from htsworkflow.submission.condorfastq import CondorFastqExtract
  38 from htsworkflow.submission.aws_submission import AWSSubmission
  39 logger = logging.getLogger(__name__)
  40
  41 INDENTED = "  " + os.linesep
  42
  43 import django
  44 if not 'DJANGO_SETTINGS_MODULE' in os.environ:
  45     os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings.local'
  46
  47 def main(cmdline=None):
  48     parser = make_parser()
  49     opts, args = parser.parse_args(cmdline)
  50     submission_uri = None
  51
  52     from django.conf import settings
  53
  54     if opts.debug:
  55         settings.LOGGING['loggers']['htsworkflow']['level'] = 'DEBUG'
  56     elif opts.verbose:
  57         settings.LOGGING['loggers']['htsworkflow']['level'] = 'INFO'
  58
  59     django.setup()
  60
  61     model = get_model(opts.model, opts.db_path)
  62     submission_names = list(list_submissions(model))
  63     name = opts.name
  64     if len(submission_names) == 0 and opts.name is None:
  65         parser.error("Please name this submission")
  66     elif opts.name and submission_names and opts.name not in submission_names:
  67         parser.error("{} is not in this model. Choose from: {}{}".format(
  68             opts.name,
  69             os.linesep,
  70             INDENTED.join(submission_names)))
  71     elif opts.name is None and len(submission_names) > 1:
  72         parser.error("Please choose submission name from: {}{}".format(
  73             os.linesep,
  74             INDENTED.join(submission_names)))
  75     elif len(submission_names) == 1:
  76         name = submission_names[0]
  77
  78     if name:
  79         submission_uri = get_submission_uri(name)
  80         logger.info('Submission URI: %s', submission_uri)
  81
  82     mapper = AWSSubmission(name, model, encode_host=opts.encoded, lims_host=opts.host)
  83
  84     if opts.load_rdf is not None:
  85         if submission_uri is None:
  86             parser.error("Please specify the submission name")
  87         load_into_model(model, 'turtle', opts.load_rdf, submission_uri)
  88
  89     results = ResultMap()
  90     for a in args:
  91         if os.path.exists(a):
  92             results.add_results_from_file(a)
  93         else:
  94             logger.warn("File %s doesn't exist.", a)
  95
  96     if opts.make_link_tree_from is not None:
  97         results.make_tree_from(opts.make_link_tree_from, link=True)
  98
  99     if opts.copy_tree_from is not None:
 100         results.make_tree_from(opts.copy_tree_from, link=False)
 101
 102     if opts.fastq:
 103         logger.info("Building fastq extraction scripts")
 104         flowcells = os.path.join(opts.sequence, 'flowcells')
 105         extractor = CondorFastqExtract(opts.host, flowcells,
 106                                        model=opts.model,
 107                                        compression=opts.compression,
 108                                        force=opts.force)
 109         extractor.create_scripts(results)
 110
 111     if opts.scan_submission:
 112         if name is None:
 113             parser.error("Please define a submission name")
 114         mapper.scan_submission_dirs(results)
 115
 116     if opts.upload:
 117         mapper.upload(results, opts.dry_run)
 118
 119     if opts.sparql:
 120         sparql_query(model, opts.sparql)
 121
 122     if opts.print_rdf:
 123         writer = get_serializer()
 124         print(writer.serialize_model_to_string(model))
 125
 126
 127 def make_parser():
 128     parser = OptionParser()
 129
 130     model = OptionGroup(parser, 'model')
 131     model.add_option('--name', help="Set submission name")
 132     model.add_option('--db-path', default=None,
 133                      help="set rdf database path")
 134     model.add_option('--model', default=None,
 135                      help="Load model database")
 136     model.add_option('--load-rdf', default=None,
 137                      help="load rdf statements into model")
 138     model.add_option('--sparql', default=None, help="execute sparql query")
 139     model.add_option('--print-rdf', action="store_true", default=False,
 140                      help="print ending model state")
 141     parser.add_option_group(model)
 142     # commands
 143     commands = OptionGroup(parser, 'commands')
 144     commands.add_option('--make-link-tree-from',
 145                         help="create directories & link data files",
 146                         default=None)
 147     commands.add_option('--copy-tree-from',
 148                         help="create directories & copy data files",
 149                         default=None)
 150     commands.add_option('--fastq', default=False, action="store_true",
 151                         help="generate scripts for making fastq files")
 152     commands.add_option('--scan-submission', default=False, action="store_true",
 153                         help="cache md5 sums")
 154     commands.add_option('--upload', default=False, action="store_true",
 155                         help="Upload files")
 156
 157     parser.add_option_group(commands)
 158
 159     parser.add_option('--force', default=False, action="store_true",
 160                       help="Force regenerating fastqs")
 161     parser.add_option('--compression', default=None, type='choice',
 162                       choices=['gzip'],
 163                       help='select compression type for fastq files')
 164     parser.add_option('--library-url', default=None,
 165                       help="specify an alternate source for library information")
 166     parser.add_option('--encoded', default='www.encodeproject.org',
 167                       help='base url for talking to encode server')
 168     parser.add_option('--dry-run', default=False, action='store_true',
 169                       help='avoid making changes to encoded')
 170     # debugging
 171     parser.add_option('--verbose', default=False, action="store_true",
 172                       help='verbose logging')
 173     parser.add_option('--debug', default=False, action="store_true",
 174                       help='debug logging')
 175
 176     api.add_auth_options(parser)
 177
 178     return parser
 179
 180 if __name__ == "__main__":
 181     main()