encode_submission/geo_gather.py

   1 #!/usr/bin/env python
   2 from __future__ import print_function, unicode_literals
   3
   4 from ConfigParser import SafeConfigParser
   5 import fnmatch
   6 from glob import glob
   7 import json
   8 import logging
   9 import netrc
  10 from optparse import OptionParser, OptionGroup
  11 import os
  12 from pprint import pprint, pformat
  13 import shlex
  14 from six.moves import StringIO
  15 import stat
  16 import sys
  17 import time
  18 import types
  19 import urllib
  20 import urllib2
  21 import urlparse
  22 from zipfile import ZipFile
  23
  24 import RDF
  25
  26 if not 'DJANGO_SETTINGS_MODULE' in os.environ:
  27     os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings'
  28
  29
  30 from htsworkflow.util import api
  31 from htsworkflow.util.rdfhelp import \
  32      dafTermOntology, \
  33      fromTypedNode, \
  34      get_model, \
  35      get_serializer, \
  36      load_into_model, \
  37      sparql_query, \
  38      submissionOntology
  39 from htsworkflow.submission.daf import get_submission_uri
  40 from htsworkflow.submission.results import ResultMap
  41 from htsworkflow.submission.geo import GEOSubmission
  42 from htsworkflow.submission.condorfastq import CondorFastqExtract
  43
  44 logger = logging.getLogger(__name__)
  45
  46 def main(cmdline=None):
  47     parser = make_parser()
  48     opts, args = parser.parse_args(cmdline)
  49     submission_uri = None
  50
  51     if opts.debug:
  52         logging.basicConfig(level = logging.DEBUG )
  53     elif opts.verbose:
  54         logging.basicConfig(level = logging.INFO )
  55     else:
  56         logging.basicConfig(level = logging.WARNING )
  57
  58     apidata = api.make_auth_from_opts(opts, parser)
  59
  60     model = get_model(opts.model, opts.db_path)
  61     mapper = None
  62     if opts.name:
  63         mapper = GEOSubmission(opts.name,  model, host=opts.host)
  64         submission_uri = get_submission_uri(opts.name)
  65
  66
  67     if opts.load_rdf is not None:
  68         if submission_uri is None:
  69             parser.error("Please specify the submission name")
  70         load_into_model(model, 'turtle', opts.load_rdf, submission_uri)
  71
  72     results = ResultMap()
  73     for a in args:
  74         if os.path.exists(a):
  75             results.add_results_from_file(a)
  76         else:
  77             logger.warn("File %s doesn't exist.", a)
  78
  79     if opts.make_tree_from is not None:
  80         results.make_tree_from(opts.make_tree_from)
  81
  82     if opts.fastq:
  83         logger.info("Building fastq extraction scripts")
  84         flowcells = os.path.join(opts.sequence, 'flowcells')
  85         extractor = CondorFastqExtract(opts.host, flowcells,
  86                                        model=opts.model,
  87                                        force=opts.force)
  88         extractor.create_scripts(results)
  89
  90     if opts.scan_submission:
  91         if opts.name is None:
  92             parser.error("Please define a submission name")
  93         mapper.scan_submission_dirs(results)
  94
  95     if opts.make_soft:
  96         mapper.make_soft(results)
  97
  98     if opts.sparql:
  99         sparql_query(model, opts.sparql)
 100
 101     if opts.print_rdf:
 102         writer = get_serializer()
 103         print(writer.serialize_model_to_string(model))
 104
 105
 106 def make_parser():
 107     parser = OptionParser()
 108
 109     model = OptionGroup(parser, 'model')
 110     model.add_option('--name', help="Set submission name")
 111     model.add_option('--db-path', default=None,
 112                      help="set rdf database path")
 113     model.add_option('--model', default=None,
 114       help="Load model database")
 115     model.add_option('--load-rdf', default=None,
 116       help="load rdf statements into model")
 117     model.add_option('--sparql', default=None, help="execute sparql query")
 118     model.add_option('--print-rdf', action="store_true", default=False,
 119       help="print ending model state")
 120     parser.add_option_group(model)
 121     # commands
 122     commands = OptionGroup(parser, 'commands')
 123     commands.add_option('--make-tree-from',
 124                       help="create directories & link data files",
 125                       default=None)
 126     commands.add_option('--fastq', default=False, action="store_true",
 127                         help="generate scripts for making fastq files")
 128     commands.add_option('--scan-submission', default=False, action="store_true",
 129                       help="Import metadata for submission into our model")
 130     commands.add_option('--make-soft', help='make the soft file', default=False,
 131                       action="store_true")
 132
 133     parser.add_option_group(commands)
 134
 135     parser.add_option('--force', default=False, action="store_true",
 136                       help="Force regenerating fastqs")
 137     parser.add_option('--daf', default=None, help='specify daf name')
 138     parser.add_option('--library-url', default=None,
 139                       help="specify an alternate source for library information")
 140     # debugging
 141     parser.add_option('--verbose', default=False, action="store_true",
 142                       help='verbose logging')
 143     parser.add_option('--debug', default=False, action="store_true",
 144                       help='debug logging')
 145
 146     api.add_auth_options(parser)
 147
 148     return parser
 149
 150
 151 if __name__ == "__main__":
 152     main()