encode_submission/encode3.py

   1 #!/usr/bin/env python
   2 """Create a track hub
   3 """
   4 from __future__ import print_function, unicode_literals
   5
   6 import argparse
   7 import logging
   8 import os
   9
  10 from htsworkflow.util import api
  11 from htsworkflow.util.rdfhelp import \
  12     get_model, \
  13     get_serializer, \
  14     load_into_model, \
  15     sparql_query
  16 from htsworkflow.submission.daf import get_submission_uri
  17 from htsworkflow.submission.submission import list_submissions
  18 from htsworkflow.submission.results import ResultMap
  19 from htsworkflow.submission.condorfastq import CondorFastqExtract
  20 from htsworkflow.submission.aws_submission import AWSSubmission
  21
  22 logger = logging.getLogger(__name__)
  23
  24 INDENTED = "  " + os.linesep
  25
  26 import django
  27 if not 'DJANGO_SETTINGS_MODULE' in os.environ:
  28     os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings.local'
  29
  30 def main(cmdline=None):
  31     parser = make_parser()
  32     args = parser.parse_args(cmdline)
  33     submission_uri = None
  34
  35     from django.conf import settings
  36
  37     if args.debug:
  38         settings.LOGGING['loggers']['htsworkflow']['level'] = 'DEBUG'
  39     elif args.verbose:
  40         settings.LOGGING['loggers']['htsworkflow']['level'] = 'INFO'
  41
  42     django.setup()
  43
  44     model = get_model(args.model, args.db_path)
  45     submission_names = list(list_submissions(model))
  46     name = args.name
  47     if len(submission_names) == 0 and args.name is None:
  48         parser.error("Please name this submission")
  49     elif args.name and submission_names and args.name not in submission_names:
  50         parser.error("{} is not in this model. Choose from: {}{}".format(
  51             args.name,
  52             os.linesep,
  53             INDENTED.join(submission_names)))
  54     elif args.name is None and len(submission_names) > 1:
  55         parser.error("Please choose submission name from: {}{}".format(
  56             os.linesep,
  57             INDENTED.join(submission_names)))
  58     elif len(submission_names) == 1:
  59         name = submission_names[0]
  60
  61     if name:
  62         submission_uri = get_submission_uri(name)
  63         logger.info('Submission URI: %s', submission_uri)
  64
  65     mapper = AWSSubmission(name, model, encode_host=args.encoded, lims_host=args.host)
  66
  67     if args.load_rdf is not None:
  68         if submission_uri is None:
  69             parser.error("Please specify the submission name")
  70         load_into_model(model, 'turtle', args.load_rdf, submission_uri)
  71
  72     results = ResultMap()
  73     for a in args.libraries:
  74         if os.path.exists(a):
  75             results.add_results_from_file(a)
  76         else:
  77             logger.warn("File %s doesn't exist.", a)
  78
  79     if args.make_link_tree_from is not None:
  80         results.make_tree_from(args.make_link_tree_from, link=True)
  81
  82     if args.copy_tree_from is not None:
  83         results.make_tree_from(args.copy_tree_from, link=False)
  84
  85     if args.fastq:
  86         logger.info("Building fastq extraction scripts")
  87         flowcells = os.path.join(args.sequence, 'flowcells')
  88         extractor = CondorFastqExtract(args.host, flowcells,
  89                                        model=args.model,
  90                                        compression=args.compression,
  91                                        force=args.force)
  92         extractor.create_scripts(results)
  93
  94     if args.scan_submission:
  95         if name is None:
  96             parser.error("Please define a submission name")
  97         mapper.scan_submission_dirs(results)
  98
  99     if args.upload:
 100         mapper.upload(results, args.dry_run)
 101
 102     if args.check_upload:
 103         mapper.check_upload(results)
 104
 105     if args.sparql:
 106         sparql_query(model, args.sparql)
 107
 108     if args.print_rdf:
 109         writer = get_serializer()
 110         print(writer.serialize_model_to_string(model))
 111
 112
 113 def make_parser():
 114     parser = argparse.ArgumentParser()
 115
 116     model = parser.add_argument_group('model')
 117     model.add_argument('--name', help="Set submission name")
 118     model.add_argument('--db-path', default=None,
 119                      help="set rdf database path")
 120     model.add_argument('--model', default=None,
 121                      help="Load model database")
 122     model.add_argument('--load-rdf', default=None,
 123                      help="load rdf statements into model")
 124     model.add_argument('--sparql', default=None, help="execute sparql query")
 125     model.add_argument('--print-rdf', action="store_true", default=False,
 126                      help="print ending model state")
 127
 128     # commands
 129     commands = parser.add_argument_group('commands')
 130     commands.add_argument('--make-link-tree-from',
 131                         help="create directories & link data files",
 132                         default=None)
 133     commands.add_argument('--copy-tree-from',
 134                         help="create directories & copy data files",
 135                         default=None)
 136     commands.add_argument('--fastq', default=False, action="store_true",
 137                         help="generate scripts for making fastq files")
 138     commands.add_argument('--scan-submission', default=False, action="store_true",
 139                         help="cache md5 sums")
 140     commands.add_argument('--upload', default=False, action="store_true",
 141                         help="Upload files")
 142     commands.add_argument('--check-upload', default=False, action='store_true',
 143                           help='check to see files are actually uploaded')
 144
 145     parser.add_argument('--force', default=False, action="store_true",
 146                       help="Force regenerating fastqs")
 147     parser.add_argument('--compression', default=None,
 148                       choices=['gzip'],
 149                       help='select compression type for fastq files')
 150     parser.add_argument('--library-url', default=None,
 151                       help="specify an alternate source for library information")
 152     parser.add_argument('--encoded', default='www.encodeproject.org',
 153                       help='base url for talking to encode server')
 154     parser.add_argument('--dry-run', default=False, action='store_true',
 155                       help='avoid making changes to encoded')
 156
 157     # debugging
 158     parser.add_argument('--verbose', default=False, action="store_true",
 159                       help='verbose logging')
 160     parser.add_argument('--debug', default=False, action="store_true",
 161                       help='debug logging')
 162
 163     api.add_auth_options(parser)
 164
 165     parser.add_argument('libraries', nargs='+',
 166                         help='mapping of library id to directory to be processed')
 167
 168     return parser
 169
 170 if __name__ == "__main__":
 171     main()