Automatically add the flowcell subdirectory to the sequence archive path
[htsworkflow.git] / encode_submission / geo_gather.py
1 #!/usr/bin/env python
2 from ConfigParser import SafeConfigParser
3 import fnmatch
4 from glob import glob
5 import json
6 import logging
7 import netrc
8 from optparse import OptionParser, OptionGroup
9 import os
10 from pprint import pprint, pformat
11 import shlex
12 from StringIO import StringIO
13 import stat
14 import sys
15 import time
16 import types
17 import urllib
18 import urllib2
19 import urlparse
20 from zipfile import ZipFile
21
22 import RDF
23
24 if not 'DJANGO_SETTINGS_MODULE' in os.environ:
25     os.environ['DJANGO_SETTINGS_MODULE'] = 'htsworkflow.settings'
26
27
28 from htsworkflow.util import api
29 from htsworkflow.util.rdfhelp import \
30      dafTermOntology, \
31      fromTypedNode, \
32      get_model, \
33      get_serializer, \
34      load_into_model, \
35      sparql_query, \
36      submissionOntology
37 from htsworkflow.submission.daf import get_submission_uri
38 from htsworkflow.submission.results import ResultMap
39 from htsworkflow.submission.geo import GEOSubmission
40 from htsworkflow.submission.condorfastq import CondorFastqExtract
41
42 logger = logging.getLogger(__name__)
43
44 def main(cmdline=None):
45     parser = make_parser()
46     opts, args = parser.parse_args(cmdline)
47     submission_uri = None
48
49     if opts.debug:
50         logging.basicConfig(level = logging.DEBUG )
51     elif opts.verbose:
52         logging.basicConfig(level = logging.INFO )
53     else:
54         logging.basicConfig(level = logging.WARNING )
55
56     apidata = api.make_auth_from_opts(opts, parser)
57
58     model = get_model(opts.model, opts.db_path)
59     mapper = None
60     if opts.name:
61         mapper = GEOSubmission(opts.name,  model)
62         if opts.library_url is not None:
63             mapper.library_url = opts.library_url
64         submission_uri = get_submission_uri(opts.name)
65
66
67     if opts.load_rdf is not None:
68         if submission_uri is None:
69             parser.error("Please specify the submission name")
70         load_into_model(model, 'turtle', opts.load_rdf, submission_uri)
71
72     results = ResultMap()
73     for a in args:
74         results.add_results_from_file(a)
75
76     if opts.make_tree_from is not None:
77         results.make_tree_from(opts.make_tree_from)
78
79     if opts.fastq:
80         flowcells = os.path.join(opts.sequence, 'flowcells')
81         extractor = CondorFastqExtract(opts.host, flowcells,
82                                        force=opts.force)
83         extractor.create_scripts(results)
84
85     if opts.scan_submission:
86         mapper.scan_submission_dirs(results)
87
88     if opts.make_soft:
89         mapper.make_soft(results)
90
91     if opts.sparql:
92         sparql_query(model, opts.sparql)
93
94     if opts.print_rdf:
95         writer = get_serializer()
96         print writer.serialize_model_to_string(model)
97
98
99 def make_parser():
100     parser = OptionParser()
101
102     model = OptionGroup(parser, 'model')
103     model.add_option('--name', help="Set submission name")
104     model.add_option('--db-path', default=None,
105                      help="set rdf database path")
106     model.add_option('--model', default=None,
107       help="Load model database")
108     model.add_option('--load-rdf', default=None,
109       help="load rdf statements into model")
110     model.add_option('--sparql', default=None, help="execute sparql query")
111     model.add_option('--print-rdf', action="store_true", default=False,
112       help="print ending model state")
113     parser.add_option_group(model)
114     # commands
115     commands = OptionGroup(parser, 'commands')
116     commands.add_option('--make-tree-from',
117                       help="create directories & link data files",
118                       default=None)
119     commands.add_option('--fastq', default=False, action="store_true",
120                         help="generate scripts for making fastq files")
121     commands.add_option('--scan-submission', default=False, action="store_true",
122                       help="Import metadata for submission into our model")
123     commands.add_option('--make-soft', help='make the soft file', default=False,
124                       action="store_true")
125
126     parser.add_option_group(commands)
127
128     parser.add_option('--force', default=False, action="store_true",
129                       help="Force regenerating fastqs")
130     parser.add_option('--daf', default=None, help='specify daf name')
131     parser.add_option('--library-url', default=None,
132                       help="specify an alternate source for library information")
133     # debugging
134     parser.add_option('--verbose', default=False, action="store_true",
135                       help='verbose logging')
136     parser.add_option('--debug', default=False, action="store_true",
137                       help='debug logging')
138
139     api.add_auth_options(parser)
140
141     return parser
142
143
144 if __name__ == "__main__":
145     main()