2 Analyze the Summary.htm file produced by GERALD
8 from pprint import pprint
11 from lxml import etree
12 from htsworkflow.util.ethelp import indent, flatten
14 LOGGER = logging.getLogger(__name__)
17 class Summary(object):
19 Extract some useful information from the Summary.htm file
24 def __init__(self, filename=None, xml=None):
25 # lane results is a list of 1 or 2 ends containing
26 # a dictionary of all the lanes reported in this
28 self.lane_results = [{}]
30 if filename is not None:
31 self._extract_lane_results(filename)
33 self.set_elements(xml)
35 def __getitem__(self, key):
36 return self.lane_results[key]
39 return len(self.lane_results)
41 def get_elements(self):
42 summary = etree.Element(Summary.SUMMARY,
43 {'version': unicode(Summary.XML_VERSION)})
44 for end in self.lane_results:
45 for lane in end.values():
46 summary.append(lane.get_elements())
49 def set_elements(self, tree):
50 if tree.tag != Summary.SUMMARY:
51 return ValueError("Expected %s" % (Summary.SUMMARY,))
52 xml_version = int(tree.attrib.get('version', 0))
53 if xml_version > Summary.XML_VERSION:
54 LOGGER.warn('Summary XML tree is a higher version than this class')
55 for element in list(tree):
56 lrs = LaneResultSummaryGA()
57 lrs.set_elements(element)
58 if len(self.lane_results) < (lrs.end + 1):
59 self.lane_results.append({})
60 self.lane_results[lrs.end][lrs.lane] = lrs
62 def is_paired_end(self):
63 return len(self.lane_results) == 2
67 Debugging function, report current object
69 tree = self.get_elements()
70 print etree.tostring(tree)
72 class SummaryGA(Summary):
73 def __init__(self, filename=None, xml=None):
74 super(SummaryGA, self).__init__(filename, xml)
76 def _flattened_row(self, row):
78 flatten the children of a <tr>...</tr>
80 return [flatten(x) for x in row.getchildren() ]
82 def _parse_table(self, table):
84 assumes the first line is the header of a table,
85 and that the remaining rows are data
87 rows = table.getchildren()
90 data.append(self._flattened_row(r))
93 def _extract_lane_results(self, pathname):
95 Extract just the lane results.
96 Currently those are the only ones we care about.
99 tables = self._extract_named_tables(pathname)
102 def _extract_named_tables(self, pathname):
104 extract all the 'named' tables from a Summary.htm file
105 and return as a dictionary
107 Named tables are <h2>...</h2><table>...</table> pairs
108 The contents of the h2 tag is considered to the name
111 if isxml_file(pathname):
112 tree = etree.parse(pathname).getroot()
115 tree = html.parse(pathname).getroot()
116 # tree = etree.parse(pathname).getroot()
117 # hack for 1.1rc1, this should be removed when possible.
118 #file_body = open(pathname).read()
119 #file_body = file_body.replace('CHASTITY<=', 'CHASTITY<=')
120 #tree = etree.fromstring(file_body)
122 # are we reading the xml or the html version of the Summary file?
123 if tree.tag.lower() == 'summary':
125 tables = self._extract_named_tables_from_gerald_xml(tree)
126 elif tree.tag.lower() == 'html':
128 tables = self._extract_named_tables_from_html(tree)
129 table_names = [ ('Lane Results Summary', 0),
130 ('Lane Results Summary : Read 1', 0),
131 ('Lane Results Summary : Read 2', 1),]
132 for name, end in table_names:
133 if tables.has_key(name):
134 self._extract_lane_results_for_end(tables, name, end)
136 if len(self.lane_results[0]) == 0:
137 LOGGER.warning("No Lane Results Summary Found in %s" % (pathname,))
139 def _extract_named_tables_from_gerald_xml(self, tree):
141 Extract useful named tables from a gerald created Summary.xml file
143 # using the function to convert to lower instead of just writing it
144 # makes the tag easier to read (IMO)
145 useful_tables = ['LaneResultsSummary'.lower(),]
148 for child in tree.getchildren():
149 if child.tag.lower() in useful_tables:
150 read_tree = child.find('Read')
152 read = int(read_tree.find('readNumber').text)-1
153 for element in read_tree.getchildren():
154 if element.tag.lower() == "lane":
155 lrs = LaneResultSummaryGA()
156 lrs.set_elements_from_gerald_xml(read, element)
157 self.lane_results[lrs.end][lrs.lane] = lrs
158 # probably not useful
161 ###### START HTML Table Extraction ########
162 def _extract_named_tables_from_html(self, tree):
164 for t in tree.findall('*//table'):
165 previous = t.getprevious()
167 previous = t.getparent()
169 if previous.tag.lower() == 'div':
170 previous = previous.getprevious()
172 if previous.tag in ('h2', 'p'):
174 name = flatten(previous)
175 data = self._parse_table(t)
179 def _extract_lane_results_for_end(self, tables, table_name, end):
181 extract the Lane Results Summary table
183 # parse lane result summary
184 lane_summary = tables[table_name]
185 # this is version 1 of the summary file
186 if len(lane_summary[-1]) == 8:
188 headers = lane_summary[0]
189 # grab the lane by lane data
190 lane_summary = lane_summary[1:]
192 # len(lane_summary[-1] = 10 is version 2 of the summary file
193 # = 9 is version 3 of the Summary.htm file
194 elif len(lane_summary[-1]) in (9, 10):
195 # lane_summary[0] is a different less specific header row
196 headers = lane_summary[1]
197 lane_summary = lane_summary[2:10]
198 # after the last lane, there's a set of chip wide averages
200 # append an extra dictionary if needed
201 if len(self.lane_results) < (end + 1):
202 self.lane_results.append({})
204 for r in lane_summary:
205 lrs = LaneResultSummaryGA(html=r)
207 self.lane_results[lrs.end][lrs.lane] = lrs
208 ###### END HTML Table Extraction ########
211 class SummaryHiSeq(Summary):
212 def __init__(self, filename=None, xml=None):
213 super(SummaryHiSeq, self).__init__(filename, xml)
215 def _extract_lane_results(self, filename):
216 read1 = os.path.join(filename, 'read1.xml')
217 read2 = os.path.join(filename, 'read2.xml')
219 if os.path.exists(read1):
220 self._extract_lane_results_for_end(read1, 0)
222 LOGGER.warn("No read1.xml at %s." % (read1,))
223 if os.path.exists(read2):
224 self.lane_results.append({})
225 self._extract_lane_results_for_end(read2, 1)
227 def _extract_lane_results_for_end(self, filename, end):
228 self.tree = etree.parse(filename)
229 root = self.tree.getroot()
230 for lane in root.getchildren():
231 lrs = LaneResultSummaryHiSeq(data=lane)
233 self.lane_results[lrs.end][lrs.lane] = lrs
236 class LaneResultSummary(object):
238 Parse the LaneResultSummary table out of Summary.htm
239 Mostly for the cluster number
241 LANE_RESULT_SUMMARY = 'LaneResultSummary'
243 'LaneYield': 'lane_yield',
244 'Cluster': 'cluster', # Raw
245 'ClusterPF': 'cluster_pass_filter',
246 'AverageFirstCycleIntensity': 'average_first_cycle_intensity',
247 'PercentIntensityAfter20Cycles': 'percent_intensity_after_20_cycles',
248 'PercentPassFilterClusters': 'percent_pass_filter_clusters',
249 'PercentPassFilterAlign': 'percent_pass_filter_align',
250 'AverageAlignmentScore': 'average_alignment_score',
251 'PercentErrorRate': 'percent_error_rate'
253 # These are tags that have mean/stdev as found in the GERALD Summary.xml file
255 #'laneYield': 'lane_yield', #this is just a number
256 'clusterCountRaw': 'cluster', # Raw
257 'clusterCountPF': 'cluster_pass_filter',
258 'oneSig': 'average_first_cycle_intensity',
259 'signal20AsPctOf1': 'percent_intensity_after_20_cycles',
260 'percentClustersPF': 'percent_pass_filter_clusters',
261 'percentUniquelyAlignedPF': 'percent_pass_filter_align',
262 'averageAlignScorePF': 'average_alignment_score',
263 'errorPF': 'percent_error_rate'
266 def __init__(self, html=None, xml=None):
269 self.lane_yield = None
271 self.cluster_pass_filter = None
272 self.average_first_cycle_intensity = None
273 self.percent_intensity_after_20_cycles = None
274 self.percent_pass_filter_clusters = None
275 self.percent_pass_filter_align = None
276 self.average_alignment_score = None
277 self.percent_error_rate = None
280 def get_elements(self):
281 lane_result = etree.Element(
282 LaneResultSummary.LANE_RESULT_SUMMARY,
283 {'lane': unicode(self.lane), 'end': unicode(self.end)})
284 for tag, variable_name in LaneResultSummary.TAGS.items():
285 value = getattr(self, variable_name)
288 # it looks like a sequence
289 elif type(value) in (types.TupleType, types.ListType):
290 element = make_mean_range_element(
296 element = etree.SubElement(lane_result, tag)
297 element.text = unicode(value)
300 def set_elements(self, tree):
301 if tree.tag != LaneResultSummary.LANE_RESULT_SUMMARY:
302 raise ValueError('Expected %s' % (
303 LaneResultSummary.LANE_RESULT_SUMMARY))
304 self.lane = int(tree.attrib['lane'])
305 # default to the first end, for the older summary files
306 # that are single ended
307 self.end = int(tree.attrib.get('end', 0))
308 tags = LaneResultSummary.TAGS
309 for element in list(tree):
311 variable_name = tags[element.tag]
312 setattr(self, variable_name,
313 parse_summary_element(element))
315 LOGGER.warn('Unrecognized tag %s' % (element.tag,))
318 class LaneResultSummaryGA(LaneResultSummary):
319 def __init__(self, html=None, xml=None):
320 super(LaneResultSummaryGA, self).__init__(html, xml)
323 self.set_elements_from_source(html)
325 self.set_elements(xml)
327 def set_elements_from_gerald_xml(self, read, element):
328 self.lane = int(element.find('laneNumber').text)
330 lane_yield_node = element.find('laneYield')
331 if lane_yield_node is not None:
332 self.lane_yield = int(lane_yield_node.text)
334 self.lane_yield = None
336 for GeraldName, LRSName in LaneResultSummary.GERALD_TAGS.items():
337 node = element.find(GeraldName)
339 LOGGER.info("Couldn't find %s" % (GeraldName))
340 setattr(self, LRSName, parse_xml_mean_range(node))
342 def set_elements_from_source(self, data):
343 """Read from an initial summary data file. Either xml or html
345 if not len(data) in (8,10):
346 raise RuntimeError("Summary.htm file format changed, len(data)=%d" % (len(data),))
348 # same in pre-0.3.0 Summary file and 0.3 summary file
349 self.lane = int(data[0])
352 parsed_data = [ parse_mean_range(x) for x in data[1:] ]
353 # this is the < 0.3 Pipeline version
354 self.cluster = parsed_data[0]
355 self.average_first_cycle_intensity = parsed_data[1]
356 self.percent_intensity_after_20_cycles = parsed_data[2]
357 self.percent_pass_filter_clusters = parsed_data[3]
358 self.percent_pass_filter_align = parsed_data[4]
359 self.average_alignment_score = parsed_data[5]
360 self.percent_error_rate = parsed_data[6]
361 elif len(data) == 10:
362 parsed_data = [ parse_mean_range(x) for x in data[2:] ]
363 # this is the >= 0.3 summary file
364 self.lane_yield = data[1]
365 self.cluster = parsed_data[0]
366 self.cluster_pass_filter = parsed_data[1]
367 self.average_first_cycle_intensity = parsed_data[2]
368 self.percent_intensity_after_20_cycles = parsed_data[3]
369 self.percent_pass_filter_clusters = parsed_data[4]
370 self.percent_pass_filter_align = parsed_data[5]
371 self.average_alignment_score = parsed_data[6]
372 self.percent_error_rate = parsed_data[7]
375 class LaneResultSummaryHiSeq(LaneResultSummary):
376 def __init__(self, data=None, xml=None):
377 super(LaneResultSummaryHiSeq, self).__init__(data, xml)
380 self.set_elements_from_source(data)
382 self.set_elements(xml)
384 def set_elements_from_source(self, element):
385 """Read from an initial summary data file. Either xml or html
387 # same in pre-0.3.0 Summary file and 0.3 summary file
388 lane = element.attrib
389 self.lane = int(lane['key'])
390 #self.lane_yield = data[1]
391 self.cluster = (int(lane['ClustersRaw']),
392 float(lane['ClustersRawSD']))
393 self.cluster_pass_filter = (int(lane['ClustersPF']),
394 float(lane['ClustersPFSD']))
395 self.average_first_cycle_intensity = (int(lane['FirstCycleIntPF']),
396 float(lane['FirstCycleIntPFSD']))
397 self.percent_intensity_after_20_cycles = (
398 float(lane['PrcIntensityAfter20CyclesPF']),
399 float(lane['PrcIntensityAfter20CyclesPFSD']))
400 self.percent_pass_filter_clusters = (
401 float(lane['PrcPFClusters']),
402 float(lane['PrcPFClustersSD']))
403 self.percent_pass_filter_align = (
404 float(lane['PrcAlign']),
405 float(lane['PrcAlignSD']))
406 self.percent_error_rate = (
407 float(lane['ErrRatePhiX']),
408 float(lane['ErrRatePhiXSD']),)
413 Convert a value to int if its an int otherwise a float.
417 except ValueError, e:
421 def parse_mean_range(value):
423 Parse values like 123 +/- 4.5
425 if value.strip() == 'unknown':
428 values = value.split()
430 if values[0] == '+/-':
433 return tonumber(values[0])
435 average, pm, deviation = values
437 raise RuntimeError("Summary.htm file format changed")
438 return tonumber(average), tonumber(deviation)
440 def make_mean_range_element(parent, name, mean, deviation):
442 Make an etree subelement <Name mean='mean', deviation='deviation'/>
444 element = etree.SubElement(parent, name,
445 { 'mean': unicode(mean),
446 'deviation': unicode(deviation)})
449 def parse_mean_range_element(element):
451 Grab mean/deviation out of element
453 return (tonumber(element.attrib['mean']),
454 tonumber(element.attrib['deviation']))
456 def parse_summary_element(element):
458 Determine if we have a simple element or a mean/deviation element
460 if len(element.attrib) > 0:
461 return parse_mean_range_element(element)
465 def parse_xml_mean_range(element):
467 Extract mean/stddev children from an element as a tuple
472 mean = element.find('mean')
473 stddev = element.find('stdev')
474 if mean is None or stddev is None:
475 raise RuntimeError("Summary.xml file format changed, expected mean/stddev tags")
476 if mean.text is None:
477 mean_value = float('nan')
479 mean_value = tonumber(mean.text)
481 if stddev.text is None:
482 stddev_value = float('nan')
484 stddev_value = tonumber(stddev.text)
487 return (mean_value, stddev_value)
489 def isxml_file(fname):
490 with open(fname,'r') as stream:
491 return isxml_stream(stream)
493 def isxml_stream(stream):
494 """Return true or false if its sort of an xml file
497 line = stream.readline()
499 if re.match("<\?xml.*?>", line):
505 if __name__ == "__main__":
507 from optparse import OptionParser
508 parser = OptionParser('%prog [Summary.xml/Summary.htm]+')
509 opts, args = parser.parse_args()
511 parser.error('need at least one xml/html file')