1# Lint as: python3
2# Copyright (C) 2019 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Emit warning messages to html or csv files."""
17
18# To emit html page of warning messages:
19#   flags: --byproject, --url, --separator
20# Old stuff for static html components:
21#   html_script_style:  static html scripts and styles
22#   htmlbig:
23#   dump_stats, dump_html_prologue, dump_html_epilogue:
24#   emit_buttons:
25#   dump_fixed
26#   sort_warnings:
27#   emit_stats_by_project:
28#   all_patterns,
29#   findproject, classify_warning
30#   dump_html
31#
32# New dynamic HTML page's static JavaScript data:
33#   Some data are copied from Python to JavaScript, to generate HTML elements.
34#   FlagPlatform           flags.platform
35#   FlagURL                flags.url, used by 'android'
36#   FlagSeparator          flags.separator, used by 'android'
37#   SeverityColors:        list of colors for all severity levels
38#   SeverityHeaders:       list of headers for all severity levels
39#   SeverityColumnHeaders: list of column_headers for all severity levels
40#   ProjectNames:          project_names, or project_list[*][0]
41#   WarnPatternsSeverity:     warn_patterns[*]['severity']
42#   WarnPatternsDescription:  warn_patterns[*]['description']
43#   WarningMessages:          warning_messages
44#   Warnings:                 warning_records
45#   StatsHeader:           warning count table header row
46#   StatsRows:             array of warning count table rows
47#
48# New dynamic HTML page's dynamic JavaScript data:
49#
50# New dynamic HTML related function to emit data:
51#   escape_string, strip_escape_string, emit_warning_arrays
52#   emit_js_data():
53
54from __future__ import print_function
55import cgi
56import csv
57import sys
58
59# pylint:disable=relative-beyond-top-level
60# pylint:disable=g-importing-member
61from .severity import Severity
62
63
64html_head_scripts = """\
65  <script type="text/javascript">
66  function expand(id) {
67    var e = document.getElementById(id);
68    var f = document.getElementById(id + "_mark");
69    if (e.style.display == 'block') {
70       e.style.display = 'none';
71       f.innerHTML = '&#x2295';
72    }
73    else {
74       e.style.display = 'block';
75       f.innerHTML = '&#x2296';
76    }
77  };
78  function expandCollapse(show) {
79    for (var id = 1; ; id++) {
80      var e = document.getElementById(id + "");
81      var f = document.getElementById(id + "_mark");
82      if (!e || !f) break;
83      e.style.display = (show ? 'block' : 'none');
84      f.innerHTML = (show ? '&#x2296' : '&#x2295');
85    }
86  };
87  </script>
88  <style type="text/css">
89  th,td{border-collapse:collapse; border:1px solid black;}
90  .button{color:blue;font-size:110%;font-weight:bolder;}
91  .bt{color:black;background-color:transparent;border:none;outline:none;
92      font-size:140%;font-weight:bolder;}
93  .c0{background-color:#e0e0e0;}
94  .c1{background-color:#d0d0d0;}
95  .t1{border-collapse:collapse; width:100%; border:1px solid black;}
96  </style>
97  <script src="https://www.gstatic.com/charts/loader.js"></script>
98"""
99
100
101def make_writer(output_stream):
102
103  def writer(text):
104    return output_stream.write(text + '\n')
105
106  return writer
107
108
109def html_big(param):
110  return '<font size="+2">' + param + '</font>'
111
112
113def dump_html_prologue(title, writer, warn_patterns, project_names):
114  writer('<html>\n<head>')
115  writer('<title>' + title + '</title>')
116  writer(html_head_scripts)
117  emit_stats_by_project(writer, warn_patterns, project_names)
118  writer('</head>\n<body>')
119  writer(html_big(title))
120  writer('<p>')
121
122
123def dump_html_epilogue(writer):
124  writer('</body>\n</head>\n</html>')
125
126
127def sort_warnings(warn_patterns):
128  for i in warn_patterns:
129    i['members'] = sorted(set(i['members']))
130
131
132def create_warnings(warn_patterns, project_names):
133  """Creates warnings s.t.
134
135  warnings[p][s] is as specified in above docs.
136
137  Args:
138    warn_patterns: list of warning patterns for specified platform
139    project_names: list of project names
140
141  Returns:
142    2D warnings array where warnings[p][s] is # of warnings in project name p of
143    severity level s
144  """
145  # pylint:disable=g-complex-comprehension
146  warnings = {p: {s.value: 0 for s in Severity.levels} for p in project_names}
147  for i in warn_patterns:
148    s = i['severity'].value
149    for p in i['projects']:
150      warnings[p][s] += i['projects'][p]
151  return warnings
152
153
154def get_total_by_project(warnings, project_names):
155  """Returns dict, project as key and # warnings for that project as value."""
156  # pylint:disable=g-complex-comprehension
157  return {
158      p: sum(warnings[p][s.value] for s in Severity.levels)
159      for p in project_names
160  }
161
162
163def get_total_by_severity(warnings, project_names):
164  """Returns dict, severity as key and # warnings of that severity as value."""
165  # pylint:disable=g-complex-comprehension
166  return {
167      s.value: sum(warnings[p][s.value] for p in project_names)
168      for s in Severity.levels
169  }
170
171
172def emit_table_header(total_by_severity):
173  """Returns list of HTML-formatted content for severity stats."""
174
175  stats_header = ['Project']
176  for s in Severity.levels:
177    if total_by_severity[s.value]:
178      stats_header.append(
179          '<span style=\'background-color:{}\'>{}</span>'.format(
180              s.color, s.column_header))
181  stats_header.append('TOTAL')
182  return stats_header
183
184
185def emit_row_counts_per_project(warnings, total_by_project, total_by_severity,
186                                project_names):
187  """Returns total project warnings and row of stats for each project.
188
189  Args:
190    warnings: output of create_warnings(warn_patterns, project_names)
191    total_by_project: output of get_total_by_project(project_names)
192    total_by_severity: output of get_total_by_severity(project_names)
193    project_names: list of project names
194
195  Returns:
196    total_all_projects, the total number of warnings over all projects
197    stats_rows, a 2d list where each row is [Project Name, <severity counts>,
198    total # warnings for this project]
199  """
200
201  total_all_projects = 0
202  stats_rows = []
203  for p in project_names:
204    if total_by_project[p]:
205      one_row = [p]
206      for s in Severity.levels:
207        if total_by_severity[s.value]:
208          one_row.append(warnings[p][s.value])
209      one_row.append(total_by_project[p])
210      stats_rows.append(one_row)
211      total_all_projects += total_by_project[p]
212  return total_all_projects, stats_rows
213
214
215def emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows,
216                                 total_all_projects, writer):
217  """Emits stats_header and stats_rows as specified above.
218
219  Args:
220    total_by_severity: output of get_total_by_severity()
221    stats_header: output of emit_table_header()
222    stats_rows: output of emit_row_counts_per_project()
223    total_all_projects: output of emit_row_counts_per_project()
224    writer: writer returned by make_writer(output_stream)
225  """
226
227  total_all_severities = 0
228  one_row = ['<b>TOTAL</b>']
229  for s in Severity.levels:
230    if total_by_severity[s.value]:
231      one_row.append(total_by_severity[s.value])
232      total_all_severities += total_by_severity[s.value]
233  one_row.append(total_all_projects)
234  stats_rows.append(one_row)
235  writer('<script>')
236  emit_const_string_array('StatsHeader', stats_header, writer)
237  emit_const_object_array('StatsRows', stats_rows, writer)
238  writer(draw_table_javascript)
239  writer('</script>')
240
241
242def emit_stats_by_project(writer, warn_patterns, project_names):
243  """Dump a google chart table of warnings per project and severity."""
244
245  warnings = create_warnings(warn_patterns, project_names)
246  total_by_project = get_total_by_project(warnings, project_names)
247  total_by_severity = get_total_by_severity(warnings, project_names)
248  stats_header = emit_table_header(total_by_severity)
249  total_all_projects, stats_rows = \
250    emit_row_counts_per_project(warnings, total_by_project, total_by_severity, project_names)
251  emit_row_counts_per_severity(total_by_severity, stats_header, stats_rows,
252                               total_all_projects, writer)
253
254
255def dump_stats(writer, warn_patterns):
256  """Dump some stats about total number of warnings and such."""
257
258  known = 0
259  skipped = 0
260  unknown = 0
261  sort_warnings(warn_patterns)
262  for i in warn_patterns:
263    if i['severity'] == Severity.UNMATCHED:
264      unknown += len(i['members'])
265    elif i['severity'] == Severity.SKIP:
266      skipped += len(i['members'])
267    else:
268      known += len(i['members'])
269  writer('Number of classified warnings: <b>' + str(known) + '</b><br>')
270  writer('Number of skipped warnings: <b>' + str(skipped) + '</b><br>')
271  writer('Number of unclassified warnings: <b>' + str(unknown) + '</b><br>')
272  total = unknown + known + skipped
273  extra_msg = ''
274  if total < 1000:
275    extra_msg = ' (low count may indicate incremental build)'
276  writer('Total number of warnings: <b>' + str(total) + '</b>' + extra_msg)
277
278
279# New base table of warnings, [severity, warn_id, project, warning_message]
280# Need buttons to show warnings in different grouping options.
281# (1) Current, group by severity, id for each warning pattern
282#     sort by severity, warn_id, warning_message
283# (2) Current --byproject, group by severity,
284#     id for each warning pattern + project name
285#     sort by severity, warn_id, project, warning_message
286# (3) New, group by project + severity,
287#     id for each warning pattern
288#     sort by project, severity, warn_id, warning_message
289def emit_buttons(writer):
290  writer('<button class="button" onclick="expandCollapse(1);">'
291         'Expand all warnings</button>\n'
292         '<button class="button" onclick="expandCollapse(0);">'
293         'Collapse all warnings</button>\n'
294         '<button class="button" onclick="groupBySeverity();">'
295         'Group warnings by severity</button>\n'
296         '<button class="button" onclick="groupByProject();">'
297         'Group warnings by project</button><br>')
298
299
300def all_patterns(category):
301  patterns = ''
302  for i in category['patterns']:
303    patterns += i
304    patterns += ' / '
305  return patterns
306
307
308def dump_fixed(writer, warn_patterns):
309  """Show which warnings no longer occur."""
310  anchor = 'fixed_warnings'
311  mark = anchor + '_mark'
312  writer('\n<br><p style="background-color:lightblue"><b>'
313         '<button id="' + mark + '" '
314         'class="bt" onclick="expand(\'' + anchor + '\');">'
315         '&#x2295</button> Fixed warnings. '
316         'No more occurrences. Please consider turning these into '
317         'errors if possible, before they are reintroduced in to the build'
318         ':</b></p>')
319  writer('<blockquote>')
320  fixed_patterns = []
321  for i in warn_patterns:
322    if not i['members']:
323      fixed_patterns.append(i['description'] + ' (' + all_patterns(i) + ')')
324  fixed_patterns = sorted(fixed_patterns)
325  writer('<div id="' + anchor + '" style="display:none;"><table>')
326  cur_row_class = 0
327  for text in fixed_patterns:
328    cur_row_class = 1 - cur_row_class
329    # remove last '\n'
330    t = text[:-1] if text[-1] == '\n' else text
331    writer('<tr><td class="c' + str(cur_row_class) + '">' + t + '</td></tr>')
332  writer('</table></div>')
333  writer('</blockquote>')
334
335
336def write_severity(csvwriter, sev, kind, warn_patterns):
337  """Count warnings of given severity and write CSV entries to writer."""
338  total = 0
339  for pattern in warn_patterns:
340    if pattern['severity'] == sev and pattern['members']:
341      n = len(pattern['members'])
342      total += n
343      warning = kind + ': ' + (pattern['description'] or '?')
344      csvwriter.writerow([n, '', warning])
345      # print number of warnings for each project, ordered by project name
346      projects = sorted(pattern['projects'].keys())
347      for project in projects:
348        csvwriter.writerow([pattern['projects'][project], project, warning])
349  csvwriter.writerow([total, '', kind + ' warnings'])
350  return total
351
352
353def dump_csv(csvwriter, warn_patterns):
354  """Dump number of warnings in CSV format to writer."""
355  sort_warnings(warn_patterns)
356  total = 0
357  for s in Severity.levels:
358    total += write_severity(csvwriter, s, s.column_header, warn_patterns)
359  csvwriter.writerow([total, '', 'All warnings'])
360
361
362# Return s with escaped backslash and quotation characters.
363def escape_string(s):
364  return s.replace('\\', '\\\\').replace('"', '\\"')
365
366
367# Return s without trailing '\n' and escape the quotation characters.
368def strip_escape_string(s):
369  if not s:
370    return s
371  s = s[:-1] if s[-1] == '\n' else s
372  return escape_string(s)
373
374
375def emit_warning_array(name, writer, warn_patterns):
376  writer('var warning_{} = ['.format(name))
377  for w in warn_patterns:
378    if name == 'severity':
379      writer('{},'.format(w[name].value))
380    else:
381      writer('{},'.format(w[name]))
382  writer('];')
383
384
385def emit_warning_arrays(writer, warn_patterns):
386  emit_warning_array('severity', writer, warn_patterns)
387  writer('var warning_description = [')
388  for w in warn_patterns:
389    if w['members']:
390      writer('"{}",'.format(escape_string(w['description'])))
391    else:
392      writer('"",')  # no such warning
393  writer('];')
394
395
396scripts_for_warning_groups = """
397  function compareMessages(x1, x2) { // of the same warning type
398    return (WarningMessages[x1[2]] <= WarningMessages[x2[2]]) ? -1 : 1;
399  }
400  function byMessageCount(x1, x2) {
401    return x2[2] - x1[2];  // reversed order
402  }
403  function bySeverityMessageCount(x1, x2) {
404    // orer by severity first
405    if (x1[1] != x2[1])
406      return  x1[1] - x2[1];
407    return byMessageCount(x1, x2);
408  }
409  const ParseLinePattern = /^([^ :]+):(\\d+):(.+)/;
410  function addURL(line) { // used by Android
411    if (FlagURL == "") return line;
412    if (FlagSeparator == "") {
413      return line.replace(ParseLinePattern,
414        "<a target='_blank' href='" + FlagURL + "/$1'>$1</a>:$2:$3");
415    }
416    return line.replace(ParseLinePattern,
417      "<a target='_blank' href='" + FlagURL + "/$1" + FlagSeparator +
418        "$2'>$1:$2</a>:$3");
419  }
420  function addURLToLine(line, link) { // used by Chrome
421      let line_split = line.split(":");
422      let path = line_split.slice(0,3).join(":");
423      let msg = line_split.slice(3).join(":");
424      let html_link = `<a target="_blank" href="${link}">${path}</a>${msg}`;
425      return html_link;
426  }
427  function createArrayOfDictionaries(n) {
428    var result = [];
429    for (var i=0; i<n; i++) result.push({});
430    return result;
431  }
432  function groupWarningsBySeverity() {
433    // groups is an array of dictionaries,
434    // each dictionary maps from warning type to array of warning messages.
435    var groups = createArrayOfDictionaries(SeverityColors.length);
436    for (var i=0; i<Warnings.length; i++) {
437      var w = Warnings[i][0];
438      var s = WarnPatternsSeverity[w];
439      var k = w.toString();
440      if (!(k in groups[s]))
441        groups[s][k] = [];
442      groups[s][k].push(Warnings[i]);
443    }
444    return groups;
445  }
446  function groupWarningsByProject() {
447    var groups = createArrayOfDictionaries(ProjectNames.length);
448    for (var i=0; i<Warnings.length; i++) {
449      var w = Warnings[i][0];
450      var p = Warnings[i][1];
451      var k = w.toString();
452      if (!(k in groups[p]))
453        groups[p][k] = [];
454      groups[p][k].push(Warnings[i]);
455    }
456    return groups;
457  }
458  var GlobalAnchor = 0;
459  function createWarningSection(header, color, group) {
460    var result = "";
461    var groupKeys = [];
462    var totalMessages = 0;
463    for (var k in group) {
464       totalMessages += group[k].length;
465       groupKeys.push([k, WarnPatternsSeverity[parseInt(k)], group[k].length]);
466    }
467    groupKeys.sort(bySeverityMessageCount);
468    for (var idx=0; idx<groupKeys.length; idx++) {
469      var k = groupKeys[idx][0];
470      var messages = group[k];
471      var w = parseInt(k);
472      var wcolor = SeverityColors[WarnPatternsSeverity[w]];
473      var description = WarnPatternsDescription[w];
474      if (description.length == 0)
475          description = "???";
476      GlobalAnchor += 1;
477      result += "<table class='t1'><tr bgcolor='" + wcolor + "'><td>" +
478                "<button class='bt' id='" + GlobalAnchor + "_mark" +
479                "' onclick='expand(\\"" + GlobalAnchor + "\\");'>" +
480                "&#x2295</button> " +
481                description + " (" + messages.length + ")</td></tr></table>";
482      result += "<div id='" + GlobalAnchor +
483                "' style='display:none;'><table class='t1'>";
484      var c = 0;
485      messages.sort(compareMessages);
486      if (FlagPlatform == "chrome") {
487        for (var i=0; i<messages.length; i++) {
488          result += "<tr><td class='c" + c + "'>" +
489                    addURLToLine(WarningMessages[messages[i][2]], WarningLinks[messages[i][3]]) + "</td></tr>";
490          c = 1 - c;
491        }
492      } else {
493        for (var i=0; i<messages.length; i++) {
494          result += "<tr><td class='c" + c + "'>" +
495                    addURL(WarningMessages[messages[i][2]]) + "</td></tr>";
496          c = 1 - c;
497        }
498      }
499      result += "</table></div>";
500    }
501    if (result.length > 0) {
502      return "<br><span style='background-color:" + color + "'><b>" +
503             header + ": " + totalMessages +
504             "</b></span><blockquote><table class='t1'>" +
505             result + "</table></blockquote>";
506
507    }
508    return "";  // empty section
509  }
510  function generateSectionsBySeverity() {
511    var result = "";
512    var groups = groupWarningsBySeverity();
513    for (s=0; s<SeverityColors.length; s++) {
514      result += createWarningSection(SeverityHeaders[s], SeverityColors[s],
515                                     groups[s]);
516    }
517    return result;
518  }
519  function generateSectionsByProject() {
520    var result = "";
521    var groups = groupWarningsByProject();
522    for (i=0; i<groups.length; i++) {
523      result += createWarningSection(ProjectNames[i], 'lightgrey', groups[i]);
524    }
525    return result;
526  }
527  function groupWarnings(generator) {
528    GlobalAnchor = 0;
529    var e = document.getElementById("warning_groups");
530    e.innerHTML = generator();
531  }
532  function groupBySeverity() {
533    groupWarnings(generateSectionsBySeverity);
534  }
535  function groupByProject() {
536    groupWarnings(generateSectionsByProject);
537  }
538"""
539
540
541# Emit a JavaScript const string
542def emit_const_string(name, value, writer):
543  writer('const ' + name + ' = "' + escape_string(value) + '";')
544
545
546# Emit a JavaScript const integer array.
547def emit_const_int_array(name, array, writer):
548  writer('const ' + name + ' = [')
549  for n in array:
550    writer(str(n) + ',')
551  writer('];')
552
553
554# Emit a JavaScript const string array.
555def emit_const_string_array(name, array, writer):
556  writer('const ' + name + ' = [')
557  for s in array:
558    writer('"' + strip_escape_string(s) + '",')
559  writer('];')
560
561
562# Emit a JavaScript const string array for HTML.
563def emit_const_html_string_array(name, array, writer):
564  writer('const ' + name + ' = [')
565  for s in array:
566    # Not using html.escape yet, to work for both python 2 and 3,
567    # until all users switch to python 3.
568    # pylint:disable=deprecated-method
569    writer('"' + cgi.escape(strip_escape_string(s)) + '",')
570  writer('];')
571
572
573# Emit a JavaScript const object array.
574def emit_const_object_array(name, array, writer):
575  writer('const ' + name + ' = [')
576  for x in array:
577    writer(str(x) + ',')
578  writer('];')
579
580
581def emit_js_data(writer, flags, warning_messages, warning_links,
582                 warning_records, warn_patterns, project_names):
583  """Dump dynamic HTML page's static JavaScript data."""
584  emit_const_string('FlagPlatform', flags.platform, writer)
585  emit_const_string('FlagURL', flags.url, writer)
586  emit_const_string('FlagSeparator', flags.separator, writer)
587  emit_const_string_array('SeverityColors', [s.color for s in Severity.levels],
588                          writer)
589  emit_const_string_array('SeverityHeaders',
590                          [s.header for s in Severity.levels], writer)
591  emit_const_string_array('SeverityColumnHeaders',
592                          [s.column_header for s in Severity.levels], writer)
593  emit_const_string_array('ProjectNames', project_names, writer)
594  # pytype: disable=attribute-error
595  emit_const_int_array('WarnPatternsSeverity',
596                       [w['severity'].value for w in warn_patterns], writer)
597  # pytype: enable=attribute-error
598  emit_const_html_string_array('WarnPatternsDescription',
599                               [w['description'] for w in warn_patterns],
600                               writer)
601  emit_const_html_string_array('WarningMessages', warning_messages, writer)
602  emit_const_object_array('Warnings', warning_records, writer)
603  if flags.platform == 'chrome':
604    emit_const_html_string_array('WarningLinks', warning_links, writer)
605
606
607draw_table_javascript = """
608google.charts.load('current', {'packages':['table']});
609google.charts.setOnLoadCallback(drawTable);
610function drawTable() {
611  var data = new google.visualization.DataTable();
612  data.addColumn('string', StatsHeader[0]);
613  for (var i=1; i<StatsHeader.length; i++) {
614    data.addColumn('number', StatsHeader[i]);
615  }
616  data.addRows(StatsRows);
617  for (var i=0; i<StatsRows.length; i++) {
618    for (var j=0; j<StatsHeader.length; j++) {
619      data.setProperty(i, j, 'style', 'border:1px solid black;');
620    }
621  }
622  var table = new google.visualization.Table(
623      document.getElementById('stats_table'));
624  table.draw(data, {allowHtml: true, alternatingRowStyle: true});
625}
626"""
627
628
629def dump_html(flags, output_stream, warning_messages, warning_links,
630              warning_records, header_str, warn_patterns, project_names):
631  """Dump the flags output to output_stream."""
632  writer = make_writer(output_stream)
633  dump_html_prologue('Warnings for ' + header_str, writer, warn_patterns,
634                     project_names)
635  dump_stats(writer, warn_patterns)
636  writer('<br><div id="stats_table"></div><br>')
637  writer('\n<script>')
638  emit_js_data(writer, flags, warning_messages, warning_links, warning_records,
639               warn_patterns, project_names)
640  writer(scripts_for_warning_groups)
641  writer('</script>')
642  emit_buttons(writer)
643  # Warning messages are grouped by severities or project names.
644  writer('<br><div id="warning_groups"></div>')
645  if flags.byproject:
646    writer('<script>groupByProject();</script>')
647  else:
648    writer('<script>groupBySeverity();</script>')
649  dump_fixed(writer, warn_patterns)
650  dump_html_epilogue(writer)
651
652
653def write_html(flags, project_names, warn_patterns, html_path, warning_messages,
654               warning_links, warning_records, header_str):
655  """Write warnings html file."""
656  if html_path:
657    with open(html_path, 'w') as f:
658      dump_html(flags, f, warning_messages, warning_links, warning_records,
659                header_str, warn_patterns, project_names)
660
661
662def write_out_csv(flags, warn_patterns, warning_messages, warning_links,
663                  warning_records, header_str, project_names):
664  """Write warnings csv file."""
665  if flags.csvpath:
666    with open(flags.csvpath, 'w') as f:
667      dump_csv(csv.writer(f, lineterminator='\n'), warn_patterns)
668
669  if flags.gencsv:
670    dump_csv(csv.writer(sys.stdout, lineterminator='\n'), warn_patterns)
671  else:
672    dump_html(flags, sys.stdout, warning_messages, warning_links,
673              warning_records, header_str, warn_patterns, project_names)
674