#!/usr/bin/env python # # Copyright (C) 2012 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Usage: generate-notice-files --text-output [plain text output file] \ --html-output [html output file] \ --xml-output [xml output file] \ -t [file title] -s [directory of notices] Generate the Android notice files, including both text and html files. -h to display this usage message and exit. """ from collections import defaultdict import argparse import hashlib import itertools import os import os.path import re import sys MD5_BLOCKSIZE = 1024 * 1024 HTML_ESCAPE_TABLE = { "&": "&", '"': """, "'": "'", ">": ">", "<": "<", } def hexify(s): return ("%02x"*len(s)) % tuple(map(ord, s)) def md5sum(filename): """Calculate an MD5 of the file given by FILENAME, and return hex digest as a string. Output should be compatible with md5sum command""" f = open(filename, "rb") sum = hashlib.md5() while 1: block = f.read(MD5_BLOCKSIZE) if not block: break sum.update(block) f.close() return hexify(sum.digest()) def html_escape(text): """Produce entities within text.""" return "".join(HTML_ESCAPE_TABLE.get(c,c) for c in text) HTML_OUTPUT_CSS=""" """ def combine_notice_files_html(file_hash, input_dir, output_filename): """Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME.""" SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") # Set up a filename to row id table (anchors inside tables don't work in # most browsers, but href's to table row ids do) id_table = {} id_count = 0 for value in file_hash: for filename in value: id_table[filename] = id_count id_count += 1 # Open the output file, and output the header pieces output_file = open(output_filename, "wb") print >> output_file, "" print >> output_file, HTML_OUTPUT_CSS print >> output_file, '' # Output our table of contents print >> output_file, '
' print >> output_file, "" print >> output_file, "
" # Output the individual notice file lists print >>output_file, '' for value in file_hash: print >> output_file, '" print >> output_file print >> output_file print >> output_file # Finish off the file output print >> output_file, "
' % id_table.get(value[0]) print >> output_file, '
Notices for file(s):
' print >> output_file, '
' for filename in value: print >> output_file, "%s
" % (SRC_DIR_STRIP_RE.sub(r"\1", filename)) print >> output_file, "
" print >> output_file print >> output_file, '
'
        print >> output_file, html_escape(open(value[0]).read())
        print >> output_file, "
" print >> output_file, "
" print >> output_file, "" output_file.close() def combine_notice_files_text(file_hash, input_dir, output_filename, file_title): """Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME.""" SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") output_file = open(output_filename, "wb") print >> output_file, file_title for value in file_hash: print >> output_file, "============================================================" print >> output_file, "Notices for file(s):" for filename in value: print >> output_file, SRC_DIR_STRIP_RE.sub(r"\1", filename) print >> output_file, "------------------------------------------------------------" print >> output_file, open(value[0]).read() output_file.close() def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename): """Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME.""" SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") # Set up a filename to row id table (anchors inside tables don't work in # most browsers, but href's to table row ids do) id_table = {} for file_key in files_with_same_hash.keys(): for filename in files_with_same_hash[file_key]: id_table[filename] = file_key # Open the output file, and output the header pieces output_file = open(output_filename, "wb") print >> output_file, '' print >> output_file, "" # Flatten the list of lists into a single list of filenames sorted_filenames = sorted(id_table.keys()) # Print out a nice table of contents for filename in sorted_filenames: stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) print >> output_file, '%s' % (id_table.get(filename), stripped_filename) print >> output_file print >> output_file processed_file_keys = [] # Output the individual notice file lists for filename in sorted_filenames: file_key = id_table.get(filename) if file_key in processed_file_keys: continue processed_file_keys.append(file_key) print >> output_file, '' % (file_key, html_escape(open(filename).read())) print >> output_file # Finish off the file output print >> output_file, "" output_file.close() def get_args(): parser = argparse.ArgumentParser() parser.add_argument( '--text-output', required=True, help='The text output file path.') parser.add_argument( '--html-output', help='The html output file path.') parser.add_argument( '--xml-output', help='The xml output file path.') parser.add_argument( '-t', '--title', required=True, help='The file title.') parser.add_argument( '-s', '--source-dir', required=True, help='The directory containing notices.') parser.add_argument( '-i', '--included-subdirs', action='append', help='The sub directories which should be included.') parser.add_argument( '-e', '--excluded-subdirs', action='append', help='The sub directories which should be excluded.') return parser.parse_args() def main(argv): args = get_args() txt_output_file = args.text_output html_output_file = args.html_output xml_output_file = args.xml_output file_title = args.title included_subdirs = [] excluded_subdirs = [] if args.included_subdirs is not None: included_subdirs = args.included_subdirs if args.excluded_subdirs is not None: excluded_subdirs = args.excluded_subdirs # Find all the notice files and md5 them input_dir = os.path.normpath(args.source_dir) files_with_same_hash = defaultdict(list) for root, dir, files in os.walk(input_dir): for file in files: matched = True if len(included_subdirs) > 0: matched = False for subdir in included_subdirs: if (root == (input_dir + '/' + subdir) or root.startswith(input_dir + '/' + subdir + '/')): matched = True break elif len(excluded_subdirs) > 0: for subdir in excluded_subdirs: if (root == (input_dir + '/' + subdir) or root.startswith(input_dir + '/' + subdir + '/')): matched = False break if matched and file.endswith(".txt"): filename = os.path.join(root, file) file_md5sum = md5sum(filename) files_with_same_hash[file_md5sum].append(filename) filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(files_with_same_hash.keys())] combine_notice_files_text(filesets, input_dir, txt_output_file, file_title) if html_output_file is not None: combine_notice_files_html(filesets, input_dir, html_output_file) if xml_output_file is not None: combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file) if __name__ == "__main__": main(sys.argv)