build/gen_mach_buildprops.py@7e26c7da4463 (annotated)
build/gen_mach_buildprops.py
Wed, 31 Dec 2014 06:55:50 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:55:50 +0100
- changeset 2
- 7e26c7da4463
- permissions
- -rw-r--r--
Added tag UPSTREAM_283F7C6 for changeset ca08bd8f51b2
| michael@0 | 1 | #!/usr/bin/python |
| michael@0 | 2 | # |
| michael@0 | 3 | # This Source Code Form is subject to the terms of the Mozilla Public |
| michael@0 | 4 | # License, v. 2.0. If a copy of the MPL was not distributed with this |
| michael@0 | 5 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| michael@0 | 6 | |
| michael@0 | 7 | import sys |
| michael@0 | 8 | import os |
| michael@0 | 9 | import hashlib |
| michael@0 | 10 | import json |
| michael@0 | 11 | import re |
| michael@0 | 12 | import errno |
| michael@0 | 13 | from argparse import ArgumentParser |
| michael@0 | 14 | |
| michael@0 | 15 | def getFileHashAndSize(filename): |
| michael@0 | 16 | sha512Hash = 'UNKNOWN' |
| michael@0 | 17 | size = 'UNKNOWN' |
| michael@0 | 18 | |
| michael@0 | 19 | try: |
| michael@0 | 20 | # open in binary mode to make sure we get consistent results |
| michael@0 | 21 | # across all platforms |
| michael@0 | 22 | f = open(filename, "rb") |
| michael@0 | 23 | shaObj = hashlib.sha512(f.read()) |
| michael@0 | 24 | sha512Hash = shaObj.hexdigest() |
| michael@0 | 25 | |
| michael@0 | 26 | size = os.path.getsize(filename) |
| michael@0 | 27 | except: |
| michael@0 | 28 | pass |
| michael@0 | 29 | |
| michael@0 | 30 | return (sha512Hash, size) |
| michael@0 | 31 | |
| michael@0 | 32 | def getMarProperties(filename, partial=False): |
| michael@0 | 33 | if not os.path.exists(filename): |
| michael@0 | 34 | return {} |
| michael@0 | 35 | (mar_hash, mar_size) = getFileHashAndSize(filename) |
| michael@0 | 36 | martype = 'partial' if partial else 'complete' |
| michael@0 | 37 | return { |
| michael@0 | 38 | '%sMarFilename' % martype: os.path.basename(filename), |
| michael@0 | 39 | '%sMarSize' % martype: mar_size, |
| michael@0 | 40 | '%sMarHash' % martype: mar_hash, |
| michael@0 | 41 | } |
| michael@0 | 42 | |
| michael@0 | 43 | def getUrlProperties(filename): |
| michael@0 | 44 | # let's create a switch case using name-spaces/dict |
| michael@0 | 45 | # rather than a long if/else with duplicate code |
| michael@0 | 46 | property_conditions = [ |
| michael@0 | 47 | # key: property name, value: condition |
| michael@0 | 48 | ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or |
| michael@0 | 49 | m.endswith('crashreporter-symbols-full.zip')), |
| michael@0 | 50 | ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))), |
| michael@0 | 51 | ('unsignedApkUrl', lambda m: m.endswith('apk') and |
| michael@0 | 52 | 'unsigned-unaligned' in m), |
| michael@0 | 53 | ('robocopApkUrl', lambda m: m.endswith('apk') and 'robocop' in m), |
| michael@0 | 54 | ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')), |
| michael@0 | 55 | ('completeMarUrl', lambda m: m.endswith('.complete.mar')), |
| michael@0 | 56 | ('partialMarUrl', lambda m: m.endswith('.mar') and '.partial.' in m), |
| michael@0 | 57 | # packageUrl must be last! |
| michael@0 | 58 | ('packageUrl', lambda m: True), |
| michael@0 | 59 | ] |
| michael@0 | 60 | url_re = re.compile(r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$''') |
| michael@0 | 61 | properties = {} |
| michael@0 | 62 | |
| michael@0 | 63 | try: |
| michael@0 | 64 | with open(filename) as f: |
| michael@0 | 65 | for line in f: |
| michael@0 | 66 | m = url_re.match(line) |
| michael@0 | 67 | if m: |
| michael@0 | 68 | m = m.group(1) |
| michael@0 | 69 | for prop, condition in property_conditions: |
| michael@0 | 70 | if condition(m): |
| michael@0 | 71 | properties.update({prop: m}) |
| michael@0 | 72 | break |
| michael@0 | 73 | except IOError as e: |
| michael@0 | 74 | if e.errno != errno.ENOENT: |
| michael@0 | 75 | raise |
| michael@0 | 76 | properties = {prop: 'UNKNOWN' for prop, condition in property_conditions} |
| michael@0 | 77 | return properties |
| michael@0 | 78 | |
| michael@0 | 79 | def getPartialInfo(props): |
| michael@0 | 80 | return [{ |
| michael@0 | 81 | "from_buildid": props.get("previous_buildid"), |
| michael@0 | 82 | "size": props.get("partialMarSize"), |
| michael@0 | 83 | "hash": props.get("partialMarHash"), |
| michael@0 | 84 | "url": props.get("partialMarUrl"), |
| michael@0 | 85 | }] |
| michael@0 | 86 | |
| michael@0 | 87 | if __name__ == '__main__': |
| michael@0 | 88 | parser = ArgumentParser(description='Generate mach_build_properties.json for automation builds.') |
| michael@0 | 89 | parser.add_argument("--complete-mar-file", required=True, |
| michael@0 | 90 | action="store", dest="complete_mar_file", |
| michael@0 | 91 | help="Path to the complete MAR file, relative to the objdir.") |
| michael@0 | 92 | parser.add_argument("--partial-mar-file", required=False, |
| michael@0 | 93 | action="store", dest="partial_mar_file", |
| michael@0 | 94 | help="Path to the partial MAR file, relative to the objdir.") |
| michael@0 | 95 | parser.add_argument("--upload-output", required=True, |
| michael@0 | 96 | action="store", dest="upload_output", |
| michael@0 | 97 | help="Path to the text output of 'make upload'") |
| michael@0 | 98 | args = parser.parse_args() |
| michael@0 | 99 | |
| michael@0 | 100 | json_data = getMarProperties(args.complete_mar_file) |
| michael@0 | 101 | json_data.update(getUrlProperties(args.upload_output)) |
| michael@0 | 102 | if args.partial_mar_file: |
| michael@0 | 103 | json_data.update(getMarProperties(args.partial_mar_file, partial=True)) |
| michael@0 | 104 | |
| michael@0 | 105 | # Pull the previous buildid from the partial mar filename. |
| michael@0 | 106 | res = re.match(r'.*\.([0-9]+)-[0-9]+.mar', args.partial_mar_file) |
| michael@0 | 107 | if res: |
| michael@0 | 108 | json_data['previous_buildid'] = res.group(1) |
| michael@0 | 109 | |
| michael@0 | 110 | # Set partialInfo to be a collection of the partial mar properties |
| michael@0 | 111 | # useful for balrog. |
| michael@0 | 112 | json_data['partialInfo'] = getPartialInfo(json_data) |
| michael@0 | 113 | |
| michael@0 | 114 | with open('mach_build_properties.json', 'w') as outfile: |
| michael@0 | 115 | json.dump(json_data, outfile, indent=4) |
