testing/talos/talos_from_code.py@7e26c7da4463 (annotated)
testing/talos/talos_from_code.py
Wed, 31 Dec 2014 06:55:50 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:55:50 +0100
- changeset 2
- 7e26c7da4463
- permissions
- -rw-r--r--
Added tag UPSTREAM_283F7C6 for changeset ca08bd8f51b2
| michael@0 | 1 | #! /usr/bin/env python |
| michael@0 | 2 | # This Source Code Form is subject to the terms of the Mozilla Public |
| michael@0 | 3 | # License, v. 2.0. If a copy of the MPL was not distributed with this |
| michael@0 | 4 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| michael@0 | 5 | |
| michael@0 | 6 | # |
| michael@0 | 7 | # Script name: talos_from_code.py |
| michael@0 | 8 | # Purpose: Read from a talos.json file the different files to download for a talos job |
| michael@0 | 9 | # Author(s): Zambrano Gasparnian, Armen <armenzg@mozilla.com> |
| michael@0 | 10 | # Target: Python 2.5 |
| michael@0 | 11 | # |
| michael@0 | 12 | from optparse import OptionParser |
| michael@0 | 13 | import json |
| michael@0 | 14 | import re |
| michael@0 | 15 | import urllib2 |
| michael@0 | 16 | import urlparse |
| michael@0 | 17 | import sys |
| michael@0 | 18 | import os |
| michael@0 | 19 | |
| michael@0 | 20 | def main(): |
| michael@0 | 21 | ''' |
| michael@0 | 22 | This script downloads a talos.json file which indicates which files to download |
| michael@0 | 23 | for a talos job. |
| michael@0 | 24 | See a talos.json file for a better understand: |
| michael@0 | 25 | http://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json |
| michael@0 | 26 | ''' |
| michael@0 | 27 | parser = OptionParser() |
| michael@0 | 28 | parser.add_option("--talos-json-url", dest="talos_json_url", type="string", |
| michael@0 | 29 | help="It indicates from where to download the talos.json file.") |
| michael@0 | 30 | (options, args) = parser.parse_args() |
| michael@0 | 31 | |
| michael@0 | 32 | # 1) check that the url was passed |
| michael@0 | 33 | if options.talos_json_url == None: |
| michael@0 | 34 | print "You need to specify --talos-json-url." |
| michael@0 | 35 | sys.exit(1) |
| michael@0 | 36 | |
| michael@0 | 37 | # 2) try to download the talos.json file |
| michael@0 | 38 | try: |
| michael@0 | 39 | jsonFilename = download_file(options.talos_json_url) |
| michael@0 | 40 | except Exception, e: |
| michael@0 | 41 | print "ERROR: We tried to download the talos.json file but something failed." |
| michael@0 | 42 | print "ERROR: %s" % str(e) |
| michael@0 | 43 | sys.exit(1) |
| michael@0 | 44 | |
| michael@0 | 45 | # 3) download the necessary files |
| michael@0 | 46 | print "INFO: talos.json URL: %s" % options.talos_json_url |
| michael@0 | 47 | try: |
| michael@0 | 48 | key = 'talos.zip' |
| michael@0 | 49 | entity = get_value(jsonFilename, key) |
| michael@0 | 50 | if passesRestrictions(options.talos_json_url, entity["url"]): |
| michael@0 | 51 | # the key is at the same time the filename e.g. talos.zip |
| michael@0 | 52 | print "INFO: Downloading %s as %s" % (entity["url"], os.path.join(entity["path"], key)) |
| michael@0 | 53 | download_file(entity["url"], entity["path"], key) |
| michael@0 | 54 | else: |
| michael@0 | 55 | print "ERROR: You have tried to download a file " + \ |
| michael@0 | 56 | "from: %s " % fileUrl + \ |
| michael@0 | 57 | "which is a location different than http://talos-bundles.pvt.build.mozilla.org/" |
| michael@0 | 58 | print "ERROR: This is only allowed for the certain branches." |
| michael@0 | 59 | sys.exit(1) |
| michael@0 | 60 | except Exception, e: |
| michael@0 | 61 | print "ERROR: %s" % str(e) |
| michael@0 | 62 | sys.exit(1) |
| michael@0 | 63 | |
| michael@0 | 64 | def passesRestrictions(talosJsonUrl, fileUrl): |
| michael@0 | 65 | ''' |
| michael@0 | 66 | Only certain branches are exempted from having to host their downloadable files |
| michael@0 | 67 | in talos-bundles.pvt.build.mozilla.org |
| michael@0 | 68 | ''' |
| michael@0 | 69 | if talosJsonUrl.startswith("http://hg.mozilla.org/try/") == True or \ |
| michael@0 | 70 | talosJsonUrl.startswith("https://hg.mozilla.org/try/") == True or \ |
| michael@0 | 71 | talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/") == True or \ |
| michael@0 | 72 | talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/") == True: |
| michael@0 | 73 | return True |
| michael@0 | 74 | else: |
| michael@0 | 75 | p = re.compile('^http://talos-bundles.pvt.build.mozilla.org/') |
| michael@0 | 76 | m = p.match(fileUrl) |
| michael@0 | 77 | if m == None: |
| michael@0 | 78 | return False |
| michael@0 | 79 | return True |
| michael@0 | 80 | |
| michael@0 | 81 | def get_filename_from_url(url): |
| michael@0 | 82 | ''' |
| michael@0 | 83 | This returns the filename of the file we're trying to download |
| michael@0 | 84 | ''' |
| michael@0 | 85 | parsed = urlparse.urlsplit(url.rstrip('/')) |
| michael@0 | 86 | if parsed.path != '': |
| michael@0 | 87 | return parsed.path.rsplit('/', 1)[-1] |
| michael@0 | 88 | else: |
| michael@0 | 89 | print "ERROR: We were trying to download a file from %s " + \ |
| michael@0 | 90 | "but the URL seems to be incorrect." |
| michael@0 | 91 | sys.exit(1) |
| michael@0 | 92 | |
| michael@0 | 93 | def download_file(url, path="", saveAs=None): |
| michael@0 | 94 | ''' |
| michael@0 | 95 | It downloads a file from URL to the indicated path |
| michael@0 | 96 | ''' |
| michael@0 | 97 | req = urllib2.Request(url) |
| michael@0 | 98 | f = urllib2.urlopen(req) |
| michael@0 | 99 | if path != "" and not os.path.isdir(path): |
| michael@0 | 100 | try: |
| michael@0 | 101 | os.makedirs(path) |
| michael@0 | 102 | print "INFO: directory %s created" % path |
| michael@0 | 103 | except Exception, e: |
| michael@0 | 104 | print "ERROR: %s" % str(e) |
| michael@0 | 105 | sys.exit(1) |
| michael@0 | 106 | filename = saveAs if saveAs else get_filename_from_url(url) |
| michael@0 | 107 | local_file = open(os.path.join(path, filename), 'wb') |
| michael@0 | 108 | local_file.write(f.read()) |
| michael@0 | 109 | local_file.close() |
| michael@0 | 110 | return filename |
| michael@0 | 111 | |
| michael@0 | 112 | def get_value(json_filename, key): |
| michael@0 | 113 | ''' |
| michael@0 | 114 | It loads up a JSON file and returns the value for the given string |
| michael@0 | 115 | ''' |
| michael@0 | 116 | f = open(json_filename, 'r') |
| michael@0 | 117 | return json.load(f)[key] |
| michael@0 | 118 | |
| michael@0 | 119 | if __name__ == '__main__': |
| michael@0 | 120 | main() |
