The Tor Browser / file comparison
comparison: testing/talos/talos_from_code.py
testing/talos/talos_from_code.py
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:35a9d87e180d |
|---|---|
| 1 #! /usr/bin/env python | |
| 2 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 3 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 5 | |
| 6 # | |
| 7 # Script name: talos_from_code.py | |
| 8 # Purpose: Read from a talos.json file the different files to download for a talos job | |
| 9 # Author(s): Zambrano Gasparnian, Armen <armenzg@mozilla.com> | |
| 10 # Target: Python 2.5 | |
| 11 # | |
| 12 from optparse import OptionParser | |
| 13 import json | |
| 14 import re | |
| 15 import urllib2 | |
| 16 import urlparse | |
| 17 import sys | |
| 18 import os | |
| 19 | |
| 20 def main(): | |
| 21 ''' | |
| 22 This script downloads a talos.json file which indicates which files to download | |
| 23 for a talos job. | |
| 24 See a talos.json file for a better understand: | |
| 25 http://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json | |
| 26 ''' | |
| 27 parser = OptionParser() | |
| 28 parser.add_option("--talos-json-url", dest="talos_json_url", type="string", | |
| 29 help="It indicates from where to download the talos.json file.") | |
| 30 (options, args) = parser.parse_args() | |
| 31 | |
| 32 # 1) check that the url was passed | |
| 33 if options.talos_json_url == None: | |
| 34 print "You need to specify --talos-json-url." | |
| 35 sys.exit(1) | |
| 36 | |
| 37 # 2) try to download the talos.json file | |
| 38 try: | |
| 39 jsonFilename = download_file(options.talos_json_url) | |
| 40 except Exception, e: | |
| 41 print "ERROR: We tried to download the talos.json file but something failed." | |
| 42 print "ERROR: %s" % str(e) | |
| 43 sys.exit(1) | |
| 44 | |
| 45 # 3) download the necessary files | |
| 46 print "INFO: talos.json URL: %s" % options.talos_json_url | |
| 47 try: | |
| 48 key = 'talos.zip' | |
| 49 entity = get_value(jsonFilename, key) | |
| 50 if passesRestrictions(options.talos_json_url, entity["url"]): | |
| 51 # the key is at the same time the filename e.g. talos.zip | |
| 52 print "INFO: Downloading %s as %s" % (entity["url"], os.path.join(entity["path"], key)) | |
| 53 download_file(entity["url"], entity["path"], key) | |
| 54 else: | |
| 55 print "ERROR: You have tried to download a file " + \ | |
| 56 "from: %s " % fileUrl + \ | |
| 57 "which is a location different than http://talos-bundles.pvt.build.mozilla.org/" | |
| 58 print "ERROR: This is only allowed for the certain branches." | |
| 59 sys.exit(1) | |
| 60 except Exception, e: | |
| 61 print "ERROR: %s" % str(e) | |
| 62 sys.exit(1) | |
| 63 | |
| 64 def passesRestrictions(talosJsonUrl, fileUrl): | |
| 65 ''' | |
| 66 Only certain branches are exempted from having to host their downloadable files | |
| 67 in talos-bundles.pvt.build.mozilla.org | |
| 68 ''' | |
| 69 if talosJsonUrl.startswith("http://hg.mozilla.org/try/") == True or \ | |
| 70 talosJsonUrl.startswith("https://hg.mozilla.org/try/") == True or \ | |
| 71 talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/") == True or \ | |
| 72 talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/") == True: | |
| 73 return True | |
| 74 else: | |
| 75 p = re.compile('^http://talos-bundles.pvt.build.mozilla.org/') | |
| 76 m = p.match(fileUrl) | |
| 77 if m == None: | |
| 78 return False | |
| 79 return True | |
| 80 | |
| 81 def get_filename_from_url(url): | |
| 82 ''' | |
| 83 This returns the filename of the file we're trying to download | |
| 84 ''' | |
| 85 parsed = urlparse.urlsplit(url.rstrip('/')) | |
| 86 if parsed.path != '': | |
| 87 return parsed.path.rsplit('/', 1)[-1] | |
| 88 else: | |
| 89 print "ERROR: We were trying to download a file from %s " + \ | |
| 90 "but the URL seems to be incorrect." | |
| 91 sys.exit(1) | |
| 92 | |
| 93 def download_file(url, path="", saveAs=None): | |
| 94 ''' | |
| 95 It downloads a file from URL to the indicated path | |
| 96 ''' | |
| 97 req = urllib2.Request(url) | |
| 98 f = urllib2.urlopen(req) | |
| 99 if path != "" and not os.path.isdir(path): | |
| 100 try: | |
| 101 os.makedirs(path) | |
| 102 print "INFO: directory %s created" % path | |
| 103 except Exception, e: | |
| 104 print "ERROR: %s" % str(e) | |
| 105 sys.exit(1) | |
| 106 filename = saveAs if saveAs else get_filename_from_url(url) | |
| 107 local_file = open(os.path.join(path, filename), 'wb') | |
| 108 local_file.write(f.read()) | |
| 109 local_file.close() | |
| 110 return filename | |
| 111 | |
| 112 def get_value(json_filename, key): | |
| 113 ''' | |
| 114 It loads up a JSON file and returns the value for the given string | |
| 115 ''' | |
| 116 f = open(json_filename, 'r') | |
| 117 return json.load(f)[key] | |
| 118 | |
| 119 if __name__ == '__main__': | |
| 120 main() |
