1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/talos/talos_from_code.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,120 @@
1.4 +#! /usr/bin/env python
1.5 +# This Source Code Form is subject to the terms of the Mozilla Public
1.6 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.8 +
1.9 +#
1.10 +# Script name: talos_from_code.py
1.11 +# Purpose: Read from a talos.json file the different files to download for a talos job
1.12 +# Author(s): Zambrano Gasparnian, Armen <armenzg@mozilla.com>
1.13 +# Target: Python 2.5
1.14 +#
1.15 +from optparse import OptionParser
1.16 +import json
1.17 +import re
1.18 +import urllib2
1.19 +import urlparse
1.20 +import sys
1.21 +import os
1.22 +
1.23 +def main():
1.24 + '''
1.25 + This script downloads a talos.json file which indicates which files to download
1.26 + for a talos job.
1.27 + See a talos.json file for a better understand:
1.28 + http://hg.mozilla.org/mozilla-central/raw-file/default/testing/talos/talos.json
1.29 + '''
1.30 + parser = OptionParser()
1.31 + parser.add_option("--talos-json-url", dest="talos_json_url", type="string",
1.32 + help="It indicates from where to download the talos.json file.")
1.33 + (options, args) = parser.parse_args()
1.34 +
1.35 + # 1) check that the url was passed
1.36 + if options.talos_json_url == None:
1.37 + print "You need to specify --talos-json-url."
1.38 + sys.exit(1)
1.39 +
1.40 + # 2) try to download the talos.json file
1.41 + try:
1.42 + jsonFilename = download_file(options.talos_json_url)
1.43 + except Exception, e:
1.44 + print "ERROR: We tried to download the talos.json file but something failed."
1.45 + print "ERROR: %s" % str(e)
1.46 + sys.exit(1)
1.47 +
1.48 + # 3) download the necessary files
1.49 + print "INFO: talos.json URL: %s" % options.talos_json_url
1.50 + try:
1.51 + key = 'talos.zip'
1.52 + entity = get_value(jsonFilename, key)
1.53 + if passesRestrictions(options.talos_json_url, entity["url"]):
1.54 + # the key is at the same time the filename e.g. talos.zip
1.55 + print "INFO: Downloading %s as %s" % (entity["url"], os.path.join(entity["path"], key))
1.56 + download_file(entity["url"], entity["path"], key)
1.57 + else:
1.58 + print "ERROR: You have tried to download a file " + \
1.59 + "from: %s " % fileUrl + \
1.60 + "which is a location different than http://talos-bundles.pvt.build.mozilla.org/"
1.61 + print "ERROR: This is only allowed for the certain branches."
1.62 + sys.exit(1)
1.63 + except Exception, e:
1.64 + print "ERROR: %s" % str(e)
1.65 + sys.exit(1)
1.66 +
1.67 +def passesRestrictions(talosJsonUrl, fileUrl):
1.68 + '''
1.69 + Only certain branches are exempted from having to host their downloadable files
1.70 + in talos-bundles.pvt.build.mozilla.org
1.71 + '''
1.72 + if talosJsonUrl.startswith("http://hg.mozilla.org/try/") == True or \
1.73 + talosJsonUrl.startswith("https://hg.mozilla.org/try/") == True or \
1.74 + talosJsonUrl.startswith("http://hg.mozilla.org/projects/pine/") == True or \
1.75 + talosJsonUrl.startswith("https://hg.mozilla.org/projects/pine/") == True:
1.76 + return True
1.77 + else:
1.78 + p = re.compile('^http://talos-bundles.pvt.build.mozilla.org/')
1.79 + m = p.match(fileUrl)
1.80 + if m == None:
1.81 + return False
1.82 + return True
1.83 +
1.84 +def get_filename_from_url(url):
1.85 + '''
1.86 + This returns the filename of the file we're trying to download
1.87 + '''
1.88 + parsed = urlparse.urlsplit(url.rstrip('/'))
1.89 + if parsed.path != '':
1.90 + return parsed.path.rsplit('/', 1)[-1]
1.91 + else:
1.92 + print "ERROR: We were trying to download a file from %s " + \
1.93 + "but the URL seems to be incorrect."
1.94 + sys.exit(1)
1.95 +
1.96 +def download_file(url, path="", saveAs=None):
1.97 + '''
1.98 + It downloads a file from URL to the indicated path
1.99 + '''
1.100 + req = urllib2.Request(url)
1.101 + f = urllib2.urlopen(req)
1.102 + if path != "" and not os.path.isdir(path):
1.103 + try:
1.104 + os.makedirs(path)
1.105 + print "INFO: directory %s created" % path
1.106 + except Exception, e:
1.107 + print "ERROR: %s" % str(e)
1.108 + sys.exit(1)
1.109 + filename = saveAs if saveAs else get_filename_from_url(url)
1.110 + local_file = open(os.path.join(path, filename), 'wb')
1.111 + local_file.write(f.read())
1.112 + local_file.close()
1.113 + return filename
1.114 +
1.115 +def get_value(json_filename, key):
1.116 + '''
1.117 + It loads up a JSON file and returns the value for the given string
1.118 + '''
1.119 + f = open(json_filename, 'r')
1.120 + return json.load(f)[key]
1.121 +
1.122 +if __name__ == '__main__':
1.123 + main()
