1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/testing/marionette/update-smoketests/smoketest.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,195 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +import json
1.9 +import os
1.10 +import subprocess
1.11 +import sys
1.12 +import tempfile
1.13 +import threading
1.14 +import zipfile
1.15 +
1.16 +from ConfigParser import ConfigParser
1.17 +
1.18 +this_dir = os.path.abspath(os.path.dirname(__file__))
1.19 +marionette_dir = os.path.dirname(this_dir)
1.20 +marionette_client_dir = os.path.join(marionette_dir, 'client', 'marionette')
1.21 +
1.22 +def find_b2g():
1.23 + sys.path.append(marionette_client_dir)
1.24 + from b2gbuild import B2GBuild
1.25 + return B2GBuild()
1.26 +
1.27 +class DictObject(dict):
1.28 + def __getattr__(self, item):
1.29 + try:
1.30 + return self.__getitem__(item)
1.31 + except KeyError:
1.32 + raise AttributeError(item)
1.33 +
1.34 + def __getitem__(self, item):
1.35 + value = dict.__getitem__(self, item)
1.36 + if isinstance(value, dict):
1.37 + return DictObject(value)
1.38 + return value
1.39 +
1.40 +class SmokeTestError(Exception):
1.41 + pass
1.42 +
1.43 +class SmokeTestConfigError(SmokeTestError):
1.44 + def __init__(self, message):
1.45 + SmokeTestError.__init__(self, 'smoketest-config.json: ' + message)
1.46 +
1.47 +class SmokeTestConfig(DictObject):
1.48 + TOP_LEVEL_REQUIRED = ('devices', 'public_key', 'private_key')
1.49 + DEVICE_REQUIRED = ('system_fs_type', 'system_location', 'data_fs_type',
1.50 + 'data_location', 'sdcard', 'sdcard_recovery',
1.51 + 'serials')
1.52 +
1.53 + def __init__(self, build_dir):
1.54 + self.top_dir = build_dir
1.55 + self.build_data = {}
1.56 + self.flash_template = None
1.57 +
1.58 + with open(os.path.join(build_dir, 'smoketest-config.json')) as f:
1.59 + DictObject.__init__(self, json.loads(f.read()))
1.60 +
1.61 + for required in self.TOP_LEVEL_REQUIRED:
1.62 + if required not in self:
1.63 + raise SmokeTestConfigError('No "%s" found' % required)
1.64 +
1.65 + if len(self.devices) == 0:
1.66 + raise SmokeTestConfigError('No devices found')
1.67 +
1.68 + for name, device in self.devices.iteritems():
1.69 + for required in self.DEVICE_REQUIRED:
1.70 + if required not in device:
1.71 + raise SmokeTestConfigError('No "%s" found in device "%s"' % (required, name))
1.72 +
1.73 + def get_build_data(self, device, build_id):
1.74 + if device in self.build_data:
1.75 + if build_id in self.build_data[device]:
1.76 + return self.build_data[device][build_id]
1.77 + else:
1.78 + self.build_data[device] = {}
1.79 +
1.80 + build_dir = os.path.join(self.top_dir, device, build_id)
1.81 + flash_zip = os.path.join(build_dir, 'flash.zip')
1.82 + with zipfile.ZipFile(flash_zip) as zip:
1.83 + app_ini = ConfigParser()
1.84 + app_ini.readfp(zip.open('system/b2g/application.ini'))
1.85 + platform_ini = ConfigParser()
1.86 + platform_ini.readfp(zip.open('system/b2g/platform.ini'))
1.87 +
1.88 + build_data = self.build_data[device][build_id] = DictObject({
1.89 + 'app_version': app_ini.get('App', 'version'),
1.90 + 'app_build_id': app_ini.get('App', 'buildid'),
1.91 + 'platform_build_id': platform_ini.get('Build', 'buildid'),
1.92 + 'platform_milestone': platform_ini.get('Build', 'milestone'),
1.93 + 'complete_mar': os.path.join(build_dir, 'complete.mar'),
1.94 + 'flash_script': os.path.join(build_dir, 'flash.sh')
1.95 + })
1.96 +
1.97 + return build_data
1.98 +
1.99 +class SmokeTestRunner(object):
1.100 + DEVICE_TIMEOUT = 30
1.101 +
1.102 + def __init__(self, config, b2g, run_dir=None):
1.103 + self.config = config
1.104 + self.b2g = b2g
1.105 + self.run_dir = run_dir or tempfile.mkdtemp()
1.106 +
1.107 + update_tools = self.b2g.import_update_tools()
1.108 + self.b2g_config = update_tools.B2GConfig()
1.109 +
1.110 + def run_b2g_update_test(self, serial, testvars, tests):
1.111 + b2g_update_test = os.path.join(marionette_client_dir,
1.112 + 'venv_b2g_update_test.sh')
1.113 +
1.114 + if not tests:
1.115 + tests = [os.path.join(marionette_client_dir, 'tests',
1.116 + 'update-tests.ini')]
1.117 +
1.118 + args = ['bash', b2g_update_test, sys.executable,
1.119 + '--homedir', self.b2g.homedir,
1.120 + '--address', 'localhost:2828',
1.121 + '--type', 'b2g+smoketest',
1.122 + '--device', serial,
1.123 + '--testvars', testvars]
1.124 + args.extend(tests)
1.125 +
1.126 + print ' '.join(args)
1.127 + subprocess.check_call(args)
1.128 +
1.129 + def build_testvars(self, device, start_id, finish_id):
1.130 + run_dir = os.path.join(self.run_dir, device, start_id, finish_id)
1.131 + if not os.path.exists(run_dir):
1.132 + os.makedirs(run_dir)
1.133 +
1.134 + start_data = self.config.get_build_data(device, start_id)
1.135 + finish_data = self.config.get_build_data(device, finish_id)
1.136 +
1.137 + partial_mar = os.path.join(run_dir, 'partial.mar')
1.138 + if not os.path.exists(partial_mar):
1.139 + build_gecko_mar = os.path.join(self.b2g.update_tools,
1.140 + 'build-gecko-mar.py')
1.141 + subprocess.check_call([sys.executable, build_gecko_mar,
1.142 + '--from', start_data.complete_mar,
1.143 + '--to', finish_data.complete_mar,
1.144 + partial_mar])
1.145 + finish_data['partial_mar'] = partial_mar
1.146 +
1.147 + testvars = os.path.join(run_dir, 'testvars.json')
1.148 + if not os.path.exists(testvars):
1.149 + open(testvars, 'w').write(json.dumps({
1.150 + 'start': start_data,
1.151 + 'finish': finish_data
1.152 + }))
1.153 +
1.154 + return testvars
1.155 +
1.156 + def wait_for_device(self, device):
1.157 + for serial in self.config.devices[device].serials:
1.158 + proc = subprocess.Popen([self.b2g.adb_path, '-s', serial,
1.159 + 'wait-for-device'])
1.160 + def wait_for_adb():
1.161 + proc.communicate()
1.162 +
1.163 + thread = threading.Thread(target=wait_for_adb)
1.164 + thread.start()
1.165 + thread.join(self.DEVICE_TIMEOUT)
1.166 +
1.167 + if thread.isAlive():
1.168 + print >>sys.stderr, '%s device %s is not recognized by ADB, ' \
1.169 + 'trying next device' % (device, serial)
1.170 + proc.kill()
1.171 + thread.join()
1.172 + continue
1.173 +
1.174 + return serial
1.175 + return None
1.176 +
1.177 + def run_smoketests_for_device(self, device, start_id, finish_id, tests):
1.178 + testvars = self.build_testvars(device, start_id, finish_id)
1.179 + serial = self.wait_for_device(device)
1.180 + if not serial:
1.181 + raise SmokeTestError('No connected serials for device "%s" could ' \
1.182 + 'be found' % device)
1.183 +
1.184 + try:
1.185 + self.run_b2g_update_test(serial, testvars, tests)
1.186 + except subprocess.CalledProcessError:
1.187 + print >>sys.stderr, 'SMOKETEST-FAIL | START=%s | FINISH=%s | ' \
1.188 + 'DEVICE=%s/%s | %s' % (start_id, finish_id,
1.189 + device, serial, testvars)
1.190 +
1.191 + def run_smoketests(self, build_ids, tests):
1.192 + build_ids.sort()
1.193 +
1.194 + latest_build_id = build_ids.pop(-1)
1.195 + for build_id in build_ids:
1.196 + for device in self.config.devices:
1.197 + self.run_smoketests_for_device(device, build_id,
1.198 + latest_build_id, tests)
