The Tor Browser / file comparison
comparison: build/pymake/tests/runtests.py
build/pymake/tests/runtests.py
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
equal
deleted
inserted
replaced
| -1:000000000000 | 0:b8e1f09664e2 |
|---|---|
| 1 #!/usr/bin/env python | |
| 2 """ | |
| 3 Run the test(s) listed on the command line. If a directory is listed, the script will recursively | |
| 4 walk the directory for files named .mk and run each. | |
| 5 | |
| 6 For each test, we run gmake -f test.mk. By default, make must exit with an exit code of 0, and must print 'TEST-PASS'. | |
| 7 | |
| 8 Each test is run in an empty directory. | |
| 9 | |
| 10 The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python: | |
| 11 | |
| 12 #T commandline: ['extra', 'params', 'here'] | |
| 13 #T returncode: 2 | |
| 14 #T returncode-on: {'win32': 2} | |
| 15 #T environment: {'VAR': 'VALUE} | |
| 16 #T grep-for: "text" | |
| 17 """ | |
| 18 | |
| 19 from subprocess import Popen, PIPE, STDOUT | |
| 20 from optparse import OptionParser | |
| 21 import os, re, sys, shutil, glob | |
| 22 | |
| 23 class ParentDict(dict): | |
| 24 def __init__(self, parent, **kwargs): | |
| 25 self.d = dict(kwargs) | |
| 26 self.parent = parent | |
| 27 | |
| 28 def __setitem__(self, k, v): | |
| 29 self.d[k] = v | |
| 30 | |
| 31 def __getitem__(self, k): | |
| 32 if k in self.d: | |
| 33 return self.d[k] | |
| 34 | |
| 35 return self.parent[k] | |
| 36 | |
| 37 thisdir = os.path.dirname(os.path.abspath(__file__)) | |
| 38 | |
| 39 pymake = [sys.executable, os.path.join(os.path.dirname(thisdir), 'make.py')] | |
| 40 manifest = os.path.join(thisdir, 'tests.manifest') | |
| 41 | |
| 42 o = OptionParser() | |
| 43 o.add_option('-g', '--gmake', | |
| 44 dest="gmake", default="gmake") | |
| 45 o.add_option('-d', '--tempdir', | |
| 46 dest="tempdir", default="_mktests") | |
| 47 opts, args = o.parse_args() | |
| 48 | |
| 49 if len(args) == 0: | |
| 50 args = [thisdir] | |
| 51 | |
| 52 makefiles = [] | |
| 53 for a in args: | |
| 54 if os.path.isfile(a): | |
| 55 makefiles.append(a) | |
| 56 elif os.path.isdir(a): | |
| 57 makefiles.extend(sorted(glob.glob(os.path.join(a, '*.mk')))) | |
| 58 | |
| 59 def runTest(makefile, make, logfile, options): | |
| 60 """ | |
| 61 Given a makefile path, test it with a given `make` and return | |
| 62 (pass, message). | |
| 63 """ | |
| 64 | |
| 65 if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir) | |
| 66 os.mkdir(opts.tempdir, 0755) | |
| 67 | |
| 68 logfd = open(logfile, 'w') | |
| 69 p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env']) | |
| 70 logfd.close() | |
| 71 retcode = p.wait() | |
| 72 | |
| 73 if retcode != options['returncode']: | |
| 74 return False, "FAIL (returncode=%i)" % retcode | |
| 75 | |
| 76 logfd = open(logfile) | |
| 77 stdout = logfd.read() | |
| 78 logfd.close() | |
| 79 | |
| 80 if stdout.find('TEST-FAIL') != -1: | |
| 81 print stdout | |
| 82 return False, "FAIL (TEST-FAIL printed)" | |
| 83 | |
| 84 if options['grepfor'] and stdout.find(options['grepfor']) == -1: | |
| 85 print stdout | |
| 86 return False, "FAIL (%s not in output)" % options['grepfor'] | |
| 87 | |
| 88 if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1: | |
| 89 print stdout | |
| 90 return False, 'FAIL (No TEST-PASS printed)' | |
| 91 | |
| 92 if options['returncode'] != 0: | |
| 93 return True, 'PASS (retcode=%s)' % retcode | |
| 94 | |
| 95 return True, 'PASS' | |
| 96 | |
| 97 print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:") | |
| 98 | |
| 99 gmakefails = 0 | |
| 100 pymakefails = 0 | |
| 101 | |
| 102 tre = re.compile('^#T (gmake |pymake )?([a-z-]+)(?:: (.*))?$') | |
| 103 | |
| 104 for makefile in makefiles: | |
| 105 # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows | |
| 106 # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH | |
| 107 cline = ['-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir] | |
| 108 if sys.platform == 'win32': | |
| 109 #XXX: hack so we can specialize the separator character on windows. | |
| 110 # we really shouldn't need this, but y'know | |
| 111 cline += ['__WIN32__=1'] | |
| 112 | |
| 113 options = { | |
| 114 'returncode': 0, | |
| 115 'grepfor': None, | |
| 116 'env': dict(os.environ), | |
| 117 'commandline': cline, | |
| 118 'pass': True, | |
| 119 'skip': False, | |
| 120 } | |
| 121 | |
| 122 gmakeoptions = ParentDict(options) | |
| 123 pymakeoptions = ParentDict(options) | |
| 124 | |
| 125 dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions} | |
| 126 | |
| 127 mdata = open(makefile) | |
| 128 for line in mdata: | |
| 129 line = line.strip() | |
| 130 m = tre.search(line) | |
| 131 if m is None: | |
| 132 break | |
| 133 | |
| 134 make, key, data = m.group(1, 2, 3) | |
| 135 d = dmap[make] | |
| 136 if data is not None: | |
| 137 data = eval(data) | |
| 138 if key == 'commandline': | |
| 139 assert make is None | |
| 140 d['commandline'].extend(data) | |
| 141 elif key == 'returncode': | |
| 142 d['returncode'] = data | |
| 143 elif key == 'returncode-on': | |
| 144 if sys.platform in data: | |
| 145 d['returncode'] = data[sys.platform] | |
| 146 elif key == 'environment': | |
| 147 for k, v in data.iteritems(): | |
| 148 d['env'][k] = v | |
| 149 elif key == 'grep-for': | |
| 150 d['grepfor'] = data | |
| 151 elif key == 'fail': | |
| 152 d['pass'] = False | |
| 153 elif key == 'skip': | |
| 154 d['skip'] = True | |
| 155 else: | |
| 156 print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key) | |
| 157 sys.exit(1) | |
| 158 | |
| 159 mdata.close() | |
| 160 | |
| 161 if gmakeoptions['skip']: | |
| 162 gmakepass, gmakemsg = True, '' | |
| 163 else: | |
| 164 gmakepass, gmakemsg = runTest(makefile, [opts.gmake], | |
| 165 makefile + '.gmakelog', gmakeoptions) | |
| 166 | |
| 167 if gmakeoptions['pass']: | |
| 168 if not gmakepass: | |
| 169 gmakefails += 1 | |
| 170 else: | |
| 171 if gmakepass: | |
| 172 gmakefails += 1 | |
| 173 gmakemsg = "UNEXPECTED PASS" | |
| 174 else: | |
| 175 gmakemsg = "KNOWN FAIL" | |
| 176 | |
| 177 if pymakeoptions['skip']: | |
| 178 pymakepass, pymakemsg = True, '' | |
| 179 else: | |
| 180 pymakepass, pymakemsg = runTest(makefile, pymake, | |
| 181 makefile + '.pymakelog', pymakeoptions) | |
| 182 | |
| 183 if pymakeoptions['pass']: | |
| 184 if not pymakepass: | |
| 185 pymakefails += 1 | |
| 186 else: | |
| 187 if pymakepass: | |
| 188 pymakefails += 1 | |
| 189 pymakemsg = "UNEXPECTED PASS" | |
| 190 else: | |
| 191 pymakemsg = "OK (known fail)" | |
| 192 | |
| 193 print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile), | |
| 194 gmakemsg, pymakemsg) | |
| 195 | |
| 196 print | |
| 197 print "Summary:" | |
| 198 print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:") | |
| 199 | |
| 200 if gmakefails == 0: | |
| 201 gmakemsg = 'PASS' | |
| 202 else: | |
| 203 gmakemsg = 'FAIL (%i failures)' % gmakefails | |
| 204 | |
| 205 if pymakefails == 0: | |
| 206 pymakemsg = 'PASS' | |
| 207 else: | |
| 208 pymakemsg = 'FAIL (%i failures)' % pymakefails | |
| 209 | |
| 210 print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg) | |
| 211 | |
| 212 shutil.rmtree(opts.tempdir) | |
| 213 | |
| 214 if gmakefails or pymakefails: | |
| 215 sys.exit(1) |
