1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/build/pymake/tests/runtests.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,215 @@
1.4 +#!/usr/bin/env python
1.5 +"""
1.6 +Run the test(s) listed on the command line. If a directory is listed, the script will recursively
1.7 +walk the directory for files named .mk and run each.
1.8 +
1.9 +For each test, we run gmake -f test.mk. By default, make must exit with an exit code of 0, and must print 'TEST-PASS'.
1.10 +
1.11 +Each test is run in an empty directory.
1.12 +
1.13 +The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python:
1.14 +
1.15 +#T commandline: ['extra', 'params', 'here']
1.16 +#T returncode: 2
1.17 +#T returncode-on: {'win32': 2}
1.18 +#T environment: {'VAR': 'VALUE}
1.19 +#T grep-for: "text"
1.20 +"""
1.21 +
1.22 +from subprocess import Popen, PIPE, STDOUT
1.23 +from optparse import OptionParser
1.24 +import os, re, sys, shutil, glob
1.25 +
1.26 +class ParentDict(dict):
1.27 + def __init__(self, parent, **kwargs):
1.28 + self.d = dict(kwargs)
1.29 + self.parent = parent
1.30 +
1.31 + def __setitem__(self, k, v):
1.32 + self.d[k] = v
1.33 +
1.34 + def __getitem__(self, k):
1.35 + if k in self.d:
1.36 + return self.d[k]
1.37 +
1.38 + return self.parent[k]
1.39 +
1.40 +thisdir = os.path.dirname(os.path.abspath(__file__))
1.41 +
1.42 +pymake = [sys.executable, os.path.join(os.path.dirname(thisdir), 'make.py')]
1.43 +manifest = os.path.join(thisdir, 'tests.manifest')
1.44 +
1.45 +o = OptionParser()
1.46 +o.add_option('-g', '--gmake',
1.47 + dest="gmake", default="gmake")
1.48 +o.add_option('-d', '--tempdir',
1.49 + dest="tempdir", default="_mktests")
1.50 +opts, args = o.parse_args()
1.51 +
1.52 +if len(args) == 0:
1.53 + args = [thisdir]
1.54 +
1.55 +makefiles = []
1.56 +for a in args:
1.57 + if os.path.isfile(a):
1.58 + makefiles.append(a)
1.59 + elif os.path.isdir(a):
1.60 + makefiles.extend(sorted(glob.glob(os.path.join(a, '*.mk'))))
1.61 +
1.62 +def runTest(makefile, make, logfile, options):
1.63 + """
1.64 + Given a makefile path, test it with a given `make` and return
1.65 + (pass, message).
1.66 + """
1.67 +
1.68 + if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir)
1.69 + os.mkdir(opts.tempdir, 0755)
1.70 +
1.71 + logfd = open(logfile, 'w')
1.72 + p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env'])
1.73 + logfd.close()
1.74 + retcode = p.wait()
1.75 +
1.76 + if retcode != options['returncode']:
1.77 + return False, "FAIL (returncode=%i)" % retcode
1.78 +
1.79 + logfd = open(logfile)
1.80 + stdout = logfd.read()
1.81 + logfd.close()
1.82 +
1.83 + if stdout.find('TEST-FAIL') != -1:
1.84 + print stdout
1.85 + return False, "FAIL (TEST-FAIL printed)"
1.86 +
1.87 + if options['grepfor'] and stdout.find(options['grepfor']) == -1:
1.88 + print stdout
1.89 + return False, "FAIL (%s not in output)" % options['grepfor']
1.90 +
1.91 + if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1:
1.92 + print stdout
1.93 + return False, 'FAIL (No TEST-PASS printed)'
1.94 +
1.95 + if options['returncode'] != 0:
1.96 + return True, 'PASS (retcode=%s)' % retcode
1.97 +
1.98 + return True, 'PASS'
1.99 +
1.100 +print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:")
1.101 +
1.102 +gmakefails = 0
1.103 +pymakefails = 0
1.104 +
1.105 +tre = re.compile('^#T (gmake |pymake )?([a-z-]+)(?:: (.*))?$')
1.106 +
1.107 +for makefile in makefiles:
1.108 + # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows
1.109 + # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH
1.110 + cline = ['-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir]
1.111 + if sys.platform == 'win32':
1.112 + #XXX: hack so we can specialize the separator character on windows.
1.113 + # we really shouldn't need this, but y'know
1.114 + cline += ['__WIN32__=1']
1.115 +
1.116 + options = {
1.117 + 'returncode': 0,
1.118 + 'grepfor': None,
1.119 + 'env': dict(os.environ),
1.120 + 'commandline': cline,
1.121 + 'pass': True,
1.122 + 'skip': False,
1.123 + }
1.124 +
1.125 + gmakeoptions = ParentDict(options)
1.126 + pymakeoptions = ParentDict(options)
1.127 +
1.128 + dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions}
1.129 +
1.130 + mdata = open(makefile)
1.131 + for line in mdata:
1.132 + line = line.strip()
1.133 + m = tre.search(line)
1.134 + if m is None:
1.135 + break
1.136 +
1.137 + make, key, data = m.group(1, 2, 3)
1.138 + d = dmap[make]
1.139 + if data is not None:
1.140 + data = eval(data)
1.141 + if key == 'commandline':
1.142 + assert make is None
1.143 + d['commandline'].extend(data)
1.144 + elif key == 'returncode':
1.145 + d['returncode'] = data
1.146 + elif key == 'returncode-on':
1.147 + if sys.platform in data:
1.148 + d['returncode'] = data[sys.platform]
1.149 + elif key == 'environment':
1.150 + for k, v in data.iteritems():
1.151 + d['env'][k] = v
1.152 + elif key == 'grep-for':
1.153 + d['grepfor'] = data
1.154 + elif key == 'fail':
1.155 + d['pass'] = False
1.156 + elif key == 'skip':
1.157 + d['skip'] = True
1.158 + else:
1.159 + print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key)
1.160 + sys.exit(1)
1.161 +
1.162 + mdata.close()
1.163 +
1.164 + if gmakeoptions['skip']:
1.165 + gmakepass, gmakemsg = True, ''
1.166 + else:
1.167 + gmakepass, gmakemsg = runTest(makefile, [opts.gmake],
1.168 + makefile + '.gmakelog', gmakeoptions)
1.169 +
1.170 + if gmakeoptions['pass']:
1.171 + if not gmakepass:
1.172 + gmakefails += 1
1.173 + else:
1.174 + if gmakepass:
1.175 + gmakefails += 1
1.176 + gmakemsg = "UNEXPECTED PASS"
1.177 + else:
1.178 + gmakemsg = "KNOWN FAIL"
1.179 +
1.180 + if pymakeoptions['skip']:
1.181 + pymakepass, pymakemsg = True, ''
1.182 + else:
1.183 + pymakepass, pymakemsg = runTest(makefile, pymake,
1.184 + makefile + '.pymakelog', pymakeoptions)
1.185 +
1.186 + if pymakeoptions['pass']:
1.187 + if not pymakepass:
1.188 + pymakefails += 1
1.189 + else:
1.190 + if pymakepass:
1.191 + pymakefails += 1
1.192 + pymakemsg = "UNEXPECTED PASS"
1.193 + else:
1.194 + pymakemsg = "OK (known fail)"
1.195 +
1.196 + print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile),
1.197 + gmakemsg, pymakemsg)
1.198 +
1.199 +print
1.200 +print "Summary:"
1.201 +print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:")
1.202 +
1.203 +if gmakefails == 0:
1.204 + gmakemsg = 'PASS'
1.205 +else:
1.206 + gmakemsg = 'FAIL (%i failures)' % gmakefails
1.207 +
1.208 +if pymakefails == 0:
1.209 + pymakemsg = 'PASS'
1.210 +else:
1.211 + pymakemsg = 'FAIL (%i failures)' % pymakefails
1.212 +
1.213 +print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg)
1.214 +
1.215 +shutil.rmtree(opts.tempdir)
1.216 +
1.217 +if gmakefails or pymakefails:
1.218 + sys.exit(1)
