michael@0: #!/usr/bin/env python
michael@0: """
michael@0: Run the test(s) listed on the command line. If a directory is listed, the script will recursively
michael@0: walk the directory for files named .mk and run each.
michael@0: 
michael@0: For each test, we run gmake -f test.mk. By default, make must exit with an exit code of 0, and must print 'TEST-PASS'.
michael@0: 
michael@0: Each test is run in an empty directory.
michael@0: 
michael@0: The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python:
michael@0: 
michael@0: #T commandline: ['extra', 'params', 'here']
michael@0: #T returncode: 2
michael@0: #T returncode-on: {'win32': 2}
michael@0: #T environment: {'VAR': 'VALUE}
michael@0: #T grep-for: "text"
michael@0: """
michael@0: 
michael@0: from subprocess import Popen, PIPE, STDOUT
michael@0: from optparse import OptionParser
michael@0: import os, re, sys, shutil, glob
michael@0: 
michael@0: class ParentDict(dict):
michael@0:     def __init__(self, parent, **kwargs):
michael@0:         self.d = dict(kwargs)
michael@0:         self.parent = parent
michael@0: 
michael@0:     def __setitem__(self, k, v):
michael@0:         self.d[k] = v
michael@0: 
michael@0:     def __getitem__(self, k):
michael@0:         if k in self.d:
michael@0:             return self.d[k]
michael@0: 
michael@0:         return self.parent[k]
michael@0: 
michael@0: thisdir = os.path.dirname(os.path.abspath(__file__))
michael@0: 
michael@0: pymake = [sys.executable, os.path.join(os.path.dirname(thisdir), 'make.py')]
michael@0: manifest = os.path.join(thisdir, 'tests.manifest')
michael@0: 
michael@0: o = OptionParser()
michael@0: o.add_option('-g', '--gmake',
michael@0:              dest="gmake", default="gmake")
michael@0: o.add_option('-d', '--tempdir',
michael@0:              dest="tempdir", default="_mktests")
michael@0: opts, args = o.parse_args()
michael@0: 
michael@0: if len(args) == 0:
michael@0:     args = [thisdir]
michael@0: 
michael@0: makefiles = []
michael@0: for a in args:
michael@0:     if os.path.isfile(a):
michael@0:         makefiles.append(a)
michael@0:     elif os.path.isdir(a):
michael@0:         makefiles.extend(sorted(glob.glob(os.path.join(a, '*.mk'))))
michael@0: 
michael@0: def runTest(makefile, make, logfile, options):
michael@0:     """
michael@0:     Given a makefile path, test it with a given `make` and return
michael@0:     (pass, message).
michael@0:     """
michael@0: 
michael@0:     if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir)
michael@0:     os.mkdir(opts.tempdir, 0755)
michael@0: 
michael@0:     logfd = open(logfile, 'w')
michael@0:     p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env'])
michael@0:     logfd.close()
michael@0:     retcode = p.wait()
michael@0: 
michael@0:     if retcode != options['returncode']:
michael@0:         return False, "FAIL (returncode=%i)" % retcode
michael@0:         
michael@0:     logfd = open(logfile)
michael@0:     stdout = logfd.read()
michael@0:     logfd.close()
michael@0: 
michael@0:     if stdout.find('TEST-FAIL') != -1:
michael@0:         print stdout
michael@0:         return False, "FAIL (TEST-FAIL printed)"
michael@0: 
michael@0:     if options['grepfor'] and stdout.find(options['grepfor']) == -1:
michael@0:         print stdout
michael@0:         return False, "FAIL (%s not in output)" % options['grepfor']
michael@0: 
michael@0:     if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1:
michael@0:         print stdout
michael@0:         return False, 'FAIL (No TEST-PASS printed)'
michael@0: 
michael@0:     if options['returncode'] != 0:
michael@0:         return True, 'PASS (retcode=%s)' % retcode
michael@0: 
michael@0:     return True, 'PASS'
michael@0: 
michael@0: print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:")
michael@0: 
michael@0: gmakefails = 0
michael@0: pymakefails = 0
michael@0: 
michael@0: tre = re.compile('^#T (gmake |pymake )?([a-z-]+)(?:: (.*))?$')
michael@0: 
michael@0: for makefile in makefiles:
michael@0:     # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows
michael@0:     # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH
michael@0:     cline = ['-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir]
michael@0:     if sys.platform == 'win32':
michael@0:         #XXX: hack so we can specialize the separator character on windows.
michael@0:         # we really shouldn't need this, but y'know
michael@0:         cline += ['__WIN32__=1']
michael@0: 
michael@0:     options = {
michael@0:         'returncode': 0,
michael@0:         'grepfor': None,
michael@0:         'env': dict(os.environ),
michael@0:         'commandline': cline,
michael@0:         'pass': True,
michael@0:         'skip': False,
michael@0:         }
michael@0: 
michael@0:     gmakeoptions = ParentDict(options)
michael@0:     pymakeoptions = ParentDict(options)
michael@0: 
michael@0:     dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions}
michael@0: 
michael@0:     mdata = open(makefile)
michael@0:     for line in mdata:
michael@0:         line = line.strip()
michael@0:         m = tre.search(line)
michael@0:         if m is None:
michael@0:             break
michael@0: 
michael@0:         make, key, data = m.group(1, 2, 3)
michael@0:         d = dmap[make]
michael@0:         if data is not None:
michael@0:             data = eval(data)
michael@0:         if key == 'commandline':
michael@0:             assert make is None
michael@0:             d['commandline'].extend(data)
michael@0:         elif key == 'returncode':
michael@0:             d['returncode'] = data
michael@0:         elif key == 'returncode-on':
michael@0:             if sys.platform in data:
michael@0:                 d['returncode'] = data[sys.platform]
michael@0:         elif key == 'environment':
michael@0:             for k, v in data.iteritems():
michael@0:                 d['env'][k] = v
michael@0:         elif key == 'grep-for':
michael@0:             d['grepfor'] = data
michael@0:         elif key == 'fail':
michael@0:             d['pass'] = False
michael@0:         elif key == 'skip':
michael@0:             d['skip'] = True
michael@0:         else:
michael@0:             print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key)
michael@0:             sys.exit(1)
michael@0: 
michael@0:     mdata.close()
michael@0: 
michael@0:     if gmakeoptions['skip']:
michael@0:         gmakepass, gmakemsg = True, ''
michael@0:     else:
michael@0:         gmakepass, gmakemsg = runTest(makefile, [opts.gmake],
michael@0:                                       makefile + '.gmakelog', gmakeoptions)
michael@0: 
michael@0:     if gmakeoptions['pass']:
michael@0:         if not gmakepass:
michael@0:             gmakefails += 1
michael@0:     else:
michael@0:         if gmakepass:
michael@0:             gmakefails += 1
michael@0:             gmakemsg = "UNEXPECTED PASS"
michael@0:         else:
michael@0:             gmakemsg = "KNOWN FAIL"
michael@0: 
michael@0:     if pymakeoptions['skip']:
michael@0:         pymakepass, pymakemsg = True, ''
michael@0:     else:
michael@0:         pymakepass, pymakemsg = runTest(makefile, pymake,
michael@0:                                         makefile + '.pymakelog', pymakeoptions)
michael@0: 
michael@0:     if pymakeoptions['pass']:
michael@0:         if not pymakepass:
michael@0:             pymakefails += 1
michael@0:     else:
michael@0:         if pymakepass:
michael@0:             pymakefails += 1
michael@0:             pymakemsg = "UNEXPECTED PASS"
michael@0:         else:
michael@0:             pymakemsg = "OK (known fail)"
michael@0: 
michael@0:     print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile),
michael@0:                                         gmakemsg, pymakemsg)
michael@0: 
michael@0: print
michael@0: print "Summary:"
michael@0: print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:")
michael@0: 
michael@0: if gmakefails == 0:
michael@0:     gmakemsg = 'PASS'
michael@0: else:
michael@0:     gmakemsg = 'FAIL (%i failures)' % gmakefails
michael@0: 
michael@0: if pymakefails == 0:
michael@0:     pymakemsg = 'PASS'
michael@0: else:
michael@0:     pymakemsg = 'FAIL (%i failures)' % pymakefails
michael@0: 
michael@0: print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg)
michael@0: 
michael@0: shutil.rmtree(opts.tempdir)
michael@0: 
michael@0: if gmakefails or pymakefails:
michael@0:     sys.exit(1)