michael@0: #!/usr/bin/env python
michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this file,
michael@0: # You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: """
michael@0: Imports a test suite from a remote repository. Takes one argument, a file in
michael@0: the format described in README.
michael@0: Note: removes both source and destination directory before starting. Do not
michael@0:       use with outstanding changes in either directory.
michael@0: """
michael@0: 
michael@0: from __future__ import print_function, unicode_literals
michael@0: 
michael@0: import os
michael@0: import shutil
michael@0: import subprocess
michael@0: import sys
michael@0: 
michael@0: import parseManifest
michael@0: import writeBuildFiles
michael@0: 
michael@0: def readManifests(iden, dirs):
michael@0:     def parseManifestFile(iden, path):
michael@0:         pathstr = "hg-%s/%s/MANIFEST" % (iden, path)
michael@0:         subdirs, mochitests, reftests, _, supportfiles = parseManifest.parseManifestFile(pathstr)
michael@0:         return subdirs, mochitests, reftests, supportfiles
michael@0: 
michael@0:     data = []
michael@0:     for path in dirs:
michael@0:         subdirs, mochitests, reftests, supportfiles = parseManifestFile(iden, path)
michael@0:         data.append({
michael@0:           "path": path,
michael@0:           "mochitests": mochitests,
michael@0:           "reftests": reftests,
michael@0:           "supportfiles": supportfiles,
michael@0:         })
michael@0:         data.extend(readManifests(iden, ["%s/%s" % (path, d) for d in subdirs]))
michael@0:     return data
michael@0: 
michael@0: 
michael@0: def getData(confFile):
michael@0:     """This function parses a file of the form
michael@0:     (hg or git)|URL of remote repository|identifier for the local directory
michael@0:     First directory of tests
michael@0:     ...
michael@0:     Last directory of tests"""
michael@0:     vcs = ""
michael@0:     url = ""
michael@0:     iden = ""
michael@0:     directories = []
michael@0:     try:
michael@0:         with open(confFile, 'r') as fp:
michael@0:             first = True
michael@0:             for line in fp:
michael@0:                 if first:
michael@0:                     vcs, url, iden = line.strip().split("|")
michael@0:                     first = False
michael@0:                 else:
michael@0:                     directories.append(line.strip())
michael@0:     finally:
michael@0:         return vcs, url, iden, directories
michael@0: 
michael@0: 
michael@0: def makePathInternal(a, b):
michael@0:     if not b:
michael@0:         # Empty directory, i.e., the repository root.
michael@0:         return a
michael@0:     return "%s/%s" % (a, b)
michael@0: 
michael@0: 
michael@0: def makeSourcePath(a, b):
michael@0:     """Make a path in the source (upstream) directory."""
michael@0:     return makePathInternal("hg-%s" % a, b)
michael@0: 
michael@0: 
michael@0: def makeDestPath(a, b):
michael@0:     """Make a path in the destination (mozilla-central) directory, shortening as
michael@0:     appropriate."""
michael@0:     def shorten(path):
michael@0:         path = path.replace('dom-tree-accessors', 'dta')
michael@0:         path = path.replace('document.getElementsByName', 'doc.gEBN')
michael@0:         path = path.replace('requirements-for-implementations', 'implreq')
michael@0:         path = path.replace('other-elements-attributes-and-apis', 'oeaaa')
michael@0:         return path
michael@0: 
michael@0:     return shorten(makePathInternal(a, b))
michael@0: 
michael@0: 
michael@0: def extractReftestFiles(reftests):
michael@0:     """Returns the set of files referenced in the reftests argument"""
michael@0:     files = set()
michael@0:     for line in reftests:
michael@0:         files.update([line[1], line[2]])
michael@0:     return files
michael@0: 
michael@0: 
michael@0: def copy(dest, directories):
michael@0:     """Copy mochitests and support files from the external HG directory to their
michael@0:     place in mozilla-central.
michael@0:     """
michael@0:     print("Copying tests...")
michael@0:     for d in directories:
michael@0:         sourcedir = makeSourcePath(dest, d["path"])
michael@0:         destdir = makeDestPath(dest, d["path"])
michael@0:         os.makedirs(destdir)
michael@0: 
michael@0:         reftestfiles = extractReftestFiles(d["reftests"])
michael@0: 
michael@0:         for mochitest in d["mochitests"]:
michael@0:             shutil.copy("%s/%s" % (sourcedir, mochitest), "%s/test_%s" % (destdir, mochitest))
michael@0:         for reftest in sorted(reftestfiles):
michael@0:             shutil.copy("%s/%s" % (sourcedir, reftest), "%s/%s" % (destdir, reftest))
michael@0:         for support in d["supportfiles"]:
michael@0:             shutil.copy("%s/%s" % (sourcedir, support), "%s/%s" % (destdir, support))
michael@0: 
michael@0: def printBuildFiles(dest, directories):
michael@0:     """Create a mochitest.ini that all the contains tests we import.
michael@0:     """
michael@0:     print("Creating manifest...")
michael@0:     all_mochitests = set()
michael@0:     all_support = set()
michael@0: 
michael@0:     for d in directories:
michael@0:         path = makeDestPath(dest, d["path"])
michael@0: 
michael@0:         all_mochitests |= set('%s/test_%s' % (d['path'], mochitest)
michael@0:             for mochitest in d['mochitests'])
michael@0:         all_support |= set('%s/%s' % (d['path'], p) for p in d['supportfiles'])
michael@0: 
michael@0:         if d["reftests"]:
michael@0:             with open(path + "/reftest.list", "w") as fh:
michael@0:                 result = writeBuildFiles.substReftestList("importTestsuite.py",
michael@0:                     d["reftests"])
michael@0:                 fh.write(result)
michael@0: 
michael@0:     manifest_path = dest + '/mochitest.ini'
michael@0:     with open(manifest_path, 'w') as fh:
michael@0:         result = writeBuildFiles.substManifest('importTestsuite.py',
michael@0:             all_mochitests, all_support)
michael@0:         fh.write(result)
michael@0:     subprocess.check_call(["hg", "add", manifest_path])
michael@0: 
michael@0: def hgadd(dest, directories):
michael@0:     """Inform hg of the files in |directories|."""
michael@0:     print("hg addremoving...")
michael@0:     for d in directories:
michael@0:         subprocess.check_call(["hg", "addremove", makeDestPath(dest, d)])
michael@0: 
michael@0: def removeAndCloneRepo(vcs, url, dest):
michael@0:     """Replaces the repo at dest by a fresh clone from url using vcs"""
michael@0:     assert vcs in ('hg', 'git')
michael@0: 
michael@0:     print("Removing %s..." % dest)
michael@0:     subprocess.check_call(["rm", "-rf", dest])
michael@0: 
michael@0:     print("Cloning %s to %s with %s..." % (url, dest, vcs))
michael@0:     subprocess.check_call([vcs, "clone", url, dest])
michael@0: 
michael@0: def importRepo(confFile):
michael@0:     try:
michael@0:         vcs, url, iden, directories = getData(confFile)
michael@0:         dest = iden
michael@0:         hgdest = "hg-%s" % iden
michael@0: 
michael@0:         print("Removing %s..." % dest)
michael@0:         subprocess.check_call(["rm", "-rf", dest])
michael@0: 
michael@0:         removeAndCloneRepo(vcs, url, hgdest)
michael@0: 
michael@0:         data = readManifests(iden, directories)
michael@0:         print("Going to import %s..." % [d["path"] for d in data])
michael@0: 
michael@0:         copy(dest, data)
michael@0:         printBuildFiles(dest, data)
michael@0:         hgadd(dest, directories)
michael@0:         print("Removing %s again..." % hgdest)
michael@0:         subprocess.check_call(["rm", "-rf", hgdest])
michael@0:     except subprocess.CalledProcessError as e:
michael@0:         print(e.returncode)
michael@0:     finally:
michael@0:         print("Done")
michael@0: 
michael@0: if __name__ == "__main__":
michael@0:     if len(sys.argv) != 2:
michael@0:         print("Need one argument.")
michael@0:     else:
michael@0:         importRepo(sys.argv[1])
michael@0: