1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/python/which/build.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,442 @@
1.4 +#!/usr/bin/env python
1.5 +# Copyright (c) 2002-2005 ActiveState
1.6 +# See LICENSE.txt for license details.
1.7 +
1.8 +"""
1.9 + which.py dev build script
1.10 +
1.11 + Usage:
1.12 + python build.py [<options>...] [<targets>...]
1.13 +
1.14 + Options:
1.15 + --help, -h Print this help and exit.
1.16 + --targets, -t List all available targets.
1.17 +
1.18 + This is the primary build script for the which.py project. It exists
1.19 + to assist in building, maintaining, and distributing this project.
1.20 +
1.21 + It is intended to have Makefile semantics. I.e. 'python build.py'
1.22 + will build execute the default target, 'python build.py foo' will
1.23 + build target foo, etc. However, there is no intelligent target
1.24 + interdependency tracking (I suppose I could do that with function
1.25 + attributes).
1.26 +"""
1.27 +
1.28 +import os
1.29 +from os.path import basename, dirname, splitext, isfile, isdir, exists, \
1.30 + join, abspath, normpath
1.31 +import sys
1.32 +import getopt
1.33 +import types
1.34 +import getpass
1.35 +import shutil
1.36 +import glob
1.37 +import logging
1.38 +import re
1.39 +
1.40 +
1.41 +
1.42 +#---- exceptions
1.43 +
1.44 +class Error(Exception):
1.45 + pass
1.46 +
1.47 +
1.48 +
1.49 +#---- globals
1.50 +
1.51 +log = logging.getLogger("build")
1.52 +
1.53 +
1.54 +
1.55 +
1.56 +#---- globals
1.57 +
1.58 +_project_name_ = "which"
1.59 +
1.60 +
1.61 +
1.62 +#---- internal support routines
1.63 +
1.64 +def _get_trentm_com_dir():
1.65 + """Return the path to the local trentm.com source tree."""
1.66 + d = normpath(join(dirname(__file__), os.pardir, "trentm.com"))
1.67 + if not isdir(d):
1.68 + raise Error("could not find 'trentm.com' src dir at '%s'" % d)
1.69 + return d
1.70 +
1.71 +def _get_local_bits_dir():
1.72 + import imp
1.73 + info = imp.find_module("tmconfig", [_get_trentm_com_dir()])
1.74 + tmconfig = imp.load_module("tmconfig", *info)
1.75 + return tmconfig.bitsDir
1.76 +
1.77 +def _get_project_bits_dir():
1.78 + d = normpath(join(dirname(__file__), "bits"))
1.79 + return d
1.80 +
1.81 +def _get_project_version():
1.82 + import imp, os
1.83 + data = imp.find_module(_project_name_, [os.path.dirname(__file__)])
1.84 + mod = imp.load_module(_project_name_, *data)
1.85 + return mod.__version__
1.86 +
1.87 +
1.88 +# Recipe: run (0.5.1) in /Users/trentm/tm/recipes/cookbook
1.89 +_RUN_DEFAULT_LOGSTREAM = ("RUN", "DEFAULT", "LOGSTREAM")
1.90 +def __run_log(logstream, msg, *args, **kwargs):
1.91 + if not logstream:
1.92 + pass
1.93 + elif logstream is _RUN_DEFAULT_LOGSTREAM:
1.94 + try:
1.95 + log.debug(msg, *args, **kwargs)
1.96 + except NameError:
1.97 + pass
1.98 + else:
1.99 + logstream(msg, *args, **kwargs)
1.100 +
1.101 +def _run(cmd, logstream=_RUN_DEFAULT_LOGSTREAM):
1.102 + """Run the given command.
1.103 +
1.104 + "cmd" is the command to run
1.105 + "logstream" is an optional logging stream on which to log the command.
1.106 + If None, no logging is done. If unspecifed, this looks for a Logger
1.107 + instance named 'log' and logs the command on log.debug().
1.108 +
1.109 + Raises OSError is the command returns a non-zero exit status.
1.110 + """
1.111 + __run_log(logstream, "running '%s'", cmd)
1.112 + retval = os.system(cmd)
1.113 + if hasattr(os, "WEXITSTATUS"):
1.114 + status = os.WEXITSTATUS(retval)
1.115 + else:
1.116 + status = retval
1.117 + if status:
1.118 + #TODO: add std OSError attributes or pick more approp. exception
1.119 + raise OSError("error running '%s': %r" % (cmd, status))
1.120 +
1.121 +def _run_in_dir(cmd, cwd, logstream=_RUN_DEFAULT_LOGSTREAM):
1.122 + old_dir = os.getcwd()
1.123 + try:
1.124 + os.chdir(cwd)
1.125 + __run_log(logstream, "running '%s' in '%s'", cmd, cwd)
1.126 + _run(cmd, logstream=None)
1.127 + finally:
1.128 + os.chdir(old_dir)
1.129 +
1.130 +
1.131 +# Recipe: rmtree (0.5) in /Users/trentm/tm/recipes/cookbook
1.132 +def _rmtree_OnError(rmFunction, filePath, excInfo):
1.133 + if excInfo[0] == OSError:
1.134 + # presuming because file is read-only
1.135 + os.chmod(filePath, 0777)
1.136 + rmFunction(filePath)
1.137 +def _rmtree(dirname):
1.138 + import shutil
1.139 + shutil.rmtree(dirname, 0, _rmtree_OnError)
1.140 +
1.141 +
1.142 +# Recipe: pretty_logging (0.1) in /Users/trentm/tm/recipes/cookbook
1.143 +class _PerLevelFormatter(logging.Formatter):
1.144 + """Allow multiple format string -- depending on the log level.
1.145 +
1.146 + A "fmtFromLevel" optional arg is added to the constructor. It can be
1.147 + a dictionary mapping a log record level to a format string. The
1.148 + usual "fmt" argument acts as the default.
1.149 + """
1.150 + def __init__(self, fmt=None, datefmt=None, fmtFromLevel=None):
1.151 + logging.Formatter.__init__(self, fmt, datefmt)
1.152 + if fmtFromLevel is None:
1.153 + self.fmtFromLevel = {}
1.154 + else:
1.155 + self.fmtFromLevel = fmtFromLevel
1.156 + def format(self, record):
1.157 + record.levelname = record.levelname.lower()
1.158 + if record.levelno in self.fmtFromLevel:
1.159 + #XXX This is a non-threadsafe HACK. Really the base Formatter
1.160 + # class should provide a hook accessor for the _fmt
1.161 + # attribute. *Could* add a lock guard here (overkill?).
1.162 + _saved_fmt = self._fmt
1.163 + self._fmt = self.fmtFromLevel[record.levelno]
1.164 + try:
1.165 + return logging.Formatter.format(self, record)
1.166 + finally:
1.167 + self._fmt = _saved_fmt
1.168 + else:
1.169 + return logging.Formatter.format(self, record)
1.170 +
1.171 +def _setup_logging():
1.172 + hdlr = logging.StreamHandler()
1.173 + defaultFmt = "%(name)s: %(levelname)s: %(message)s"
1.174 + infoFmt = "%(name)s: %(message)s"
1.175 + fmtr = _PerLevelFormatter(fmt=defaultFmt,
1.176 + fmtFromLevel={logging.INFO: infoFmt})
1.177 + hdlr.setFormatter(fmtr)
1.178 + logging.root.addHandler(hdlr)
1.179 + log.setLevel(logging.INFO)
1.180 +
1.181 +
1.182 +def _getTargets():
1.183 + """Find all targets and return a dict of targetName:targetFunc items."""
1.184 + targets = {}
1.185 + for name, attr in sys.modules[__name__].__dict__.items():
1.186 + if name.startswith('target_'):
1.187 + targets[ name[len('target_'):] ] = attr
1.188 + return targets
1.189 +
1.190 +def _listTargets(targets):
1.191 + """Pretty print a list of targets."""
1.192 + width = 77
1.193 + nameWidth = 15 # min width
1.194 + for name in targets.keys():
1.195 + nameWidth = max(nameWidth, len(name))
1.196 + nameWidth += 2 # space btwn name and doc
1.197 + format = "%%-%ds%%s" % nameWidth
1.198 + print format % ("TARGET", "DESCRIPTION")
1.199 + for name, func in sorted(targets.items()):
1.200 + doc = _first_paragraph(func.__doc__ or "", True)
1.201 + if len(doc) > (width - nameWidth):
1.202 + doc = doc[:(width-nameWidth-3)] + "..."
1.203 + print format % (name, doc)
1.204 +
1.205 +
1.206 +# Recipe: first_paragraph (1.0.1) in /Users/trentm/tm/recipes/cookbook
1.207 +def _first_paragraph(text, join_lines=False):
1.208 + """Return the first paragraph of the given text."""
1.209 + para = text.lstrip().split('\n\n', 1)[0]
1.210 + if join_lines:
1.211 + lines = [line.strip() for line in para.splitlines(0)]
1.212 + para = ' '.join(lines)
1.213 + return para
1.214 +
1.215 +
1.216 +
1.217 +#---- build targets
1.218 +
1.219 +def target_default():
1.220 + target_all()
1.221 +
1.222 +def target_all():
1.223 + """Build all release packages."""
1.224 + log.info("target: default")
1.225 + if sys.platform == "win32":
1.226 + target_launcher()
1.227 + target_sdist()
1.228 + target_webdist()
1.229 +
1.230 +
1.231 +def target_clean():
1.232 + """remove all build/generated bits"""
1.233 + log.info("target: clean")
1.234 + if sys.platform == "win32":
1.235 + _run("nmake -f Makefile.win clean")
1.236 +
1.237 + ver = _get_project_version()
1.238 + dirs = ["dist", "build", "%s-%s" % (_project_name_, ver)]
1.239 + for d in dirs:
1.240 + print "removing '%s'" % d
1.241 + if os.path.isdir(d): _rmtree(d)
1.242 +
1.243 + patterns = ["*.pyc", "*~", "MANIFEST",
1.244 + os.path.join("test", "*~"),
1.245 + os.path.join("test", "*.pyc"),
1.246 + ]
1.247 + for pattern in patterns:
1.248 + for file in glob.glob(pattern):
1.249 + print "removing '%s'" % file
1.250 + os.unlink(file)
1.251 +
1.252 +
1.253 +def target_launcher():
1.254 + """Build the Windows launcher executable."""
1.255 + log.info("target: launcher")
1.256 + assert sys.platform == "win32", "'launcher' target only supported on Windows"
1.257 + _run("nmake -f Makefile.win")
1.258 +
1.259 +
1.260 +def target_docs():
1.261 + """Regenerate some doc bits from project-info.xml."""
1.262 + log.info("target: docs")
1.263 + _run("projinfo -f project-info.xml -R -o README.txt --force")
1.264 + _run("projinfo -f project-info.xml --index-markdown -o index.markdown --force")
1.265 +
1.266 +
1.267 +def target_sdist():
1.268 + """Build a source distribution."""
1.269 + log.info("target: sdist")
1.270 + target_docs()
1.271 + bitsDir = _get_project_bits_dir()
1.272 + _run("python setup.py sdist -f --formats zip -d %s" % bitsDir,
1.273 + log.info)
1.274 +
1.275 +
1.276 +def target_webdist():
1.277 + """Build a web dist package.
1.278 +
1.279 + "Web dist" packages are zip files with '.web' package. All files in
1.280 + the zip must be under a dir named after the project. There must be a
1.281 + webinfo.xml file at <projname>/webinfo.xml. This file is "defined"
1.282 + by the parsing in trentm.com/build.py.
1.283 + """
1.284 + assert sys.platform != "win32", "'webdist' not implemented for win32"
1.285 + log.info("target: webdist")
1.286 + bitsDir = _get_project_bits_dir()
1.287 + buildDir = join("build", "webdist")
1.288 + distDir = join(buildDir, _project_name_)
1.289 + if exists(buildDir):
1.290 + _rmtree(buildDir)
1.291 + os.makedirs(distDir)
1.292 +
1.293 + target_docs()
1.294 +
1.295 + # Copy the webdist bits to the build tree.
1.296 + manifest = [
1.297 + "project-info.xml",
1.298 + "index.markdown",
1.299 + "LICENSE.txt",
1.300 + "which.py",
1.301 + "logo.jpg",
1.302 + ]
1.303 + for src in manifest:
1.304 + if dirname(src):
1.305 + dst = join(distDir, dirname(src))
1.306 + os.makedirs(dst)
1.307 + else:
1.308 + dst = distDir
1.309 + _run("cp %s %s" % (src, dst))
1.310 +
1.311 + # Zip up the webdist contents.
1.312 + ver = _get_project_version()
1.313 + bit = abspath(join(bitsDir, "%s-%s.web" % (_project_name_, ver)))
1.314 + if exists(bit):
1.315 + os.remove(bit)
1.316 + _run_in_dir("zip -r %s %s" % (bit, _project_name_), buildDir, log.info)
1.317 +
1.318 +
1.319 +def target_install():
1.320 + """Use the setup.py script to install."""
1.321 + log.info("target: install")
1.322 + _run("python setup.py install")
1.323 +
1.324 +
1.325 +def target_upload_local():
1.326 + """Update release bits to *local* trentm.com bits-dir location.
1.327 +
1.328 + This is different from the "upload" target, which uploads release
1.329 + bits remotely to trentm.com.
1.330 + """
1.331 + log.info("target: upload_local")
1.332 + assert sys.platform != "win32", "'upload_local' not implemented for win32"
1.333 +
1.334 + ver = _get_project_version()
1.335 + localBitsDir = _get_local_bits_dir()
1.336 + uploadDir = join(localBitsDir, _project_name_, ver)
1.337 +
1.338 + bitsPattern = join(_get_project_bits_dir(),
1.339 + "%s-*%s*" % (_project_name_, ver))
1.340 + bits = glob.glob(bitsPattern)
1.341 + if not bits:
1.342 + log.info("no bits matching '%s' to upload", bitsPattern)
1.343 + else:
1.344 + if not exists(uploadDir):
1.345 + os.makedirs(uploadDir)
1.346 + for bit in bits:
1.347 + _run("cp %s %s" % (bit, uploadDir), log.info)
1.348 +
1.349 +
1.350 +def target_upload():
1.351 + """Upload binary and source distribution to trentm.com bits
1.352 + directory.
1.353 + """
1.354 + log.info("target: upload")
1.355 +
1.356 + ver = _get_project_version()
1.357 + bitsDir = _get_project_bits_dir()
1.358 + bitsPattern = join(bitsDir, "%s-*%s*" % (_project_name_, ver))
1.359 + bits = glob.glob(bitsPattern)
1.360 + if not bits:
1.361 + log.info("no bits matching '%s' to upload", bitsPattern)
1.362 + return
1.363 +
1.364 + # Ensure have all the expected bits.
1.365 + expectedBits = [
1.366 + re.compile("%s-.*\.zip$" % _project_name_),
1.367 + re.compile("%s-.*\.web$" % _project_name_)
1.368 + ]
1.369 + for expectedBit in expectedBits:
1.370 + for bit in bits:
1.371 + if expectedBit.search(bit):
1.372 + break
1.373 + else:
1.374 + raise Error("can't find expected bit matching '%s' in '%s' dir"
1.375 + % (expectedBit.pattern, bitsDir))
1.376 +
1.377 + # Upload the bits.
1.378 + user = "trentm"
1.379 + host = "trentm.com"
1.380 + remoteBitsBaseDir = "~/data/bits"
1.381 + remoteBitsDir = join(remoteBitsBaseDir, _project_name_, ver)
1.382 + if sys.platform == "win32":
1.383 + ssh = "plink"
1.384 + scp = "pscp -unsafe"
1.385 + else:
1.386 + ssh = "ssh"
1.387 + scp = "scp"
1.388 + _run("%s %s@%s 'mkdir -p %s'" % (ssh, user, host, remoteBitsDir), log.info)
1.389 + for bit in bits:
1.390 + _run("%s %s %s@%s:%s" % (scp, bit, user, host, remoteBitsDir),
1.391 + log.info)
1.392 +
1.393 +
1.394 +def target_check_version():
1.395 + """grep for version strings in source code
1.396 +
1.397 + List all things that look like version strings in the source code.
1.398 + Used for checking that versioning is updated across the board.
1.399 + """
1.400 + sources = [
1.401 + "which.py",
1.402 + "project-info.xml",
1.403 + ]
1.404 + pattern = r'[0-9]\+\(\.\|, \)[0-9]\+\(\.\|, \)[0-9]\+'
1.405 + _run('grep -n "%s" %s' % (pattern, ' '.join(sources)), None)
1.406 +
1.407 +
1.408 +
1.409 +#---- mainline
1.410 +
1.411 +def build(targets=[]):
1.412 + log.debug("build(targets=%r)" % targets)
1.413 + available = _getTargets()
1.414 + if not targets:
1.415 + if available.has_key('default'):
1.416 + return available['default']()
1.417 + else:
1.418 + log.warn("No default target available. Doing nothing.")
1.419 + else:
1.420 + for target in targets:
1.421 + if available.has_key(target):
1.422 + retval = available[target]()
1.423 + if retval:
1.424 + raise Error("Error running '%s' target: retval=%s"\
1.425 + % (target, retval))
1.426 + else:
1.427 + raise Error("Unknown target: '%s'" % target)
1.428 +
1.429 +def main(argv):
1.430 + _setup_logging()
1.431 +
1.432 + # Process options.
1.433 + optlist, targets = getopt.getopt(argv[1:], 'ht', ['help', 'targets'])
1.434 + for opt, optarg in optlist:
1.435 + if opt in ('-h', '--help'):
1.436 + sys.stdout.write(__doc__ + '\n')
1.437 + return 0
1.438 + elif opt in ('-t', '--targets'):
1.439 + return _listTargets(_getTargets())
1.440 +
1.441 + return build(targets)
1.442 +
1.443 +if __name__ == "__main__":
1.444 + sys.exit( main(sys.argv) )
1.445 +
