The Tor Browser / file comparison
comparison: python/which/build.py
python/which/build.py
- changeset 0
- 6474c204b198
equal
deleted
inserted
replaced
| -1:000000000000 | 0:9c3e9de9d422 |
|---|---|
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2002-2005 ActiveState | |
| 3 # See LICENSE.txt for license details. | |
| 4 | |
| 5 """ | |
| 6 which.py dev build script | |
| 7 | |
| 8 Usage: | |
| 9 python build.py [<options>...] [<targets>...] | |
| 10 | |
| 11 Options: | |
| 12 --help, -h Print this help and exit. | |
| 13 --targets, -t List all available targets. | |
| 14 | |
| 15 This is the primary build script for the which.py project. It exists | |
| 16 to assist in building, maintaining, and distributing this project. | |
| 17 | |
| 18 It is intended to have Makefile semantics. I.e. 'python build.py' | |
| 19 will build execute the default target, 'python build.py foo' will | |
| 20 build target foo, etc. However, there is no intelligent target | |
| 21 interdependency tracking (I suppose I could do that with function | |
| 22 attributes). | |
| 23 """ | |
| 24 | |
| 25 import os | |
| 26 from os.path import basename, dirname, splitext, isfile, isdir, exists, \ | |
| 27 join, abspath, normpath | |
| 28 import sys | |
| 29 import getopt | |
| 30 import types | |
| 31 import getpass | |
| 32 import shutil | |
| 33 import glob | |
| 34 import logging | |
| 35 import re | |
| 36 | |
| 37 | |
| 38 | |
| 39 #---- exceptions | |
| 40 | |
| 41 class Error(Exception): | |
| 42 pass | |
| 43 | |
| 44 | |
| 45 | |
| 46 #---- globals | |
| 47 | |
| 48 log = logging.getLogger("build") | |
| 49 | |
| 50 | |
| 51 | |
| 52 | |
| 53 #---- globals | |
| 54 | |
| 55 _project_name_ = "which" | |
| 56 | |
| 57 | |
| 58 | |
| 59 #---- internal support routines | |
| 60 | |
| 61 def _get_trentm_com_dir(): | |
| 62 """Return the path to the local trentm.com source tree.""" | |
| 63 d = normpath(join(dirname(__file__), os.pardir, "trentm.com")) | |
| 64 if not isdir(d): | |
| 65 raise Error("could not find 'trentm.com' src dir at '%s'" % d) | |
| 66 return d | |
| 67 | |
| 68 def _get_local_bits_dir(): | |
| 69 import imp | |
| 70 info = imp.find_module("tmconfig", [_get_trentm_com_dir()]) | |
| 71 tmconfig = imp.load_module("tmconfig", *info) | |
| 72 return tmconfig.bitsDir | |
| 73 | |
| 74 def _get_project_bits_dir(): | |
| 75 d = normpath(join(dirname(__file__), "bits")) | |
| 76 return d | |
| 77 | |
| 78 def _get_project_version(): | |
| 79 import imp, os | |
| 80 data = imp.find_module(_project_name_, [os.path.dirname(__file__)]) | |
| 81 mod = imp.load_module(_project_name_, *data) | |
| 82 return mod.__version__ | |
| 83 | |
| 84 | |
| 85 # Recipe: run (0.5.1) in /Users/trentm/tm/recipes/cookbook | |
| 86 _RUN_DEFAULT_LOGSTREAM = ("RUN", "DEFAULT", "LOGSTREAM") | |
| 87 def __run_log(logstream, msg, *args, **kwargs): | |
| 88 if not logstream: | |
| 89 pass | |
| 90 elif logstream is _RUN_DEFAULT_LOGSTREAM: | |
| 91 try: | |
| 92 log.debug(msg, *args, **kwargs) | |
| 93 except NameError: | |
| 94 pass | |
| 95 else: | |
| 96 logstream(msg, *args, **kwargs) | |
| 97 | |
| 98 def _run(cmd, logstream=_RUN_DEFAULT_LOGSTREAM): | |
| 99 """Run the given command. | |
| 100 | |
| 101 "cmd" is the command to run | |
| 102 "logstream" is an optional logging stream on which to log the command. | |
| 103 If None, no logging is done. If unspecifed, this looks for a Logger | |
| 104 instance named 'log' and logs the command on log.debug(). | |
| 105 | |
| 106 Raises OSError is the command returns a non-zero exit status. | |
| 107 """ | |
| 108 __run_log(logstream, "running '%s'", cmd) | |
| 109 retval = os.system(cmd) | |
| 110 if hasattr(os, "WEXITSTATUS"): | |
| 111 status = os.WEXITSTATUS(retval) | |
| 112 else: | |
| 113 status = retval | |
| 114 if status: | |
| 115 #TODO: add std OSError attributes or pick more approp. exception | |
| 116 raise OSError("error running '%s': %r" % (cmd, status)) | |
| 117 | |
| 118 def _run_in_dir(cmd, cwd, logstream=_RUN_DEFAULT_LOGSTREAM): | |
| 119 old_dir = os.getcwd() | |
| 120 try: | |
| 121 os.chdir(cwd) | |
| 122 __run_log(logstream, "running '%s' in '%s'", cmd, cwd) | |
| 123 _run(cmd, logstream=None) | |
| 124 finally: | |
| 125 os.chdir(old_dir) | |
| 126 | |
| 127 | |
| 128 # Recipe: rmtree (0.5) in /Users/trentm/tm/recipes/cookbook | |
| 129 def _rmtree_OnError(rmFunction, filePath, excInfo): | |
| 130 if excInfo[0] == OSError: | |
| 131 # presuming because file is read-only | |
| 132 os.chmod(filePath, 0777) | |
| 133 rmFunction(filePath) | |
| 134 def _rmtree(dirname): | |
| 135 import shutil | |
| 136 shutil.rmtree(dirname, 0, _rmtree_OnError) | |
| 137 | |
| 138 | |
| 139 # Recipe: pretty_logging (0.1) in /Users/trentm/tm/recipes/cookbook | |
| 140 class _PerLevelFormatter(logging.Formatter): | |
| 141 """Allow multiple format string -- depending on the log level. | |
| 142 | |
| 143 A "fmtFromLevel" optional arg is added to the constructor. It can be | |
| 144 a dictionary mapping a log record level to a format string. The | |
| 145 usual "fmt" argument acts as the default. | |
| 146 """ | |
| 147 def __init__(self, fmt=None, datefmt=None, fmtFromLevel=None): | |
| 148 logging.Formatter.__init__(self, fmt, datefmt) | |
| 149 if fmtFromLevel is None: | |
| 150 self.fmtFromLevel = {} | |
| 151 else: | |
| 152 self.fmtFromLevel = fmtFromLevel | |
| 153 def format(self, record): | |
| 154 record.levelname = record.levelname.lower() | |
| 155 if record.levelno in self.fmtFromLevel: | |
| 156 #XXX This is a non-threadsafe HACK. Really the base Formatter | |
| 157 # class should provide a hook accessor for the _fmt | |
| 158 # attribute. *Could* add a lock guard here (overkill?). | |
| 159 _saved_fmt = self._fmt | |
| 160 self._fmt = self.fmtFromLevel[record.levelno] | |
| 161 try: | |
| 162 return logging.Formatter.format(self, record) | |
| 163 finally: | |
| 164 self._fmt = _saved_fmt | |
| 165 else: | |
| 166 return logging.Formatter.format(self, record) | |
| 167 | |
| 168 def _setup_logging(): | |
| 169 hdlr = logging.StreamHandler() | |
| 170 defaultFmt = "%(name)s: %(levelname)s: %(message)s" | |
| 171 infoFmt = "%(name)s: %(message)s" | |
| 172 fmtr = _PerLevelFormatter(fmt=defaultFmt, | |
| 173 fmtFromLevel={logging.INFO: infoFmt}) | |
| 174 hdlr.setFormatter(fmtr) | |
| 175 logging.root.addHandler(hdlr) | |
| 176 log.setLevel(logging.INFO) | |
| 177 | |
| 178 | |
| 179 def _getTargets(): | |
| 180 """Find all targets and return a dict of targetName:targetFunc items.""" | |
| 181 targets = {} | |
| 182 for name, attr in sys.modules[__name__].__dict__.items(): | |
| 183 if name.startswith('target_'): | |
| 184 targets[ name[len('target_'):] ] = attr | |
| 185 return targets | |
| 186 | |
| 187 def _listTargets(targets): | |
| 188 """Pretty print a list of targets.""" | |
| 189 width = 77 | |
| 190 nameWidth = 15 # min width | |
| 191 for name in targets.keys(): | |
| 192 nameWidth = max(nameWidth, len(name)) | |
| 193 nameWidth += 2 # space btwn name and doc | |
| 194 format = "%%-%ds%%s" % nameWidth | |
| 195 print format % ("TARGET", "DESCRIPTION") | |
| 196 for name, func in sorted(targets.items()): | |
| 197 doc = _first_paragraph(func.__doc__ or "", True) | |
| 198 if len(doc) > (width - nameWidth): | |
| 199 doc = doc[:(width-nameWidth-3)] + "..." | |
| 200 print format % (name, doc) | |
| 201 | |
| 202 | |
| 203 # Recipe: first_paragraph (1.0.1) in /Users/trentm/tm/recipes/cookbook | |
| 204 def _first_paragraph(text, join_lines=False): | |
| 205 """Return the first paragraph of the given text.""" | |
| 206 para = text.lstrip().split('\n\n', 1)[0] | |
| 207 if join_lines: | |
| 208 lines = [line.strip() for line in para.splitlines(0)] | |
| 209 para = ' '.join(lines) | |
| 210 return para | |
| 211 | |
| 212 | |
| 213 | |
| 214 #---- build targets | |
| 215 | |
| 216 def target_default(): | |
| 217 target_all() | |
| 218 | |
| 219 def target_all(): | |
| 220 """Build all release packages.""" | |
| 221 log.info("target: default") | |
| 222 if sys.platform == "win32": | |
| 223 target_launcher() | |
| 224 target_sdist() | |
| 225 target_webdist() | |
| 226 | |
| 227 | |
| 228 def target_clean(): | |
| 229 """remove all build/generated bits""" | |
| 230 log.info("target: clean") | |
| 231 if sys.platform == "win32": | |
| 232 _run("nmake -f Makefile.win clean") | |
| 233 | |
| 234 ver = _get_project_version() | |
| 235 dirs = ["dist", "build", "%s-%s" % (_project_name_, ver)] | |
| 236 for d in dirs: | |
| 237 print "removing '%s'" % d | |
| 238 if os.path.isdir(d): _rmtree(d) | |
| 239 | |
| 240 patterns = ["*.pyc", "*~", "MANIFEST", | |
| 241 os.path.join("test", "*~"), | |
| 242 os.path.join("test", "*.pyc"), | |
| 243 ] | |
| 244 for pattern in patterns: | |
| 245 for file in glob.glob(pattern): | |
| 246 print "removing '%s'" % file | |
| 247 os.unlink(file) | |
| 248 | |
| 249 | |
| 250 def target_launcher(): | |
| 251 """Build the Windows launcher executable.""" | |
| 252 log.info("target: launcher") | |
| 253 assert sys.platform == "win32", "'launcher' target only supported on Windows" | |
| 254 _run("nmake -f Makefile.win") | |
| 255 | |
| 256 | |
| 257 def target_docs(): | |
| 258 """Regenerate some doc bits from project-info.xml.""" | |
| 259 log.info("target: docs") | |
| 260 _run("projinfo -f project-info.xml -R -o README.txt --force") | |
| 261 _run("projinfo -f project-info.xml --index-markdown -o index.markdown --force") | |
| 262 | |
| 263 | |
| 264 def target_sdist(): | |
| 265 """Build a source distribution.""" | |
| 266 log.info("target: sdist") | |
| 267 target_docs() | |
| 268 bitsDir = _get_project_bits_dir() | |
| 269 _run("python setup.py sdist -f --formats zip -d %s" % bitsDir, | |
| 270 log.info) | |
| 271 | |
| 272 | |
| 273 def target_webdist(): | |
| 274 """Build a web dist package. | |
| 275 | |
| 276 "Web dist" packages are zip files with '.web' package. All files in | |
| 277 the zip must be under a dir named after the project. There must be a | |
| 278 webinfo.xml file at <projname>/webinfo.xml. This file is "defined" | |
| 279 by the parsing in trentm.com/build.py. | |
| 280 """ | |
| 281 assert sys.platform != "win32", "'webdist' not implemented for win32" | |
| 282 log.info("target: webdist") | |
| 283 bitsDir = _get_project_bits_dir() | |
| 284 buildDir = join("build", "webdist") | |
| 285 distDir = join(buildDir, _project_name_) | |
| 286 if exists(buildDir): | |
| 287 _rmtree(buildDir) | |
| 288 os.makedirs(distDir) | |
| 289 | |
| 290 target_docs() | |
| 291 | |
| 292 # Copy the webdist bits to the build tree. | |
| 293 manifest = [ | |
| 294 "project-info.xml", | |
| 295 "index.markdown", | |
| 296 "LICENSE.txt", | |
| 297 "which.py", | |
| 298 "logo.jpg", | |
| 299 ] | |
| 300 for src in manifest: | |
| 301 if dirname(src): | |
| 302 dst = join(distDir, dirname(src)) | |
| 303 os.makedirs(dst) | |
| 304 else: | |
| 305 dst = distDir | |
| 306 _run("cp %s %s" % (src, dst)) | |
| 307 | |
| 308 # Zip up the webdist contents. | |
| 309 ver = _get_project_version() | |
| 310 bit = abspath(join(bitsDir, "%s-%s.web" % (_project_name_, ver))) | |
| 311 if exists(bit): | |
| 312 os.remove(bit) | |
| 313 _run_in_dir("zip -r %s %s" % (bit, _project_name_), buildDir, log.info) | |
| 314 | |
| 315 | |
| 316 def target_install(): | |
| 317 """Use the setup.py script to install.""" | |
| 318 log.info("target: install") | |
| 319 _run("python setup.py install") | |
| 320 | |
| 321 | |
| 322 def target_upload_local(): | |
| 323 """Update release bits to *local* trentm.com bits-dir location. | |
| 324 | |
| 325 This is different from the "upload" target, which uploads release | |
| 326 bits remotely to trentm.com. | |
| 327 """ | |
| 328 log.info("target: upload_local") | |
| 329 assert sys.platform != "win32", "'upload_local' not implemented for win32" | |
| 330 | |
| 331 ver = _get_project_version() | |
| 332 localBitsDir = _get_local_bits_dir() | |
| 333 uploadDir = join(localBitsDir, _project_name_, ver) | |
| 334 | |
| 335 bitsPattern = join(_get_project_bits_dir(), | |
| 336 "%s-*%s*" % (_project_name_, ver)) | |
| 337 bits = glob.glob(bitsPattern) | |
| 338 if not bits: | |
| 339 log.info("no bits matching '%s' to upload", bitsPattern) | |
| 340 else: | |
| 341 if not exists(uploadDir): | |
| 342 os.makedirs(uploadDir) | |
| 343 for bit in bits: | |
| 344 _run("cp %s %s" % (bit, uploadDir), log.info) | |
| 345 | |
| 346 | |
| 347 def target_upload(): | |
| 348 """Upload binary and source distribution to trentm.com bits | |
| 349 directory. | |
| 350 """ | |
| 351 log.info("target: upload") | |
| 352 | |
| 353 ver = _get_project_version() | |
| 354 bitsDir = _get_project_bits_dir() | |
| 355 bitsPattern = join(bitsDir, "%s-*%s*" % (_project_name_, ver)) | |
| 356 bits = glob.glob(bitsPattern) | |
| 357 if not bits: | |
| 358 log.info("no bits matching '%s' to upload", bitsPattern) | |
| 359 return | |
| 360 | |
| 361 # Ensure have all the expected bits. | |
| 362 expectedBits = [ | |
| 363 re.compile("%s-.*\.zip$" % _project_name_), | |
| 364 re.compile("%s-.*\.web$" % _project_name_) | |
| 365 ] | |
| 366 for expectedBit in expectedBits: | |
| 367 for bit in bits: | |
| 368 if expectedBit.search(bit): | |
| 369 break | |
| 370 else: | |
| 371 raise Error("can't find expected bit matching '%s' in '%s' dir" | |
| 372 % (expectedBit.pattern, bitsDir)) | |
| 373 | |
| 374 # Upload the bits. | |
| 375 user = "trentm" | |
| 376 host = "trentm.com" | |
| 377 remoteBitsBaseDir = "~/data/bits" | |
| 378 remoteBitsDir = join(remoteBitsBaseDir, _project_name_, ver) | |
| 379 if sys.platform == "win32": | |
| 380 ssh = "plink" | |
| 381 scp = "pscp -unsafe" | |
| 382 else: | |
| 383 ssh = "ssh" | |
| 384 scp = "scp" | |
| 385 _run("%s %s@%s 'mkdir -p %s'" % (ssh, user, host, remoteBitsDir), log.info) | |
| 386 for bit in bits: | |
| 387 _run("%s %s %s@%s:%s" % (scp, bit, user, host, remoteBitsDir), | |
| 388 log.info) | |
| 389 | |
| 390 | |
| 391 def target_check_version(): | |
| 392 """grep for version strings in source code | |
| 393 | |
| 394 List all things that look like version strings in the source code. | |
| 395 Used for checking that versioning is updated across the board. | |
| 396 """ | |
| 397 sources = [ | |
| 398 "which.py", | |
| 399 "project-info.xml", | |
| 400 ] | |
| 401 pattern = r'[0-9]\+\(\.\|, \)[0-9]\+\(\.\|, \)[0-9]\+' | |
| 402 _run('grep -n "%s" %s' % (pattern, ' '.join(sources)), None) | |
| 403 | |
| 404 | |
| 405 | |
| 406 #---- mainline | |
| 407 | |
| 408 def build(targets=[]): | |
| 409 log.debug("build(targets=%r)" % targets) | |
| 410 available = _getTargets() | |
| 411 if not targets: | |
| 412 if available.has_key('default'): | |
| 413 return available['default']() | |
| 414 else: | |
| 415 log.warn("No default target available. Doing nothing.") | |
| 416 else: | |
| 417 for target in targets: | |
| 418 if available.has_key(target): | |
| 419 retval = available[target]() | |
| 420 if retval: | |
| 421 raise Error("Error running '%s' target: retval=%s"\ | |
| 422 % (target, retval)) | |
| 423 else: | |
| 424 raise Error("Unknown target: '%s'" % target) | |
| 425 | |
| 426 def main(argv): | |
| 427 _setup_logging() | |
| 428 | |
| 429 # Process options. | |
| 430 optlist, targets = getopt.getopt(argv[1:], 'ht', ['help', 'targets']) | |
| 431 for opt, optarg in optlist: | |
| 432 if opt in ('-h', '--help'): | |
| 433 sys.stdout.write(__doc__ + '\n') | |
| 434 return 0 | |
| 435 elif opt in ('-t', '--targets'): | |
| 436 return _listTargets(_getTargets()) | |
| 437 | |
| 438 return build(targets) | |
| 439 | |
| 440 if __name__ == "__main__": | |
| 441 sys.exit( main(sys.argv) ) | |
| 442 |
