michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: import sys
michael@0: import os
michael@0: import optparse
michael@0: import webbrowser
michael@0: import time
michael@0: 
michael@0: from copy import copy
michael@0: import simplejson as json
michael@0: from cuddlefish import packaging
michael@0: from cuddlefish._version import get_versions
michael@0: 
michael@0: MOZRUNNER_BIN_NOT_FOUND = 'Mozrunner could not locate your binary'
michael@0: MOZRUNNER_BIN_NOT_FOUND_HELP = """
michael@0: I can't find the application binary in any of its default locations
michael@0: on your system. Please specify one using the -b/--binary option.
michael@0: """
michael@0: 
michael@0: UPDATE_RDF_FILENAME = "%s.update.rdf"
michael@0: XPI_FILENAME = "%s.xpi"
michael@0: 
michael@0: usage = """
michael@0: %prog [options] command [command-specific options]
michael@0: 
michael@0: Supported Commands:
michael@0:   init       - create a sample addon in an empty directory
michael@0:   test       - run tests
michael@0:   run        - run program
michael@0:   xpi        - generate an xpi
michael@0: 
michael@0: Internal Commands:
michael@0:   testcfx    - test the cfx tool
michael@0:   testex     - test all example code
michael@0:   testpkgs   - test all installed packages
michael@0:   testall    - test whole environment
michael@0: 
michael@0: Experimental and internal commands and options are not supported and may be
michael@0: changed or removed in the future.
michael@0: """
michael@0: 
michael@0: global_options = [
michael@0:     (("-v", "--verbose",), dict(dest="verbose",
michael@0:                                 help="enable lots of output",
michael@0:                                 action="store_true",
michael@0:                                 default=False)),
michael@0:     ]
michael@0: 
michael@0: parser_groups = (
michael@0:     ("Supported Command-Specific Options", [
michael@0:         (("", "--update-url",), dict(dest="update_url",
michael@0:                                      help="update URL in install.rdf",
michael@0:                                      metavar=None,
michael@0:                                      default=None,
michael@0:                                      cmds=['xpi'])),
michael@0:         (("", "--update-link",), dict(dest="update_link",
michael@0:                                       help="generate update.rdf",
michael@0:                                       metavar=None,
michael@0:                                       default=None,
michael@0:                                       cmds=['xpi'])),
michael@0:         (("-p", "--profiledir",), dict(dest="profiledir",
michael@0:                                        help=("profile directory to pass to "
michael@0:                                              "app"),
michael@0:                                        metavar=None,
michael@0:                                        default=None,
michael@0:                                        cmds=['test', 'run', 'testex',
michael@0:                                              'testpkgs', 'testall'])),
michael@0:         (("-b", "--binary",), dict(dest="binary",
michael@0:                                    help="path to app binary",
michael@0:                                    metavar=None,
michael@0:                                    default=None,
michael@0:                                    cmds=['test', 'run', 'testex', 'testpkgs',
michael@0:                                          'testall'])),
michael@0:         (("", "--binary-args",), dict(dest="cmdargs",
michael@0:                                  help=("additional arguments passed to the "
michael@0:                                        "binary"),
michael@0:                                  metavar=None,
michael@0:                                  default=None,
michael@0:                                  cmds=['run', 'test'])),
michael@0:         (("", "--dependencies",), dict(dest="dep_tests",
michael@0:                                        help="include tests for all deps",
michael@0:                                        action="store_true",
michael@0:                                        default=False,
michael@0:                                        cmds=['test', 'testex', 'testpkgs',
michael@0:                                              'testall'])),
michael@0:         (("", "--times",), dict(dest="iterations",
michael@0:                                 type="int",
michael@0:                                 help="number of times to run tests",
michael@0:                                 default=1,
michael@0:                                 cmds=['test', 'testex', 'testpkgs',
michael@0:                                       'testall'])),
michael@0:         (("-f", "--filter",), dict(dest="filter",
michael@0:                                    help=("only run tests whose filenames "
michael@0:                                          "match FILENAME and optionally "
michael@0:                                          "match TESTNAME, both regexps"),
michael@0:                                    metavar="FILENAME[:TESTNAME]",
michael@0:                                    default='',
michael@0:                                    cmds=['test', 'testex', 'testaddons', 'testpkgs',
michael@0:                                          'testall'])),
michael@0:         (("-g", "--use-config",), dict(dest="config",
michael@0:                                        help="use named config from local.json",
michael@0:                                        metavar=None,
michael@0:                                        default="default",
michael@0:                                        cmds=['test', 'run', 'xpi', 'testex',
michael@0:                                              'testpkgs', 'testall'])),
michael@0:         (("", "--templatedir",), dict(dest="templatedir",
michael@0:                                       help="XULRunner app/ext. template",
michael@0:                                       metavar=None,
michael@0:                                       default=None,
michael@0:                                       cmds=['run', 'xpi'])),
michael@0:         (("", "--package-path",), dict(dest="packagepath", action="append",
michael@0:                                        help="extra directories for package search",
michael@0:                                        metavar=None,
michael@0:                                        default=[],
michael@0:                                        cmds=['run', 'xpi', 'test'])),
michael@0:         (("", "--extra-packages",), dict(dest="extra_packages",
michael@0:                                          help=("extra packages to include, "
michael@0:                                                "comma-separated. Default is "
michael@0:                                                "'addon-sdk'."),
michael@0:                                          metavar=None,
michael@0:                                          default="addon-sdk",
michael@0:                                          cmds=['run', 'xpi', 'test', 'testex',
michael@0:                                                'testpkgs', 'testall',
michael@0:                                                'testcfx'])),
michael@0:         (("", "--pkgdir",), dict(dest="pkgdir",
michael@0:                                  help=("package dir containing "
michael@0:                                        "package.json; default is "
michael@0:                                        "current directory"),
michael@0:                                  metavar=None,
michael@0:                                  default=None,
michael@0:                                  cmds=['run', 'xpi', 'test'])),
michael@0:         (("", "--static-args",), dict(dest="static_args",
michael@0:                                       help="extra harness options as JSON",
michael@0:                                       type="json",
michael@0:                                       metavar=None,
michael@0:                                       default="{}",
michael@0:                                       cmds=['run', 'xpi'])),
michael@0:         (("", "--parseable",), dict(dest="parseable",
michael@0:                                     help="display test output in a parseable format",
michael@0:                                     action="store_true",
michael@0:                                     default=False,
michael@0:                                     cmds=['run', 'test', 'testex', 'testpkgs',
michael@0:                                           'testaddons', 'testall'])),
michael@0:         ]
michael@0:      ),
michael@0: 
michael@0:     ("Experimental Command-Specific Options", [
michael@0:         (("-a", "--app",), dict(dest="app",
michael@0:                                 help=("app to run: firefox (default), fennec, "
michael@0:                                       "fennec-on-device, xulrunner or "
michael@0:                                       "thunderbird"),
michael@0:                                 metavar=None,
michael@0:                                 type="choice",
michael@0:                                 choices=["firefox", "fennec",
michael@0:                                          "fennec-on-device", "thunderbird",
michael@0:                                          "xulrunner"],
michael@0:                                 default="firefox",
michael@0:                                 cmds=['test', 'run', 'testex', 'testpkgs',
michael@0:                                       'testall'])),
michael@0:         (("-o", "--overload-modules",), dict(dest="overload_modules",
michael@0:                                      help=("Overload JS modules integrated into"
michael@0:                                            " Firefox with the one from your SDK"
michael@0:                                            " repository"),
michael@0:                                      action="store_true",
michael@0:                                      default=False,
michael@0:                                      cmds=['run', 'test', 'testex', 'testpkgs',
michael@0:                                            'testall'])),
michael@0:         (("", "--strip-sdk",), dict(dest="bundle_sdk",
michael@0:                                     help=("Do not ship SDK modules in the xpi"),
michael@0:                                     action="store_false",
michael@0:                                     default=False,
michael@0:                                     cmds=['run', 'test', 'testex', 'testpkgs',
michael@0:                                           'testall', 'xpi'])),
michael@0:         (("", "--force-use-bundled-sdk",), dict(dest="force_use_bundled_sdk",
michael@0:                                     help=("When --strip-sdk isn't passed, "
michael@0:                                           "force using sdk modules shipped in "
michael@0:                                           "the xpi instead of firefox ones"),
michael@0:                                     action="store_true",
michael@0:                                     default=False,
michael@0:                                     cmds=['run', 'test', 'testex', 'testpkgs',
michael@0:                                           'testall', 'xpi'])),
michael@0:         (("", "--no-run",), dict(dest="no_run",
michael@0:                                      help=("Instead of launching the "
michael@0:                                            "application, just show the command "
michael@0:                                            "for doing so.  Use this to launch "
michael@0:                                            "the application in a debugger like "
michael@0:                                            "gdb."),
michael@0:                                      action="store_true",
michael@0:                                      default=False,
michael@0:                                      cmds=['run', 'test'])),
michael@0:         (("", "--no-strip-xpi",), dict(dest="no_strip_xpi",
michael@0:                                     help="retain unused modules in XPI",
michael@0:                                     action="store_true",
michael@0:                                     default=False,
michael@0:                                     cmds=['xpi'])),
michael@0:         (("", "--force-mobile",), dict(dest="enable_mobile",
michael@0:                                     help="Force compatibility with Firefox Mobile",
michael@0:                                     action="store_true",
michael@0:                                     default=False,
michael@0:                                     cmds=['run', 'test', 'xpi', 'testall'])),
michael@0:         (("", "--mobile-app",), dict(dest="mobile_app_name",
michael@0:                                     help=("Name of your Android application to "
michael@0:                                           "use. Possible values: 'firefox', "
michael@0:                                           "'firefox_beta', 'fennec_aurora', "
michael@0:                                           "'fennec' (for nightly)."),
michael@0:                                     metavar=None,
michael@0:                                     default=None,
michael@0:                                     cmds=['run', 'test', 'testall'])),
michael@0:         (("", "--harness-option",), dict(dest="extra_harness_option_args",
michael@0:                                          help=("Extra properties added to "
michael@0:                                                "harness-options.json"),
michael@0:                                          action="append",
michael@0:                                          metavar="KEY=VALUE",
michael@0:                                          default=[],
michael@0:                                          cmds=['xpi'])),
michael@0:         (("", "--stop-on-error",), dict(dest="stopOnError",
michael@0:                                   help="Stop running tests after the first failure",
michael@0:                                   action="store_true",
michael@0:                                   metavar=None,
michael@0:                                   default=False,
michael@0:                                   cmds=['test', 'testex', 'testpkgs'])),
michael@0:         (("", "--check-memory",), dict(dest="check_memory",
michael@0:                                        help="attempts to detect leaked compartments after a test run",
michael@0:                                        action="store_true",
michael@0:                                        default=False,
michael@0:                                        cmds=['test', 'testpkgs', 'testaddons',
michael@0:                                              'testall'])),
michael@0:         (("", "--output-file",), dict(dest="output_file",
michael@0:                                       help="Where to put the finished .xpi",
michael@0:                                       default=None,
michael@0:                                       cmds=['xpi'])),
michael@0:         (("", "--manifest-overload",), dict(dest="manifest_overload",
michael@0:                                       help="JSON file to overload package.json properties",
michael@0:                                       default=None,
michael@0:                                       cmds=['xpi'])),
michael@0:         ]
michael@0:      ),
michael@0: 
michael@0:     ("Internal Command-Specific Options", [
michael@0:         (("", "--addons",), dict(dest="addons",
michael@0:                                  help=("paths of addons to install, "
michael@0:                                        "comma-separated"),
michael@0:                                  metavar=None,
michael@0:                                  default=None,
michael@0:                                  cmds=['test', 'run', 'testex', 'testpkgs',
michael@0:                                        'testall'])),
michael@0:         (("", "--test-runner-pkg",), dict(dest="test_runner_pkg",
michael@0:                                           help=("name of package "
michael@0:                                                 "containing test runner "
michael@0:                                                 "program (default is "
michael@0:                                                 "test-harness)"),
michael@0:                                           default="addon-sdk",
michael@0:                                           cmds=['test', 'testex', 'testpkgs',
michael@0:                                                 'testall'])),
michael@0:         # --keydir was removed in 1.0b5, but we keep it around in the options
michael@0:         # parser to make life easier for frontends like FlightDeck which
michael@0:         # might still pass it. It can go away once the frontends are updated.
michael@0:         (("", "--keydir",), dict(dest="keydir",
michael@0:                                  help=("obsolete, ignored"),
michael@0:                                  metavar=None,
michael@0:                                  default=None,
michael@0:                                  cmds=['test', 'run', 'xpi', 'testex',
michael@0:                                        'testpkgs', 'testall'])),
michael@0:         (("", "--e10s",), dict(dest="enable_e10s",
michael@0:                                help="enable out-of-process Jetpacks",
michael@0:                                action="store_true",
michael@0:                                default=False,
michael@0:                                cmds=['test', 'run', 'testex', 'testpkgs'])),
michael@0:         (("", "--logfile",), dict(dest="logfile",
michael@0:                                   help="log console output to file",
michael@0:                                   metavar=None,
michael@0:                                   default=None,
michael@0:                                   cmds=['run', 'test', 'testex', 'testpkgs'])),
michael@0:         # TODO: This should default to true once our memory debugging
michael@0:         # issues are resolved; see bug 592774.
michael@0:         (("", "--profile-memory",), dict(dest="profileMemory",
michael@0:                                          help=("profile memory usage "
michael@0:                                                "(default is false)"),
michael@0:                                          type="int",
michael@0:                                          action="store",
michael@0:                                          default=0,
michael@0:                                          cmds=['test', 'testex', 'testpkgs',
michael@0:                                                'testall'])),
michael@0:         ]
michael@0:      ),
michael@0:     )
michael@0: 
michael@0: def find_parent_package(cur_dir):
michael@0:     tail = True
michael@0:     while tail:
michael@0:         if os.path.exists(os.path.join(cur_dir, 'package.json')):
michael@0:             return cur_dir
michael@0:         cur_dir, tail = os.path.split(cur_dir)
michael@0:     return None
michael@0: 
michael@0: def check_json(option, opt, value):
michael@0:     # We return the parsed JSON here; see bug 610816 for background on why.
michael@0:     try:
michael@0:         return json.loads(value)
michael@0:     except ValueError:
michael@0:         raise optparse.OptionValueError("Option %s must be JSON." % opt)
michael@0: 
michael@0: class CfxOption(optparse.Option):
michael@0:     TYPES = optparse.Option.TYPES + ('json',)
michael@0:     TYPE_CHECKER = copy(optparse.Option.TYPE_CHECKER)
michael@0:     TYPE_CHECKER['json'] = check_json
michael@0: 
michael@0: def parse_args(arguments, global_options, usage, version, parser_groups,
michael@0:                defaults=None):
michael@0:     parser = optparse.OptionParser(usage=usage.strip(), option_class=CfxOption,
michael@0:                                    version=version)
michael@0: 
michael@0:     def name_cmp(a, b):
michael@0:         # a[0]    = name sequence
michael@0:         # a[0][0] = short name (possibly empty string)
michael@0:         # a[0][1] = long name
michael@0:         names = []
michael@0:         for seq in (a, b):
michael@0:             names.append(seq[0][0][1:] if seq[0][0] else seq[0][1][2:])
michael@0:         return cmp(*names)
michael@0: 
michael@0:     global_options.sort(name_cmp)
michael@0:     for names, opts in global_options:
michael@0:         parser.add_option(*names, **opts)
michael@0: 
michael@0:     for group_name, options in parser_groups:
michael@0:         group = optparse.OptionGroup(parser, group_name)
michael@0:         options.sort(name_cmp)
michael@0:         for names, opts in options:
michael@0:             if 'cmds' in opts:
michael@0:                 cmds = opts['cmds']
michael@0:                 del opts['cmds']
michael@0:                 cmds.sort()
michael@0:                 if not 'help' in opts:
michael@0:                     opts['help'] = ""
michael@0:                 opts['help'] += " (%s)" % ", ".join(cmds)
michael@0:             group.add_option(*names, **opts)
michael@0:         parser.add_option_group(group)
michael@0: 
michael@0:     if defaults:
michael@0:         parser.set_defaults(**defaults)
michael@0: 
michael@0:     (options, args) = parser.parse_args(args=arguments)
michael@0: 
michael@0:     if not args:
michael@0:         parser.print_help()
michael@0:         parser.exit()
michael@0: 
michael@0:     return (options, args)
michael@0: 
michael@0: # all tests emit progress messages to stderr, not stdout. (the mozrunner
michael@0: # console output goes to stderr and is hard to change, and
michael@0: # unittest.TextTestRunner prefers stderr, so we send everything else there
michael@0: # too, to keep all the messages in order)
michael@0: 
michael@0: def test_all(env_root, defaults):
michael@0:     fail = False
michael@0: 
michael@0:     starttime = time.time()
michael@0: 
michael@0:     if not defaults['filter']:
michael@0:         print >>sys.stderr, "Testing cfx..."
michael@0:         sys.stderr.flush()
michael@0:         result = test_cfx(env_root, defaults['verbose'])
michael@0:         if result.failures or result.errors:
michael@0:             fail = True
michael@0: 
michael@0:     if not fail or not defaults.get("stopOnError"):
michael@0:         print >>sys.stderr, "Testing all examples..."
michael@0:         sys.stderr.flush()
michael@0: 
michael@0:         try:
michael@0:             test_all_examples(env_root, defaults)
michael@0:         except SystemExit, e:
michael@0:             fail = (e.code != 0) or fail
michael@0: 
michael@0:     if not fail or not defaults.get("stopOnError"):
michael@0:         print >>sys.stderr, "Testing all unit-test addons..."
michael@0:         sys.stderr.flush()
michael@0: 
michael@0:         try:
michael@0:             test_all_testaddons(env_root, defaults)
michael@0:         except SystemExit, e:
michael@0:             fail = (e.code != 0) or fail
michael@0: 
michael@0:     if not fail or not defaults.get("stopOnError"):
michael@0:         print >>sys.stderr, "Testing all packages..."
michael@0:         sys.stderr.flush()
michael@0:         try:
michael@0:             test_all_packages(env_root, defaults)
michael@0:         except SystemExit, e:
michael@0:             fail = (e.code != 0) or fail
michael@0: 
michael@0:     print >>sys.stderr, "Total time for all tests: %f seconds" % (time.time() - starttime)
michael@0: 
michael@0:     if fail:
michael@0:         print >>sys.stderr, "Some tests were unsuccessful."
michael@0:         sys.exit(1)
michael@0:     print >>sys.stderr, "All tests were successful. Ship it!"
michael@0:     sys.exit(0)
michael@0: 
michael@0: def test_cfx(env_root, verbose):
michael@0:     import cuddlefish.tests
michael@0: 
michael@0:     # tests write to stderr. flush everything before and after to avoid
michael@0:     # confusion later.
michael@0:     sys.stdout.flush(); sys.stderr.flush()
michael@0:     olddir = os.getcwd()
michael@0:     os.chdir(env_root)
michael@0:     retval = cuddlefish.tests.run(verbose)
michael@0:     os.chdir(olddir)
michael@0:     sys.stdout.flush(); sys.stderr.flush()
michael@0:     return retval
michael@0: 
michael@0: def test_all_testaddons(env_root, defaults):
michael@0:     addons_dir = os.path.join(env_root, "test", "addons")
michael@0:     addons = [dirname for dirname in os.listdir(addons_dir)
michael@0:                 if os.path.isdir(os.path.join(addons_dir, dirname))]
michael@0:     addons.sort()
michael@0:     fail = False
michael@0:     for dirname in addons:
michael@0:         if (not defaults['filter'].split(":")[0] in dirname):
michael@0:             continue
michael@0: 
michael@0:         print >>sys.stderr, "Testing %s..." % dirname
michael@0:         sys.stderr.flush()
michael@0:         try:
michael@0:             run(arguments=["run",
michael@0:                            "--pkgdir",
michael@0:                            os.path.join(addons_dir, dirname)],
michael@0:                 defaults=defaults,
michael@0:                 env_root=env_root)
michael@0:         except SystemExit, e:
michael@0:             fail = (e.code != 0) or fail
michael@0:         if fail and defaults.get("stopOnError"):
michael@0:             break
michael@0: 
michael@0:     if fail:
michael@0:         print >>sys.stderr, "Some test addons tests were unsuccessful."
michael@0:         sys.exit(-1)
michael@0: 
michael@0: def test_all_examples(env_root, defaults):
michael@0:     examples_dir = os.path.join(env_root, "examples")
michael@0:     examples = [dirname for dirname in os.listdir(examples_dir)
michael@0:                 if os.path.isdir(os.path.join(examples_dir, dirname))]
michael@0:     examples.sort()
michael@0:     fail = False
michael@0:     for dirname in examples:
michael@0:         if (not defaults['filter'].split(":")[0] in dirname):
michael@0:             continue
michael@0: 
michael@0:         print >>sys.stderr, "Testing %s..." % dirname
michael@0:         sys.stderr.flush()
michael@0:         try:
michael@0:             run(arguments=["test",
michael@0:                            "--pkgdir",
michael@0:                            os.path.join(examples_dir, dirname)],
michael@0:                 defaults=defaults,
michael@0:                 env_root=env_root)
michael@0:         except SystemExit, e:
michael@0:             fail = (e.code != 0) or fail
michael@0:         if fail and defaults.get("stopOnError"):
michael@0:             break
michael@0: 
michael@0:     if fail:
michael@0:         print >>sys.stderr, "Some examples tests were unsuccessful."
michael@0:         sys.exit(-1)
michael@0: 
michael@0: def test_all_packages(env_root, defaults):
michael@0:     packages_dir = os.path.join(env_root, "packages")
michael@0:     if os.path.isdir(packages_dir):
michael@0:       packages = [dirname for dirname in os.listdir(packages_dir)
michael@0:                   if os.path.isdir(os.path.join(packages_dir, dirname))]
michael@0:     else:
michael@0:       packages = []
michael@0:     packages.append(env_root)
michael@0:     packages.sort()
michael@0:     print >>sys.stderr, "Testing all available packages: %s." % (", ".join(packages))
michael@0:     sys.stderr.flush()
michael@0:     fail = False
michael@0:     for dirname in packages:
michael@0:         print >>sys.stderr, "Testing %s..." % dirname
michael@0:         sys.stderr.flush()
michael@0:         try:
michael@0:             run(arguments=["test",
michael@0:                            "--pkgdir",
michael@0:                            os.path.join(packages_dir, dirname)],
michael@0:                 defaults=defaults,
michael@0:                 env_root=env_root)
michael@0:         except SystemExit, e:
michael@0:             fail = (e.code != 0) or fail
michael@0:         if fail and defaults.get('stopOnError'):
michael@0:             break
michael@0:     if fail:
michael@0:         print >>sys.stderr, "Some package tests were unsuccessful."
michael@0:         sys.exit(-1)
michael@0: 
michael@0: def get_config_args(name, env_root):
michael@0:     local_json = os.path.join(env_root, "local.json")
michael@0:     if not (os.path.exists(local_json) and
michael@0:             os.path.isfile(local_json)):
michael@0:         if name == "default":
michael@0:             return []
michael@0:         else:
michael@0:             print >>sys.stderr, "File does not exist: %s" % local_json
michael@0:             sys.exit(1)
michael@0:     local_json = packaging.load_json_file(local_json)
michael@0:     if 'configs' not in local_json:
michael@0:         print >>sys.stderr, "'configs' key not found in local.json."
michael@0:         sys.exit(1)
michael@0:     if name not in local_json.configs:
michael@0:         if name == "default":
michael@0:             return []
michael@0:         else:
michael@0:             print >>sys.stderr, "No config found for '%s'." % name
michael@0:             sys.exit(1)
michael@0:     config = local_json.configs[name]
michael@0:     if type(config) != list:
michael@0:         print >>sys.stderr, "Config for '%s' must be a list of strings." % name
michael@0:         sys.exit(1)
michael@0:     return config
michael@0: 
michael@0: def initializer(env_root, args, out=sys.stdout, err=sys.stderr):
michael@0:     from templates import PACKAGE_JSON, TEST_MAIN_JS
michael@0:     from preflight import create_jid
michael@0:     path = os.getcwd()
michael@0:     addon = os.path.basename(path)
michael@0:     # if more than two arguments
michael@0:     if len(args) > 2:
michael@0:         print >>err, 'Too many arguments.'
michael@0:         return {"result":1}
michael@0:     if len(args) == 2:
michael@0:         path = os.path.join(path,args[1])
michael@0:         try:
michael@0:             os.mkdir(path)
michael@0:             print >>out, '*', args[1], 'package directory created'
michael@0:         except OSError:
michael@0:             print >>out, '*', args[1], 'already exists, testing if directory is empty'
michael@0:     # avoid clobbering existing files, but we tolerate things like .git
michael@0:     existing = [fn for fn in os.listdir(path) if not fn.startswith(".")]
michael@0:     if existing:
michael@0:         print >>err, 'This command must be run in an empty directory.'
michael@0:         return {"result":1}
michael@0:     for d in ['lib','data','test']:
michael@0:         os.mkdir(os.path.join(path,d))
michael@0:         print >>out, '*', d, 'directory created'
michael@0:     jid = create_jid()
michael@0:     print >>out, '* generated jID automatically:', jid
michael@0:     open(os.path.join(path,'package.json'),'w').write(PACKAGE_JSON % {'name':addon.lower(),
michael@0:                                                    'title':addon,
michael@0:                                                    'id':jid })
michael@0:     print >>out, '* package.json written'
michael@0:     open(os.path.join(path,'test','test-main.js'),'w').write(TEST_MAIN_JS)
michael@0:     print >>out, '* test/test-main.js written'
michael@0:     open(os.path.join(path,'lib','main.js'),'w').write('')
michael@0:     print >>out, '* lib/main.js written'
michael@0:     if len(args) == 1:
michael@0:         print >>out, '\nYour sample add-on is now ready.'
michael@0:         print >>out, 'Do "cfx test" to test it and "cfx run" to try it.  Have fun!'
michael@0:     else:
michael@0:         print >>out, '\nYour sample add-on is now ready in the \'' + args[1] +  '\' directory.'
michael@0:         print >>out, 'Change to that directory, then do "cfx test" to test it, \nand "cfx run" to try it.  Have fun!'
michael@0:     return {"result":0, "jid":jid}
michael@0: 
michael@0: def buildJID(target_cfg):
michael@0:     if "id" in target_cfg:
michael@0:         jid = target_cfg["id"]
michael@0:     else:
michael@0:         import uuid
michael@0:         jid = str(uuid.uuid4())
michael@0:     if not ("@" in jid or jid.startswith("{")):
michael@0:         jid = jid + "@jetpack"
michael@0:     return jid
michael@0: 
michael@0: def run(arguments=sys.argv[1:], target_cfg=None, pkg_cfg=None,
michael@0:         defaults=None, env_root=os.environ.get('CUDDLEFISH_ROOT'),
michael@0:         stdout=sys.stdout):
michael@0:     versions = get_versions()
michael@0:     sdk_version = versions["version"]
michael@0:     display_version = "Add-on SDK %s (%s)" % (sdk_version, versions["full"])
michael@0:     parser_kwargs = dict(arguments=arguments,
michael@0:                          global_options=global_options,
michael@0:                          parser_groups=parser_groups,
michael@0:                          usage=usage,
michael@0:                          version=display_version,
michael@0:                          defaults=defaults)
michael@0: 
michael@0:     (options, args) = parse_args(**parser_kwargs)
michael@0: 
michael@0:     config_args = get_config_args(options.config, env_root);
michael@0: 
michael@0:     # reparse configs with arguments from local.json
michael@0:     if config_args:
michael@0:         parser_kwargs['arguments'] += config_args
michael@0:         (options, args) = parse_args(**parser_kwargs)
michael@0: 
michael@0:     command = args[0]
michael@0: 
michael@0:     if command == "init":
michael@0:         initializer(env_root, args)
michael@0:         return
michael@0:     if command == "testpkgs":
michael@0:         test_all_packages(env_root, defaults=options.__dict__)
michael@0:         return
michael@0:     elif command == "testaddons":
michael@0:         test_all_testaddons(env_root, defaults=options.__dict__)
michael@0:         return
michael@0:     elif command == "testex":
michael@0:         test_all_examples(env_root, defaults=options.__dict__)
michael@0:         return
michael@0:     elif command == "testall":
michael@0:         test_all(env_root, defaults=options.__dict__)
michael@0:         return
michael@0:     elif command == "testcfx":
michael@0:         if options.filter:
michael@0:             print >>sys.stderr, "The filter option is not valid with the testcfx command"
michael@0:             return
michael@0:         test_cfx(env_root, options.verbose)
michael@0:         return
michael@0:     elif command not in ["xpi", "test", "run"]:
michael@0:         print >>sys.stderr, "Unknown command: %s" % command
michael@0:         print >>sys.stderr, "Try using '--help' for assistance."
michael@0:         sys.exit(1)
michael@0: 
michael@0:     target_cfg_json = None
michael@0:     if not target_cfg:
michael@0:         if not options.pkgdir:
michael@0:             options.pkgdir = find_parent_package(os.getcwd())
michael@0:             if not options.pkgdir:
michael@0:                 print >>sys.stderr, ("cannot find 'package.json' in the"
michael@0:                                      " current directory or any parent.")
michael@0:                 sys.exit(1)
michael@0:         else:
michael@0:             options.pkgdir = os.path.abspath(options.pkgdir)
michael@0:         if not os.path.exists(os.path.join(options.pkgdir, 'package.json')):
michael@0:             print >>sys.stderr, ("cannot find 'package.json' in"
michael@0:                                  " %s." % options.pkgdir)
michael@0:             sys.exit(1)
michael@0: 
michael@0:         target_cfg_json = os.path.join(options.pkgdir, 'package.json')
michael@0:         target_cfg = packaging.get_config_in_dir(options.pkgdir)
michael@0: 
michael@0:     if options.manifest_overload:
michael@0:         for k, v in packaging.load_json_file(options.manifest_overload).items():
michael@0:             target_cfg[k] = v
michael@0: 
michael@0:     # At this point, we're either building an XPI or running Jetpack code in
michael@0:     # a Mozilla application (which includes running tests).
michael@0: 
michael@0:     use_main = False
michael@0:     inherited_options = ['verbose', 'enable_e10s', 'parseable', 'check_memory']
michael@0:     enforce_timeouts = False
michael@0: 
michael@0:     if command == "xpi":
michael@0:         use_main = True
michael@0:     elif command == "test":
michael@0:         if 'tests' not in target_cfg:
michael@0:             target_cfg['tests'] = []
michael@0:         inherited_options.extend(['iterations', 'filter', 'profileMemory',
michael@0:                                   'stopOnError'])
michael@0:         enforce_timeouts = True
michael@0:     elif command == "run":
michael@0:         use_main = True
michael@0:     else:
michael@0:         assert 0, "shouldn't get here"
michael@0: 
michael@0:     if use_main and 'main' not in target_cfg:
michael@0:         # If the user supplies a template dir, then the main
michael@0:         # program may be contained in the template.
michael@0:         if not options.templatedir:
michael@0:             print >>sys.stderr, "package.json does not have a 'main' entry."
michael@0:             sys.exit(1)
michael@0: 
michael@0:     if not pkg_cfg:
michael@0:         pkg_cfg = packaging.build_config(env_root, target_cfg, options.packagepath)
michael@0: 
michael@0:     target = target_cfg.name
michael@0: 
michael@0:     # TODO: Consider keeping a cache of dynamic UUIDs, based
michael@0:     # on absolute filesystem pathname, in the root directory
michael@0:     # or something.
michael@0:     if command in ('xpi', 'run'):
michael@0:         from cuddlefish.preflight import preflight_config
michael@0:         if target_cfg_json:
michael@0:             config_was_ok, modified = preflight_config(target_cfg,
michael@0:                                                        target_cfg_json)
michael@0:             if not config_was_ok:
michael@0:                 if modified:
michael@0:                     # we need to re-read package.json . The safest approach
michael@0:                     # is to re-run the "cfx xpi"/"cfx run" command.
michael@0:                     print >>sys.stderr, ("package.json modified: please re-run"
michael@0:                                          " 'cfx %s'" % command)
michael@0:                 else:
michael@0:                     print >>sys.stderr, ("package.json needs modification:"
michael@0:                                          " please update it and then re-run"
michael@0:                                          " 'cfx %s'" % command)
michael@0:                 sys.exit(1)
michael@0:         # if we make it this far, we have a JID
michael@0:     else:
michael@0:         assert command == "test"
michael@0: 
michael@0:     jid = buildJID(target_cfg)
michael@0: 
michael@0:     targets = [target]
michael@0:     if command == "test":
michael@0:         targets.append(options.test_runner_pkg)
michael@0: 
michael@0:     extra_packages = []
michael@0:     if options.extra_packages:
michael@0:         extra_packages = options.extra_packages.split(",")
michael@0:     if extra_packages:
michael@0:         targets.extend(extra_packages)
michael@0:         target_cfg.extra_dependencies = extra_packages
michael@0: 
michael@0:     deps = packaging.get_deps_for_targets(pkg_cfg, targets)
michael@0: 
michael@0:     from cuddlefish.manifest import build_manifest, ModuleNotFoundError, \
michael@0:                                     BadChromeMarkerError
michael@0:     # Figure out what loader files should be scanned. This is normally
michael@0:     # computed inside packaging.generate_build_for_target(), by the first
michael@0:     # dependent package that defines a "loader" property in its package.json.
michael@0:     # This property is interpreted as a filename relative to the top of that
michael@0:     # file, and stored as a path in build.loader . generate_build_for_target()
michael@0:     # cannot be called yet (it needs the list of used_deps that
michael@0:     # build_manifest() computes, but build_manifest() needs the list of
michael@0:     # loader files that it computes). We could duplicate or factor out this
michael@0:     # build.loader logic, but that would be messy, so instead we hard-code
michael@0:     # the choice of loader for manifest-generation purposes. In practice,
michael@0:     # this means that alternative loaders probably won't work with
michael@0:     # --strip-xpi.
michael@0:     assert packaging.DEFAULT_LOADER == "addon-sdk"
michael@0:     assert pkg_cfg.packages["addon-sdk"].loader == "lib/sdk/loader/cuddlefish.js"
michael@0:     cuddlefish_js_path = os.path.join(pkg_cfg.packages["addon-sdk"].root_dir,
michael@0:                                       "lib", "sdk", "loader", "cuddlefish.js")
michael@0:     loader_modules = [("addon-sdk", "lib", "sdk/loader/cuddlefish", cuddlefish_js_path)]
michael@0:     scan_tests = command == "test"
michael@0:     test_filter_re = None
michael@0:     if scan_tests and options.filter:
michael@0:         test_filter_re = options.filter
michael@0:         if ":" in options.filter:
michael@0:             test_filter_re = options.filter.split(":")[0]
michael@0:     try:
michael@0:         manifest = build_manifest(target_cfg, pkg_cfg, deps,
michael@0:                                   scan_tests, test_filter_re,
michael@0:                                   loader_modules)
michael@0:     except ModuleNotFoundError, e:
michael@0:         print str(e)
michael@0:         sys.exit(1)
michael@0:     except BadChromeMarkerError, e:
michael@0:         # An error had already been displayed on stderr in manifest code
michael@0:         sys.exit(1)
michael@0:     used_deps = manifest.get_used_packages()
michael@0:     if command == "test":
michael@0:         # The test runner doesn't appear to link against any actual packages,
michael@0:         # because it loads everything at runtime (invisible to the linker).
michael@0:         # If we believe that, we won't set up URI mappings for anything, and
michael@0:         # tests won't be able to run.
michael@0:         used_deps = deps
michael@0:     for xp in extra_packages:
michael@0:         if xp not in used_deps:
michael@0:             used_deps.append(xp)
michael@0: 
michael@0:     build = packaging.generate_build_for_target(
michael@0:         pkg_cfg, target, used_deps,
michael@0:         include_dep_tests=options.dep_tests,
michael@0:         is_running_tests=(command == "test")
michael@0:         )
michael@0: 
michael@0:     harness_options = {
michael@0:         'jetpackID': jid,
michael@0:         'staticArgs': options.static_args,
michael@0:         'name': target,
michael@0:         }
michael@0: 
michael@0:     harness_options.update(build)
michael@0: 
michael@0:     # When cfx is run from sdk root directory, we will strip sdk modules and
michael@0:     # override them with local modules.
michael@0:     # So that integration tools will continue to work and use local modules
michael@0:     if os.getcwd() == env_root:
michael@0:         options.bundle_sdk = True
michael@0:         options.force_use_bundled_sdk = False
michael@0:         options.overload_modules = True
michael@0: 
michael@0:     extra_environment = {}
michael@0:     if command == "test":
michael@0:         # This should be contained in the test runner package.
michael@0:         # maybe just do: target_cfg.main = 'test-harness/run-tests'
michael@0:         harness_options['main'] = 'sdk/test/runner'
michael@0:         harness_options['mainPath'] = 'sdk/test/runner'
michael@0:     else:
michael@0:         harness_options['main'] = target_cfg.get('main')
michael@0:         harness_options['mainPath'] = manifest.top_path
michael@0:     extra_environment["CFX_COMMAND"] = command
michael@0: 
michael@0:     for option in inherited_options:
michael@0:         harness_options[option] = getattr(options, option)
michael@0: 
michael@0:     harness_options['metadata'] = packaging.get_metadata(pkg_cfg, used_deps)
michael@0: 
michael@0:     harness_options['sdkVersion'] = sdk_version
michael@0: 
michael@0:     packaging.call_plugins(pkg_cfg, used_deps)
michael@0: 
michael@0:     retval = 0
michael@0: 
michael@0:     if options.templatedir:
michael@0:         app_extension_dir = os.path.abspath(options.templatedir)
michael@0:     elif os.path.exists(os.path.join(options.pkgdir, "app-extension")):
michael@0:       app_extension_dir = os.path.join(options.pkgdir, "app-extension")
michael@0:     else:
michael@0:         mydir = os.path.dirname(os.path.abspath(__file__))
michael@0:         app_extension_dir = os.path.join(mydir, "../../app-extension")
michael@0: 
michael@0:     if target_cfg.get('preferences'):
michael@0:         harness_options['preferences'] = target_cfg.get('preferences')
michael@0: 
michael@0:     # Do not add entries for SDK modules
michael@0:     harness_options['manifest'] = manifest.get_harness_options_manifest(False)
michael@0: 
michael@0:     # Gives an hint to tell if sdk modules are bundled or not
michael@0:     harness_options['is-sdk-bundled'] = options.bundle_sdk or options.no_strip_xpi
michael@0: 
michael@0:     if options.force_use_bundled_sdk:
michael@0:         if not harness_options['is-sdk-bundled']:
michael@0:             print >>sys.stderr, ("--force-use-bundled-sdk "
michael@0:                                  "can't be used if sdk isn't bundled.")
michael@0:             sys.exit(1)
michael@0:         if options.overload_modules:
michael@0:             print >>sys.stderr, ("--force-use-bundled-sdk and --overload-modules "
michael@0:                                  "can't be used at the same time.")
michael@0:             sys.exit(1)
michael@0:         # Pass a flag in order to force using sdk modules shipped in the xpi
michael@0:         harness_options['force-use-bundled-sdk'] = True
michael@0: 
michael@0:     # Pass the list of absolute path for all test modules
michael@0:     if command == "test":
michael@0:         harness_options['allTestModules'] = manifest.get_all_test_modules()
michael@0:         if len(harness_options['allTestModules']) == 0:
michael@0:             sys.exit(0)
michael@0: 
michael@0:     from cuddlefish.rdf import gen_manifest, RDFUpdate
michael@0: 
michael@0:     manifest_rdf = gen_manifest(template_root_dir=app_extension_dir,
michael@0:                                 target_cfg=target_cfg,
michael@0:                                 jid=jid,
michael@0:                                 update_url=options.update_url,
michael@0:                                 bootstrap=True,
michael@0:                                 enable_mobile=options.enable_mobile)
michael@0: 
michael@0:     if command == "xpi" and options.update_link:
michael@0:         if not options.update_link.startswith("https"):
michael@0:             raise optparse.OptionValueError("--update-link must start with 'https': %s" % options.update_link)
michael@0:         rdf_name = UPDATE_RDF_FILENAME % target_cfg.name
michael@0:         print >>stdout, "Exporting update description to %s." % rdf_name
michael@0:         update = RDFUpdate()
michael@0:         update.add(manifest_rdf, options.update_link)
michael@0:         open(rdf_name, "w").write(str(update))
michael@0: 
michael@0:     # ask the manifest what files were used, so we can construct an XPI
michael@0:     # without the rest. This will include the loader (and everything it
michael@0:     # uses) because of the "loader_modules" starting points we passed to
michael@0:     # build_manifest earlier
michael@0:     used_files = None
michael@0:     if command == "xpi":
michael@0:         used_files = set(manifest.get_used_files(options.bundle_sdk))
michael@0: 
michael@0:     if options.no_strip_xpi:
michael@0:         used_files = None # disables the filter, includes all files
michael@0: 
michael@0:     if command == 'xpi':
michael@0:         from cuddlefish.xpi import build_xpi
michael@0:         # Generate extra options
michael@0:         extra_harness_options = {}
michael@0:         for kv in options.extra_harness_option_args:
michael@0:             key,value = kv.split("=", 1)
michael@0:             extra_harness_options[key] = value
michael@0:         # Generate xpi filepath
michael@0:         if options.output_file:
michael@0:           xpi_path = options.output_file
michael@0:         else:
michael@0:           xpi_path = XPI_FILENAME % target_cfg.name
michael@0: 
michael@0:         print >>stdout, "Exporting extension to %s." % xpi_path
michael@0:         build_xpi(template_root_dir=app_extension_dir,
michael@0:                   manifest=manifest_rdf,
michael@0:                   xpi_path=xpi_path,
michael@0:                   harness_options=harness_options,
michael@0:                   limit_to=used_files,
michael@0:                   extra_harness_options=extra_harness_options,
michael@0:                   bundle_sdk=True,
michael@0:                   pkgdir=options.pkgdir)
michael@0:     else:
michael@0:         from cuddlefish.runner import run_app
michael@0: 
michael@0:         if options.profiledir:
michael@0:             options.profiledir = os.path.expanduser(options.profiledir)
michael@0:             options.profiledir = os.path.abspath(options.profiledir)
michael@0: 
michael@0:         if options.addons is not None:
michael@0:             options.addons = options.addons.split(",")
michael@0: 
michael@0:         try:
michael@0:             retval = run_app(harness_root_dir=app_extension_dir,
michael@0:                              manifest_rdf=manifest_rdf,
michael@0:                              harness_options=harness_options,
michael@0:                              app_type=options.app,
michael@0:                              binary=options.binary,
michael@0:                              profiledir=options.profiledir,
michael@0:                              verbose=options.verbose,
michael@0:                              parseable=options.parseable,
michael@0:                              enforce_timeouts=enforce_timeouts,
michael@0:                              logfile=options.logfile,
michael@0:                              addons=options.addons,
michael@0:                              args=options.cmdargs,
michael@0:                              extra_environment=extra_environment,
michael@0:                              norun=options.no_run,
michael@0:                              used_files=used_files,
michael@0:                              enable_mobile=options.enable_mobile,
michael@0:                              mobile_app_name=options.mobile_app_name,
michael@0:                              env_root=env_root,
michael@0:                              is_running_tests=(command == "test"),
michael@0:                              overload_modules=options.overload_modules,
michael@0:                              bundle_sdk=options.bundle_sdk,
michael@0:                              pkgdir=options.pkgdir)
michael@0:         except ValueError, e:
michael@0:             print ""
michael@0:             print "A given cfx option has an inappropriate value:"
michael@0:             print >>sys.stderr, "  " + "  \n  ".join(str(e).split("\n"))
michael@0:             retval = -1
michael@0:         except Exception, e:
michael@0:             if str(e).startswith(MOZRUNNER_BIN_NOT_FOUND):
michael@0:                 print >>sys.stderr, MOZRUNNER_BIN_NOT_FOUND_HELP.strip()
michael@0:                 retval = -1
michael@0:             else:
michael@0:                 raise
michael@0:     sys.exit(retval)