The Tor Browser: build/util/hg.py@b8a032363ba2

Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6

     1 """Functions for interacting with hg"""

     2 import os

     3 import re

     4 import subprocess

     5 from urlparse import urlsplit

     6 from ConfigParser import RawConfigParser

     8 from util.commands import run_cmd, get_output, remove_path

     9 from util.retry import retry

    11 import logging

    12 log = logging.getLogger(__name__)

    15 class DefaultShareBase:

    16     pass

    17 DefaultShareBase = DefaultShareBase()

    20 class HgUtilError(Exception):

    21     pass

    24 def _make_absolute(repo):

    25     if repo.startswith("file://"):

    26         path = repo[len("file://"):]

    27         repo = "file://%s" % os.path.abspath(path)

    28     elif "://" not in repo:

    29         repo = os.path.abspath(repo)

    30     return repo

    33 def make_hg_url(hgHost, repoPath, protocol='https', revision=None,

    34                 filename=None):

    35     """construct a valid hg url from a base hg url (hg.mozilla.org),

    36     repoPath, revision and possible filename"""

    37     base = '%s://%s' % (protocol, hgHost)

    38     repo = '/'.join(p.strip('/') for p in [base, repoPath])

    39     if not filename:

    40         if not revision:

    41             return repo

    42         else:

    43             return '/'.join([p.strip('/') for p in [repo, 'rev', revision]])

    44     else:

    45         assert revision

    46         return '/'.join([p.strip('/') for p in [repo, 'raw-file', revision, filename]])

    49 def get_repo_name(repo):

    50     return repo.rstrip('/').split('/')[-1]

    53 def get_repo_path(repo):

    54     repo = _make_absolute(repo)

    55     if repo.startswith("/"):

    56         return repo.lstrip("/")

    57     else:

    58         return urlsplit(repo).path.lstrip("/")

    61 def get_revision(path):

    62     """Returns which revision directory `path` currently has checked out."""

    63     return get_output(['hg', 'parent', '--template', '{node|short}'], cwd=path)

    66 def get_branch(path):

    67     return get_output(['hg', 'branch'], cwd=path).strip()

    70 def get_branches(path):

    71     branches = []

    72     for line in get_output(['hg', 'branches', '-c'], cwd=path).splitlines():

    73         branches.append(line.split()[0])

    74     return branches

    77 def is_hg_cset(rev):

    78     """Retruns True if passed revision represents a valid HG revision

    79     (long or short(er) 40 bit hex)"""

    80     try:

    81         int(rev, 16)

    82         return True

    83     except (TypeError, ValueError):

    84         return False

    87 def hg_ver():

    88     """Returns the current version of hg, as a tuple of

    89     (major, minor, build)"""

    90     ver_string = get_output(['hg', '-q', 'version'])

    91     match = re.search("\(version ([0-9.]+)\)", ver_string)

    92     if match:

    93         bits = match.group(1).split(".")

    94         if len(bits) < 3:

    95             bits += (0,)

    96         ver = tuple(int(b) for b in bits)

    97     else:

    98         ver = (0, 0, 0)

    99     log.debug("Running hg version %s", ver)

   100     return ver

   103 def purge(dest):

   104     """Purge the repository of all untracked and ignored files."""

   105     try:

   106         run_cmd(['hg', '--config', 'extensions.purge=', 'purge',

   107                  '-a', '--all', dest], cwd=dest)

   108     except subprocess.CalledProcessError, e:

   109         log.debug('purge failed: %s' % e)

   110         raise

   113 def update(dest, branch=None, revision=None):

   114     """Updates working copy `dest` to `branch` or `revision`.  If neither is

   115     set then the working copy will be updated to the latest revision on the

   116     current branch.  Local changes will be discarded."""

   117     # If we have a revision, switch to that

   118     if revision is not None:

   119         cmd = ['hg', 'update', '-C', '-r', revision]

   120         run_cmd(cmd, cwd=dest)

   121     else:

   122         # Check & switch branch

   123         local_branch = get_output(['hg', 'branch'], cwd=dest).strip()

   125         cmd = ['hg', 'update', '-C']

   127         # If this is different, checkout the other branch

   128         if branch and branch != local_branch:

   129             cmd.append(branch)

   131         run_cmd(cmd, cwd=dest)

   132     return get_revision(dest)

   135 def clone(repo, dest, branch=None, revision=None, update_dest=True,

   136           clone_by_rev=False, mirrors=None, bundles=None):

   137     """Clones hg repo and places it at `dest`, replacing whatever else is

   138     there.  The working copy will be empty.

   140     If `revision` is set, only the specified revision and its ancestors will

   141     be cloned.

   143     If `update_dest` is set, then `dest` will be updated to `revision` if

   144     set, otherwise to `branch`, otherwise to the head of default.

   146     If `mirrors` is set, will try and clone from the mirrors before

   147     cloning from `repo`.

   149     If `bundles` is set, will try and download the bundle first and

   150     unbundle it. If successful, will pull in new revisions from mirrors or

   151     the master repo. If unbundling fails, will fall back to doing a regular

   152     clone from mirrors or the master repo.

   154     Regardless of how the repository ends up being cloned, the 'default' path

   155     will point to `repo`.

   156     """

   157     if os.path.exists(dest):

   158         remove_path(dest)

   160     if bundles:

   161         log.info("Attempting to initialize clone with bundles")

   162         for bundle in bundles:

   163             if os.path.exists(dest):

   164                 remove_path(dest)

   165             init(dest)

   166             log.info("Trying to use bundle %s", bundle)

   167             try:

   168                 if not unbundle(bundle, dest):

   169                     remove_path(dest)

   170                     continue

   171                 adjust_paths(dest, default=repo)

   172                 # Now pull / update

   173                 return pull(repo, dest, update_dest=update_dest,

   174                             mirrors=mirrors, revision=revision, branch=branch)

   175             except Exception:

   176                 remove_path(dest)

   177                 log.exception("Problem unbundling/pulling from %s", bundle)

   178                 continue

   179         else:

   180             log.info("Using bundles failed; falling back to clone")

   182     if mirrors:

   183         log.info("Attempting to clone from mirrors")

   184         for mirror in mirrors:

   185             log.info("Cloning from %s", mirror)

   186             try:

   187                 retval = clone(mirror, dest, branch, revision,

   188                                update_dest=update_dest, clone_by_rev=clone_by_rev)

   189                 adjust_paths(dest, default=repo)

   190                 return retval

   191             except:

   192                 log.exception("Problem cloning from mirror %s", mirror)

   193                 continue

   194         else:

   195             log.info("Pulling from mirrors failed; falling back to %s", repo)

   196             # We may have a partial repo here; mercurial() copes with that

   197             # We need to make sure our paths are correct though

   198             if os.path.exists(os.path.join(dest, '.hg')):

   199                 adjust_paths(dest, default=repo)

   200             return mercurial(repo, dest, branch, revision, autoPurge=True,

   201                              update_dest=update_dest, clone_by_rev=clone_by_rev)

   203     cmd = ['hg', 'clone']

   204     if not update_dest:

   205         cmd.append('-U')

   207     if clone_by_rev:

   208         if revision:

   209             cmd.extend(['-r', revision])

   210         elif branch:

   211             # hg >= 1.6 supports -b branch for cloning

   212             ver = hg_ver()

   213             if ver >= (1, 6, 0):

   214                 cmd.extend(['-b', branch])

   216     cmd.extend([repo, dest])

   217     run_cmd(cmd)

   219     if update_dest:

   220         return update(dest, branch, revision)

   223 def common_args(revision=None, branch=None, ssh_username=None, ssh_key=None):

   224     """Fill in common hg arguments, encapsulating logic checks that depend on

   225        mercurial versions and provided arguments"""

   226     args = []

   227     if ssh_username or ssh_key:

   228         opt = ['-e', 'ssh']

   229         if ssh_username:

   230             opt[1] += ' -l %s' % ssh_username

   231         if ssh_key:

   232             opt[1] += ' -i %s' % ssh_key

   233         args.extend(opt)

   234     if revision:

   235         args.extend(['-r', revision])

   236     elif branch:

   237         if hg_ver() >= (1, 6, 0):

   238             args.extend(['-b', branch])

   239     return args

   242 def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):

   243     """Pulls changes from hg repo and places it in `dest`.

   245     If `update_dest` is set, then `dest` will be updated to `revision` if

   246     set, otherwise to `branch`, otherwise to the head of default.

   248     If `mirrors` is set, will try and pull from the mirrors first before

   249     `repo`."""

   251     if mirrors:

   252         for mirror in mirrors:

   253             try:

   254                 return pull(mirror, dest, update_dest=update_dest, **kwargs)

   255             except:

   256                 log.exception("Problem pulling from mirror %s", mirror)

   257                 continue

   258         else:

   259             log.info("Pulling from mirrors failed; falling back to %s", repo)

   261     # Convert repo to an absolute path if it's a local repository

   262     repo = _make_absolute(repo)

   263     cmd = ['hg', 'pull']

   264     # Don't pass -r to "hg pull", except when it's a valid HG revision.

   265     # Pulling using tag names is dangerous: it uses the local .hgtags, so if

   266     # the tag has moved on the remote side you won't pull the new revision the

   267     # remote tag refers to.

   268     pull_kwargs = kwargs.copy()

   269     if 'revision' in pull_kwargs and \

   270        not is_hg_cset(pull_kwargs['revision']):

   271         del pull_kwargs['revision']

   273     cmd.extend(common_args(**pull_kwargs))

   275     cmd.append(repo)

   276     run_cmd(cmd, cwd=dest)

   278     if update_dest:

   279         branch = None

   280         if 'branch' in kwargs and kwargs['branch']:

   281             branch = kwargs['branch']

   282         revision = None

   283         if 'revision' in kwargs and kwargs['revision']:

   284             revision = kwargs['revision']

   285         return update(dest, branch=branch, revision=revision)

   287 # Defines the places of attributes in the tuples returned by `out'

   288 REVISION, BRANCH = 0, 1

   291 def out(src, remote, **kwargs):

   292     """Check for outgoing changesets present in a repo"""

   293     cmd = ['hg', '-q', 'out', '--template', '{node} {branches}\n']

   294     cmd.extend(common_args(**kwargs))

   295     cmd.append(remote)

   296     if os.path.exists(src):

   297         try:

   298             revs = []

   299             for line in get_output(cmd, cwd=src).rstrip().split("\n"):

   300                 try:

   301                     rev, branch = line.split()

   302                 # Mercurial displays no branch at all if the revision is on

   303                 # "default"

   304                 except ValueError:

   305                     rev = line.rstrip()

   306                     branch = "default"

   307                 revs.append((rev, branch))

   308             return revs

   309         except subprocess.CalledProcessError, inst:

   310             # In some situations, some versions of Mercurial return "1"

   311             # if no changes are found, so we need to ignore this return code

   312             if inst.returncode == 1:

   313                 return []

   314             raise

   317 def push(src, remote, push_new_branches=True, force=False, **kwargs):

   318     cmd = ['hg', 'push']

   319     cmd.extend(common_args(**kwargs))

   320     if force:

   321         cmd.append('-f')

   322     if push_new_branches:

   323         cmd.append('--new-branch')

   324     cmd.append(remote)

   325     run_cmd(cmd, cwd=src)

   328 def mercurial(repo, dest, branch=None, revision=None, update_dest=True,

   329               shareBase=DefaultShareBase, allowUnsharedLocalClones=False,

   330               clone_by_rev=False, mirrors=None, bundles=None, autoPurge=False):

   331     """Makes sure that `dest` is has `revision` or `branch` checked out from

   332     `repo`.

   334     Do what it takes to make that happen, including possibly clobbering

   335     dest.

   337     If allowUnsharedLocalClones is True and we're trying to use the share

   338     extension but fail, then we will be able to clone from the shared repo to

   339     our destination.  If this is False, the default, then if we don't have the

   340     share extension we will just clone from the remote repository.

   342     If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'.

   343     This is slower, but useful when cloning repos with lots of heads.

   345     If `mirrors` is set, will try and use the mirrors before `repo`.

   347     If `bundles` is set, will try and download the bundle first and

   348     unbundle it instead of doing a full clone. If successful, will pull in

   349     new revisions from mirrors or the master repo. If unbundling fails, will

   350     fall back to doing a regular clone from mirrors or the master repo.

   351     """

   352     dest = os.path.abspath(dest)

   353     if shareBase is DefaultShareBase:

   354         shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)

   356     log.info("Reporting hg version in use")

   357     cmd = ['hg', '-q', 'version']

   358     run_cmd(cmd, cwd='.')

   360     if shareBase:

   361         # Check that 'hg share' works

   362         try:

   363             log.info("Checking if share extension works")

   364             output = get_output(['hg', 'help', 'share'], dont_log=True)

   365             if 'no commands defined' in output:

   366                 # Share extension is enabled, but not functional

   367                 log.info("Disabling sharing since share extension doesn't seem to work (1)")

   368                 shareBase = None

   369             elif 'unknown command' in output:

   370                 # Share extension is disabled

   371                 log.info("Disabling sharing since share extension doesn't seem to work (2)")

   372                 shareBase = None

   373         except subprocess.CalledProcessError:

   374             # The command failed, so disable sharing

   375             log.info("Disabling sharing since share extension doesn't seem to work (3)")

   376             shareBase = None

   378     # Check that our default path is correct

   379     if os.path.exists(os.path.join(dest, '.hg')):

   380         hgpath = path(dest, "default")

   382         # Make sure that our default path is correct

   383         if hgpath != _make_absolute(repo):

   384             log.info("hg path isn't correct (%s should be %s); clobbering",

   385                      hgpath, _make_absolute(repo))

   386             remove_path(dest)

   388     # If the working directory already exists and isn't using share we update

   389     # the working directory directly from the repo, ignoring the sharing

   390     # settings

   391     if os.path.exists(dest):

   392         if not os.path.exists(os.path.join(dest, ".hg")):

   393             log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest)

   394             remove_path(dest)

   395         elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):

   396             try:

   397                 if autoPurge:

   398                     purge(dest)

   399                 return pull(repo, dest, update_dest=update_dest, branch=branch,

   400                             revision=revision,

   401                             mirrors=mirrors)

   402             except subprocess.CalledProcessError:

   403                 log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)

   404                 log.debug("Exception:", exc_info=True)

   405                 remove_path(dest)

   407     # If that fails for any reason, and sharing is requested, we'll try to

   408     # update the shared repository, and then update the working directory from

   409     # that.

   410     if shareBase:

   411         sharedRepo = os.path.join(shareBase, get_repo_path(repo))

   412         dest_sharedPath = os.path.join(dest, '.hg', 'sharedpath')

   414         if os.path.exists(sharedRepo):

   415             hgpath = path(sharedRepo, "default")

   417             # Make sure that our default path is correct

   418             if hgpath != _make_absolute(repo):

   419                 log.info("hg path isn't correct (%s should be %s); clobbering",

   420                          hgpath, _make_absolute(repo))

   421                 # we need to clobber both the shared checkout and the dest,

   422                 # since hgrc needs to be in both places

   423                 remove_path(sharedRepo)

   424                 remove_path(dest)

   426         if os.path.exists(dest_sharedPath):

   427             # Make sure that the sharedpath points to sharedRepo

   428             dest_sharedPath_data = os.path.normpath(

   429                 open(dest_sharedPath).read())

   430             norm_sharedRepo = os.path.normpath(os.path.join(sharedRepo, '.hg'))

   431             if dest_sharedPath_data != norm_sharedRepo:

   432                 # Clobber!

   433                 log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering",

   434                          dest_sharedPath_data, repo, norm_sharedRepo)

   435                 remove_path(dest)

   437         try:

   438             log.info("Updating shared repo")

   439             mercurial(repo, sharedRepo, branch=branch, revision=revision,

   440                       update_dest=False, shareBase=None, clone_by_rev=clone_by_rev,

   441                       mirrors=mirrors, bundles=bundles, autoPurge=False)

   442             if os.path.exists(dest):

   443                 if autoPurge:

   444                     purge(dest)

   445                 return update(dest, branch=branch, revision=revision)

   447             try:

   448                 log.info("Trying to share %s to %s", sharedRepo, dest)

   449                 return share(sharedRepo, dest, branch=branch, revision=revision)

   450             except subprocess.CalledProcessError:

   451                 if not allowUnsharedLocalClones:

   452                     # Re-raise the exception so it gets caught below.

   453                     # We'll then clobber dest, and clone from original repo

   454                     raise

   456                 log.warning("Error calling hg share from %s to %s;"

   457                             "falling back to normal clone from shared repo",

   458                             sharedRepo, dest)

   459                 # Do a full local clone first, and then update to the

   460                 # revision we want

   461                 # This lets us use hardlinks for the local clone if the OS

   462                 # supports it

   463                 clone(sharedRepo, dest, update_dest=False,

   464                       mirrors=mirrors, bundles=bundles)

   465                 return update(dest, branch=branch, revision=revision)

   466         except subprocess.CalledProcessError:

   467             log.warning(

   468                 "Error updating %s from sharedRepo (%s): ", dest, sharedRepo)

   469             log.debug("Exception:", exc_info=True)

   470             remove_path(dest)

   471     # end if shareBase

   473     if not os.path.exists(os.path.dirname(dest)):

   474         os.makedirs(os.path.dirname(dest))

   476     # Share isn't available or has failed, clone directly from the source

   477     return clone(repo, dest, branch, revision,

   478                  update_dest=update_dest, mirrors=mirrors,

   479                  bundles=bundles, clone_by_rev=clone_by_rev)

   482 def apply_and_push(localrepo, remote, changer, max_attempts=10,

   483                    ssh_username=None, ssh_key=None, force=False):

   484     """This function calls `changer' to make changes to the repo, and tries

   485        its hardest to get them to the origin repo. `changer' must be a

   486        callable object that receives two arguments: the directory of the local

   487        repository, and the attempt number. This function will push ALL

   488        changesets missing from remote."""

   489     assert callable(changer)

   490     branch = get_branch(localrepo)

   491     changer(localrepo, 1)

   492     for n in range(1, max_attempts + 1):

   493         new_revs = []

   494         try:

   495             new_revs = out(src=localrepo, remote=remote,

   496                            ssh_username=ssh_username,

   497                            ssh_key=ssh_key)

   498             if len(new_revs) < 1:

   499                 raise HgUtilError("No revs to push")

   500             push(src=localrepo, remote=remote, ssh_username=ssh_username,

   501                  ssh_key=ssh_key, force=force)

   502             return

   503         except subprocess.CalledProcessError, e:

   504             log.debug("Hit error when trying to push: %s" % str(e))

   505             if n == max_attempts:

   506                 log.debug("Tried %d times, giving up" % max_attempts)

   507                 for r in reversed(new_revs):

   508                     run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',

   509                              r[REVISION]], cwd=localrepo)

   510                 raise HgUtilError("Failed to push")

   511             pull(remote, localrepo, update_dest=False,

   512                  ssh_username=ssh_username, ssh_key=ssh_key)

   513             # After we successfully rebase or strip away heads the push is

   514             # is attempted again at the start of the loop

   515             try:

   516                 run_cmd(['hg', '--config', 'ui.merge=internal:merge',

   517                          'rebase'], cwd=localrepo)

   518             except subprocess.CalledProcessError, e:

   519                 log.debug("Failed to rebase: %s" % str(e))

   520                 update(localrepo, branch=branch)

   521                 for r in reversed(new_revs):

   522                     run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',

   523                              r[REVISION]], cwd=localrepo)

   524                 changer(localrepo, n + 1)

   527 def share(source, dest, branch=None, revision=None):

   528     """Creates a new working directory in "dest" that shares history with

   529        "source" using Mercurial's share extension"""

   530     run_cmd(['hg', 'share', '-U', source, dest])

   531     return update(dest, branch=branch, revision=revision)

   534 def cleanOutgoingRevs(reponame, remote, username, sshKey):

   535     outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,

   536                                           ssh_username=username,

   537                                           ssh_key=sshKey))

   538     for r in reversed(outgoingRevs):

   539         run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',

   540                  r[REVISION]], cwd=reponame)

   543 def path(src, name='default'):

   544     """Returns the remote path associated with "name" """

   545     try:

   546         return get_output(['hg', 'path', name], cwd=src).strip()

   547     except subprocess.CalledProcessError:

   548         return None

   551 def init(dest):

   552     """Initializes an empty repo in `dest`"""

   553     run_cmd(['hg', 'init', dest])

   556 def unbundle(bundle, dest):

   557     """Unbundles the bundle located at `bundle` into `dest`.

   559     `bundle` can be a local file or remote url."""

   560     try:

   561         get_output(['hg', 'unbundle', bundle], cwd=dest, include_stderr=True)

   562         return True

   563     except subprocess.CalledProcessError:

   564         return False

   567 def adjust_paths(dest, **paths):

   568     """Adjusts paths in `dest`/.hg/hgrc so that names in `paths` are set to

   569     paths[name].

   571     Note that any comments in the hgrc will be lost if changes are made to the

   572     file."""

   573     hgrc = os.path.join(dest, '.hg', 'hgrc')

   574     config = RawConfigParser()

   575     config.read(hgrc)

   577     if not config.has_section('paths'):

   578         config.add_section('paths')

   580     changed = False

   581     for path_name, path_value in paths.items():

   582         if (not config.has_option('paths', path_name) or

   583                 config.get('paths', path_name) != path_value):

   584             changed = True

   585             config.set('paths', path_name, path_value)

   587     if changed:

   588         config.write(open(hgrc, 'w'))

   591 def commit(dest, msg, user=None):

   592     cmd = ['hg', 'commit', '-m', msg]

   593     if user:

   594         cmd.extend(['-u', user])

   595     run_cmd(cmd, cwd=dest)

   596     return get_revision(dest)

   599 def tag(dest, tags, user=None, msg=None, rev=None, force=None):

   600     cmd = ['hg', 'tag']

   601     if user:

   602         cmd.extend(['-u', user])

   603     if msg:

   604         cmd.extend(['-m', msg])

   605     if rev:

   606         cmd.extend(['-r', rev])

   607     if force:

   608         cmd.append('-f')

   609     cmd.extend(tags)

   610     run_cmd(cmd, cwd=dest)

   611     return get_revision(dest)

The Tor Browser / file revision

build/util/hg.py@b8a032363ba2

build/util/hg.py