The Tor Browser: tools/mach_commands.py@97036ab72558

Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.

     1 # This Source Code Form is subject to the terms of the Mozilla Public

     2 # License, v. 2.0. If a copy of the MPL was not distributed with this

     3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.

     5 from __future__ import unicode_literals

     7 import sys

     8 import os

     9 import stat

    10 import platform

    11 import urllib2

    12 import errno

    14 from mach.decorators import (

    15     CommandArgument,

    16     CommandProvider,

    17     Command,

    18 )

    20 from mozbuild.base import MachCommandBase

    23 @CommandProvider

    24 class SearchProvider(object):

    25     @Command('mxr', category='misc',

    26         description='Search for something in MXR.')

    27     @CommandArgument('term', nargs='+', help='Term(s) to search for.')

    28     def mxr(self, term):

    29         import webbrowser

    30         term = ' '.join(term)

    31         uri = 'https://mxr.mozilla.org/mozilla-central/search?string=%s' % term

    32         webbrowser.open_new_tab(uri)

    34     @Command('dxr', category='misc',

    35         description='Search for something in DXR.')

    36     @CommandArgument('term', nargs='+', help='Term(s) to search for.')

    37     def dxr(self, term):

    38         import webbrowser

    39         term = ' '.join(term)

    40         uri = 'http://dxr.mozilla.org/search?tree=mozilla-central&q=%s' % term

    41         webbrowser.open_new_tab(uri)

    43     @Command('mdn', category='misc',

    44         description='Search for something on MDN.')

    45     @CommandArgument('term', nargs='+', help='Term(s) to search for.')

    46     def mdn(self, term):

    47         import webbrowser

    48         term = ' '.join(term)

    49         uri = 'https://developer.mozilla.org/search?q=%s' % term

    50         webbrowser.open_new_tab(uri)

    52     @Command('google', category='misc',

    53         description='Search for something on Google.')

    54     @CommandArgument('term', nargs='+', help='Term(s) to search for.')

    55     def google(self, term):

    56         import webbrowser

    57         term = ' '.join(term)

    58         uri = 'https://www.google.com/search?q=%s' % term

    59         webbrowser.open_new_tab(uri)

    61     @Command('search', category='misc',

    62         description='Search for something on the Internets. '

    63         'This will open 3 new browser tabs and search for the term on Google, '

    64         'MDN, and MXR.')

    65     @CommandArgument('term', nargs='+', help='Term(s) to search for.')

    66     def search(self, term):

    67         self.google(term)

    68         self.mdn(term)

    69         self.mxr(term)

    72 class Interface(object):

    73     '''

    74     Represents an XPIDL interface, in what file it is defined, what it derives

    75     from, what its uuid is, and where in the source file the uuid is.

    76     '''

    77     def __init__(self, filename, production):

    78         import xpidl

    79         assert isinstance(production, xpidl.Interface)

    80         self.name = production.name

    81         self.base = production.base

    82         self.filename = filename

    83         self.uuid = production.attributes.uuid

    84         location = production.location

    85         data = location._lexdata

    86         attr_pos = data.rfind(b'[', 0, location._lexpos)

    87         # uuid is always lowercase, but actual file content may not be.

    88         self.uuid_pos = data[attr_pos:location._lexpos].lower() \

    89                         .rfind(self.uuid) + attr_pos

    92 class InterfaceRegistry(object):

    93     '''

    94     Tracks XPIDL interfaces, and allow to search them by name and by the

    95     interface they derive from.

    96     '''

    97     def __init__(self):

    98         self.by_name = {}

    99         self.by_base = {}

   101     def get_by_name(self, name):

   102         return self.by_name.get(name, [])

   104     def get_by_base(self, base):

   105         return self.by_base.get(base, [])

   107     def add(self, interface):

   108         l = self.by_name.setdefault(interface.name, [])

   109         l.append(interface)

   110         l = self.by_base.setdefault(interface.base, [])

   111         l.append(interface)

   114 class IDLUpdater(object):

   115     '''

   116     Updates interfaces uuids in IDL files.

   117     '''

   118     def __init__(self, interfaces):

   119         from mozpack.copier import FileRegistry

   120         self.interfaces = interfaces;

   121         self.registry = FileRegistry()

   123     def add(self, name):

   124         for interface in self.interfaces.get_by_name(name):

   125             self._add(interface)

   127     def _add(self, interface):

   128         from mozpack.files import GeneratedFile

   129         from uuid import uuid4

   130         path = interface.filename

   131         if not self.registry.contains(path):

   132             self.registry.add(path, GeneratedFile(open(path).read()))

   133         content = self.registry[path].content

   134         content = content[:interface.uuid_pos] + str(uuid4()) + \

   135                   content[interface.uuid_pos + len(interface.uuid):]

   136         self.registry[path].content = content

   138         # Recurse through all the interfaces deriving from this one

   139         for derived in self.interfaces.get_by_base(interface.name):

   140             self._add(derived)

   142     def update(self):

   143         for p, f in self.registry:

   144             f.copy(p)

   147 @CommandProvider

   148 class UUIDProvider(object):

   149     @Command('uuid', category='misc',

   150         description='Generate a uuid.')

   151     @CommandArgument('--format', '-f', choices=['idl', 'cpp', 'c++'],

   152                      help='Output format for the generated uuid.')

   153     def uuid(self, format=None):

   154         import uuid

   155         u = uuid.uuid4()

   156         if format in [None, 'idl']:

   157             print(u)

   158             if format is None:

   159                 print('')

   160         if format in [None, 'cpp', 'c++']:

   161             u = u.hex

   162             print('{ 0x%s, 0x%s, 0x%s, \\' % (u[0:8], u[8:12], u[12:16]))

   163             pairs = tuple(map(lambda n: u[n:n+2], range(16, 32, 2)))

   164             print(('  { ' + '0x%s, ' * 7 + '0x%s } }') % pairs)

   166     @Command('update-uuids', category='misc',

   167         description='Update IDL files with new UUIDs.')

   168     @CommandArgument('--path', default='.',

   169                      help='Base path under which uuids will be searched.')

   170     @CommandArgument('interfaces', nargs='+',

   171                      help='Changed interfaces whose UUIDs need to be updated. ' +

   172                           'Their descendants are updated as well.')

   173     def update_uuids(self, path, interfaces):

   174         import os

   175         import xpidl

   176         from mozpack.files import FileFinder

   177         import mozpack.path

   178         from tempfile import mkdtemp

   180         finder = FileFinder(path, find_executables=False)

   181         # Avoid creating xpidllex and xpidlyacc in the current directory.

   182         tmpdir = mkdtemp()

   183         try:

   184             parser = xpidl.IDLParser(outputdir=tmpdir)

   185             registry = InterfaceRegistry()

   186             for p, f in finder.find('**/*.idl'):

   187                 p = mozpack.path.join(path, p)

   188                 try:

   189                     content = f.open().read()

   190                     idl = parser.parse(content, filename=p)

   191                 except Exception:

   192                     continue

   193                 for prod in idl.productions:

   194                     if isinstance(prod, xpidl.Interface):

   195                          registry.add(Interface(p, prod))

   196         finally:

   197             import shutil

   198             shutil.rmtree(tmpdir)

   200         updates = IDLUpdater(registry)

   202         for interface in interfaces:

   203             updates.add(interface)

   205         updates.update()

   207 @CommandProvider

   208 class PastebinProvider(object):

   209     @Command('pastebin', category='misc',

   210         description='Command line interface to pastebin.mozilla.org.')

   211     @CommandArgument('--language', default=None,

   212                      help='Language to use for syntax highlighting')

   213     @CommandArgument('--poster', default=None,

   214                      help='Specify your name for use with pastebin.mozilla.org')

   215     @CommandArgument('--duration', default='day',

   216                      choices=['d', 'day', 'm', 'month', 'f', 'forever'],

   217                      help='Keep for specified duration (default: %(default)s)')

   218     @CommandArgument('file', nargs='?', default=None,

   219                      help='Specify the file to upload to pastebin.mozilla.org')

   221     def pastebin(self, language, poster, duration, file):

   222         import sys

   223         import urllib

   225         URL = 'http://pastebin.mozilla.org/'

   227         FILE_TYPES = [{'value': 'text', 'name': 'None', 'extension': 'txt'},

   228         {'value': 'bash', 'name': 'Bash', 'extension': 'sh'},

   229         {'value': 'c', 'name': 'C', 'extension': 'c'},

   230         {'value': 'cpp', 'name': 'C++', 'extension': 'cpp'},

   231         {'value': 'html4strict', 'name': 'HTML', 'extension': 'html'},

   232         {'value': 'javascript', 'name': 'Javascript', 'extension': 'js'},

   233         {'value': 'javascript', 'name': 'Javascript', 'extension': 'jsm'},

   234         {'value': 'lua', 'name': 'Lua', 'extension': 'lua'},

   235         {'value': 'perl', 'name': 'Perl', 'extension': 'pl'},

   236         {'value': 'php', 'name': 'PHP', 'extension': 'php'},

   237         {'value': 'python', 'name': 'Python', 'extension': 'py'},

   238         {'value': 'ruby', 'name': 'Ruby', 'extension': 'rb'},

   239         {'value': 'css', 'name': 'CSS', 'extension': 'css'},

   240         {'value': 'diff', 'name': 'Diff', 'extension': 'diff'},

   241         {'value': 'ini', 'name': 'INI file', 'extension': 'ini'},

   242         {'value': 'java', 'name': 'Java', 'extension': 'java'},

   243         {'value': 'xml', 'name': 'XML', 'extension': 'xml'},

   244         {'value': 'xml', 'name': 'XML', 'extension': 'xul'}]

   246         lang = ''

   248         if file:

   249             try:

   250                 with open(file, 'r') as f:

   251                     content = f.read()

   252                 # TODO: Use mime-types instead of extensions; suprocess('file <f_name>')

   253                 # Guess File-type based on file extension

   254                 extension = file.split('.')[-1]

   255                 for l in FILE_TYPES:

   256                     if extension == l['extension']:

   257                         print('Identified file as %s' % l['name'])

   258                         lang = l['value']

   259             except IOError:

   260                 print('ERROR. No such file')

   261                 return 1

   262         else:

   263             content = sys.stdin.read()

   264         duration = duration[0]

   266         if language:

   267             lang = language

   270         params = [

   271             ('parent_pid', ''),

   272             ('format', lang),

   273             ('code2', content),

   274             ('poster', poster),

   275             ('expiry', duration),

   276             ('paste', 'Send')]

   278         data = urllib.urlencode(params)

   279         print('Uploading ...')

   280         try:

   281             req = urllib2.Request(URL, data)

   282             response = urllib2.urlopen(req)

   283             http_response_code = response.getcode()

   284             if http_response_code == 200:

   285                 print(response.geturl())

   286             else:

   287                 print('Could not upload the file, '

   288                       'HTTP Response Code %s' %(http_response_code))

   289         except urllib2.URLError:

   290             print('ERROR. Could not connect to pastebin.mozilla.org.')

   291             return 1

   292         return 0

   295 @CommandProvider

   296 class ReviewboardToolsProvider(MachCommandBase):

   297     @Command('rbt', category='devenv', allow_all_args=True,

   298         description='Run Reviewboard Tools')

   299     @CommandArgument('args', nargs='...', help='Arguments to rbt tool')

   300     def rbt(self, args):

   301         if not args:

   302             args = ['help']

   304         self._activate_virtualenv()

   305         self.virtualenv_manager.install_pip_package('RBTools==0.6')

   307         from rbtools.commands.main import main

   309         # main() doesn't accept arguments and instead reads from sys.argv. So,

   310         # we fake it out.

   311         sys.argv = ['rbt'] + args

   312         return main()

   314 @CommandProvider

   315 class FormatProvider(MachCommandBase):

   316     @Command('clang-format', category='misc',

   317         description='Run clang-format on current changes')

   318     @CommandArgument('--show', '-s', action = 'store_true',

   319         help = 'Show diff output on instead of applying changes')

   320     def clang_format(self, show=False):

   321         plat = platform.system()

   322         fmt = plat.lower() + "/clang-format-3.5"

   323         fmt_diff = "clang-format-diff-3.5"

   325         # We are currently using a modified verion of clang-format hosted on people.mozilla.org.

   326         # This is a temporary work around until we upstream the necessary changes and we can use

   327         # a system version of clang-format. See bug 961541.

   328         if plat == "Windows":

   329             fmt += ".exe"

   330         else:

   331             arch = os.uname()[4]

   332             if (plat != "Linux" and plat != "Darwin") or arch != 'x86_64':

   333                 print("Unsupported platform " + plat + "/" + arch +

   334                       ". Supported platforms are Windows/*, Linux/x86_64 and Darwin/x86_64")

   335                 return 1

   337         os.chdir(self.topsrcdir)

   338         self.prompt = True

   340         try:

   341             if not self.locate_or_fetch(fmt):

   342                 return 1

   343             clang_format_diff = self.locate_or_fetch(fmt_diff)

   344             if not clang_format_diff:

   345                 return 1

   347         except urllib2.HTTPError as e:

   348             print("HTTP error {0}: {1}".format(e.code, e.reason))

   349             return 1

   351         from subprocess import Popen, PIPE

   353         if os.path.exists(".hg"):

   354             diff_process = Popen(["hg", "diff", "-U0", "-r", "tip^",

   355                                   "--include", "glob:**.c", "--include", "glob:**.cpp", "--include", "glob:**.h",

   356                                   "--exclude", "listfile:.clang-format-ignore"], stdout=PIPE)

   357         else:

   358             git_process = Popen(["git", "diff", "-U0", "HEAD^"], stdout=PIPE)

   359             try:

   360                 diff_process = Popen(["filterdiff", "--include=*.h", "--include=*.cpp",

   361                                       "--exclude-from-file=.clang-format-ignore"],

   362                                      stdin=git_process.stdout, stdout=PIPE)

   363             except OSError as e:

   364                 if e.errno == errno.ENOENT:

   365                     print("Can't find filterdiff. Please install patchutils.")

   366                 else:

   367                     print("OSError {0}: {1}".format(e.code, e.reason))

   368                 return 1

   371         args = [sys.executable, clang_format_diff, "-p1"]

   372         if not show:

   373            args.append("-i")

   374         cf_process = Popen(args, stdin=diff_process.stdout)

   375         return cf_process.communicate()[0]

   377     def locate_or_fetch(self, root):

   378         target = os.path.join(self._mach_context.state_dir, os.path.basename(root))

   379         if not os.path.exists(target):

   380             site = "https://people.mozilla.org/~ajones/clang-format/"

   381             if self.prompt and raw_input("Download clang-format executables from {0} (yN)? ".format(site)).lower() != 'y':

   382                 print("Download aborted.")

   383                 return 1

   384             self.prompt = False

   386             u = site + root

   387             print("Downloading {0} to {1}".format(u, target))

   388             data = urllib2.urlopen(url=u).read()

   389             temp = target + ".tmp"

   390             with open(temp, "wb") as fh:

   391                 fh.write(data)

   392                 fh.close()

   393             os.chmod(temp, os.stat(temp).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)

   394             os.rename(temp, target)

   395         return target

The Tor Browser / file revision

tools/mach_commands.py@97036ab72558

tools/mach_commands.py