1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/media/webrtc/trunk/testing/gtest/scripts/upload.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,1387 @@
1.4 +#!/usr/bin/env python
1.5 +#
1.6 +# Copyright 2007 Google Inc.
1.7 +#
1.8 +# Licensed under the Apache License, Version 2.0 (the "License");
1.9 +# you may not use this file except in compliance with the License.
1.10 +# You may obtain a copy of the License at
1.11 +#
1.12 +# http://www.apache.org/licenses/LICENSE-2.0
1.13 +#
1.14 +# Unless required by applicable law or agreed to in writing, software
1.15 +# distributed under the License is distributed on an "AS IS" BASIS,
1.16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1.17 +# See the License for the specific language governing permissions and
1.18 +# limitations under the License.
1.19 +
1.20 +"""Tool for uploading diffs from a version control system to the codereview app.
1.21 +
1.22 +Usage summary: upload.py [options] [-- diff_options]
1.23 +
1.24 +Diff options are passed to the diff command of the underlying system.
1.25 +
1.26 +Supported version control systems:
1.27 + Git
1.28 + Mercurial
1.29 + Subversion
1.30 +
1.31 +It is important for Git/Mercurial users to specify a tree/node/branch to diff
1.32 +against by using the '--rev' option.
1.33 +"""
1.34 +# This code is derived from appcfg.py in the App Engine SDK (open source),
1.35 +# and from ASPN recipe #146306.
1.36 +
1.37 +import cookielib
1.38 +import getpass
1.39 +import logging
1.40 +import md5
1.41 +import mimetypes
1.42 +import optparse
1.43 +import os
1.44 +import re
1.45 +import socket
1.46 +import subprocess
1.47 +import sys
1.48 +import urllib
1.49 +import urllib2
1.50 +import urlparse
1.51 +
1.52 +try:
1.53 + import readline
1.54 +except ImportError:
1.55 + pass
1.56 +
1.57 +# The logging verbosity:
1.58 +# 0: Errors only.
1.59 +# 1: Status messages.
1.60 +# 2: Info logs.
1.61 +# 3: Debug logs.
1.62 +verbosity = 1
1.63 +
1.64 +# Max size of patch or base file.
1.65 +MAX_UPLOAD_SIZE = 900 * 1024
1.66 +
1.67 +
1.68 +def GetEmail(prompt):
1.69 + """Prompts the user for their email address and returns it.
1.70 +
1.71 + The last used email address is saved to a file and offered up as a suggestion
1.72 + to the user. If the user presses enter without typing in anything the last
1.73 + used email address is used. If the user enters a new address, it is saved
1.74 + for next time we prompt.
1.75 +
1.76 + """
1.77 + last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
1.78 + last_email = ""
1.79 + if os.path.exists(last_email_file_name):
1.80 + try:
1.81 + last_email_file = open(last_email_file_name, "r")
1.82 + last_email = last_email_file.readline().strip("\n")
1.83 + last_email_file.close()
1.84 + prompt += " [%s]" % last_email
1.85 + except IOError, e:
1.86 + pass
1.87 + email = raw_input(prompt + ": ").strip()
1.88 + if email:
1.89 + try:
1.90 + last_email_file = open(last_email_file_name, "w")
1.91 + last_email_file.write(email)
1.92 + last_email_file.close()
1.93 + except IOError, e:
1.94 + pass
1.95 + else:
1.96 + email = last_email
1.97 + return email
1.98 +
1.99 +
1.100 +def StatusUpdate(msg):
1.101 + """Print a status message to stdout.
1.102 +
1.103 + If 'verbosity' is greater than 0, print the message.
1.104 +
1.105 + Args:
1.106 + msg: The string to print.
1.107 + """
1.108 + if verbosity > 0:
1.109 + print msg
1.110 +
1.111 +
1.112 +def ErrorExit(msg):
1.113 + """Print an error message to stderr and exit."""
1.114 + print >>sys.stderr, msg
1.115 + sys.exit(1)
1.116 +
1.117 +
1.118 +class ClientLoginError(urllib2.HTTPError):
1.119 + """Raised to indicate there was an error authenticating with ClientLogin."""
1.120 +
1.121 + def __init__(self, url, code, msg, headers, args):
1.122 + urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
1.123 + self.args = args
1.124 + self.reason = args["Error"]
1.125 +
1.126 +
1.127 +class AbstractRpcServer(object):
1.128 + """Provides a common interface for a simple RPC server."""
1.129 +
1.130 + def __init__(self, host, auth_function, host_override=None, extra_headers={},
1.131 + save_cookies=False):
1.132 + """Creates a new HttpRpcServer.
1.133 +
1.134 + Args:
1.135 + host: The host to send requests to.
1.136 + auth_function: A function that takes no arguments and returns an
1.137 + (email, password) tuple when called. Will be called if authentication
1.138 + is required.
1.139 + host_override: The host header to send to the server (defaults to host).
1.140 + extra_headers: A dict of extra headers to append to every request.
1.141 + save_cookies: If True, save the authentication cookies to local disk.
1.142 + If False, use an in-memory cookiejar instead. Subclasses must
1.143 + implement this functionality. Defaults to False.
1.144 + """
1.145 + self.host = host
1.146 + self.host_override = host_override
1.147 + self.auth_function = auth_function
1.148 + self.authenticated = False
1.149 + self.extra_headers = extra_headers
1.150 + self.save_cookies = save_cookies
1.151 + self.opener = self._GetOpener()
1.152 + if self.host_override:
1.153 + logging.info("Server: %s; Host: %s", self.host, self.host_override)
1.154 + else:
1.155 + logging.info("Server: %s", self.host)
1.156 +
1.157 + def _GetOpener(self):
1.158 + """Returns an OpenerDirector for making HTTP requests.
1.159 +
1.160 + Returns:
1.161 + A urllib2.OpenerDirector object.
1.162 + """
1.163 + raise NotImplementedError()
1.164 +
1.165 + def _CreateRequest(self, url, data=None):
1.166 + """Creates a new urllib request."""
1.167 + logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
1.168 + req = urllib2.Request(url, data=data)
1.169 + if self.host_override:
1.170 + req.add_header("Host", self.host_override)
1.171 + for key, value in self.extra_headers.iteritems():
1.172 + req.add_header(key, value)
1.173 + return req
1.174 +
1.175 + def _GetAuthToken(self, email, password):
1.176 + """Uses ClientLogin to authenticate the user, returning an auth token.
1.177 +
1.178 + Args:
1.179 + email: The user's email address
1.180 + password: The user's password
1.181 +
1.182 + Raises:
1.183 + ClientLoginError: If there was an error authenticating with ClientLogin.
1.184 + HTTPError: If there was some other form of HTTP error.
1.185 +
1.186 + Returns:
1.187 + The authentication token returned by ClientLogin.
1.188 + """
1.189 + account_type = "GOOGLE"
1.190 + if self.host.endswith(".google.com"):
1.191 + # Needed for use inside Google.
1.192 + account_type = "HOSTED"
1.193 + req = self._CreateRequest(
1.194 + url="https://www.google.com/accounts/ClientLogin",
1.195 + data=urllib.urlencode({
1.196 + "Email": email,
1.197 + "Passwd": password,
1.198 + "service": "ah",
1.199 + "source": "rietveld-codereview-upload",
1.200 + "accountType": account_type,
1.201 + }),
1.202 + )
1.203 + try:
1.204 + response = self.opener.open(req)
1.205 + response_body = response.read()
1.206 + response_dict = dict(x.split("=")
1.207 + for x in response_body.split("\n") if x)
1.208 + return response_dict["Auth"]
1.209 + except urllib2.HTTPError, e:
1.210 + if e.code == 403:
1.211 + body = e.read()
1.212 + response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
1.213 + raise ClientLoginError(req.get_full_url(), e.code, e.msg,
1.214 + e.headers, response_dict)
1.215 + else:
1.216 + raise
1.217 +
1.218 + def _GetAuthCookie(self, auth_token):
1.219 + """Fetches authentication cookies for an authentication token.
1.220 +
1.221 + Args:
1.222 + auth_token: The authentication token returned by ClientLogin.
1.223 +
1.224 + Raises:
1.225 + HTTPError: If there was an error fetching the authentication cookies.
1.226 + """
1.227 + # This is a dummy value to allow us to identify when we're successful.
1.228 + continue_location = "http://localhost/"
1.229 + args = {"continue": continue_location, "auth": auth_token}
1.230 + req = self._CreateRequest("http://%s/_ah/login?%s" %
1.231 + (self.host, urllib.urlencode(args)))
1.232 + try:
1.233 + response = self.opener.open(req)
1.234 + except urllib2.HTTPError, e:
1.235 + response = e
1.236 + if (response.code != 302 or
1.237 + response.info()["location"] != continue_location):
1.238 + raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
1.239 + response.headers, response.fp)
1.240 + self.authenticated = True
1.241 +
1.242 + def _Authenticate(self):
1.243 + """Authenticates the user.
1.244 +
1.245 + The authentication process works as follows:
1.246 + 1) We get a username and password from the user
1.247 + 2) We use ClientLogin to obtain an AUTH token for the user
1.248 + (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
1.249 + 3) We pass the auth token to /_ah/login on the server to obtain an
1.250 + authentication cookie. If login was successful, it tries to redirect
1.251 + us to the URL we provided.
1.252 +
1.253 + If we attempt to access the upload API without first obtaining an
1.254 + authentication cookie, it returns a 401 response and directs us to
1.255 + authenticate ourselves with ClientLogin.
1.256 + """
1.257 + for i in range(3):
1.258 + credentials = self.auth_function()
1.259 + try:
1.260 + auth_token = self._GetAuthToken(credentials[0], credentials[1])
1.261 + except ClientLoginError, e:
1.262 + if e.reason == "BadAuthentication":
1.263 + print >>sys.stderr, "Invalid username or password."
1.264 + continue
1.265 + if e.reason == "CaptchaRequired":
1.266 + print >>sys.stderr, (
1.267 + "Please go to\n"
1.268 + "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
1.269 + "and verify you are a human. Then try again.")
1.270 + break
1.271 + if e.reason == "NotVerified":
1.272 + print >>sys.stderr, "Account not verified."
1.273 + break
1.274 + if e.reason == "TermsNotAgreed":
1.275 + print >>sys.stderr, "User has not agreed to TOS."
1.276 + break
1.277 + if e.reason == "AccountDeleted":
1.278 + print >>sys.stderr, "The user account has been deleted."
1.279 + break
1.280 + if e.reason == "AccountDisabled":
1.281 + print >>sys.stderr, "The user account has been disabled."
1.282 + break
1.283 + if e.reason == "ServiceDisabled":
1.284 + print >>sys.stderr, ("The user's access to the service has been "
1.285 + "disabled.")
1.286 + break
1.287 + if e.reason == "ServiceUnavailable":
1.288 + print >>sys.stderr, "The service is not available; try again later."
1.289 + break
1.290 + raise
1.291 + self._GetAuthCookie(auth_token)
1.292 + return
1.293 +
1.294 + def Send(self, request_path, payload=None,
1.295 + content_type="application/octet-stream",
1.296 + timeout=None,
1.297 + **kwargs):
1.298 + """Sends an RPC and returns the response.
1.299 +
1.300 + Args:
1.301 + request_path: The path to send the request to, eg /api/appversion/create.
1.302 + payload: The body of the request, or None to send an empty request.
1.303 + content_type: The Content-Type header to use.
1.304 + timeout: timeout in seconds; default None i.e. no timeout.
1.305 + (Note: for large requests on OS X, the timeout doesn't work right.)
1.306 + kwargs: Any keyword arguments are converted into query string parameters.
1.307 +
1.308 + Returns:
1.309 + The response body, as a string.
1.310 + """
1.311 + # TODO: Don't require authentication. Let the server say
1.312 + # whether it is necessary.
1.313 + if not self.authenticated:
1.314 + self._Authenticate()
1.315 +
1.316 + old_timeout = socket.getdefaulttimeout()
1.317 + socket.setdefaulttimeout(timeout)
1.318 + try:
1.319 + tries = 0
1.320 + while True:
1.321 + tries += 1
1.322 + args = dict(kwargs)
1.323 + url = "http://%s%s" % (self.host, request_path)
1.324 + if args:
1.325 + url += "?" + urllib.urlencode(args)
1.326 + req = self._CreateRequest(url=url, data=payload)
1.327 + req.add_header("Content-Type", content_type)
1.328 + try:
1.329 + f = self.opener.open(req)
1.330 + response = f.read()
1.331 + f.close()
1.332 + return response
1.333 + except urllib2.HTTPError, e:
1.334 + if tries > 3:
1.335 + raise
1.336 + elif e.code == 401:
1.337 + self._Authenticate()
1.338 +## elif e.code >= 500 and e.code < 600:
1.339 +## # Server Error - try again.
1.340 +## continue
1.341 + else:
1.342 + raise
1.343 + finally:
1.344 + socket.setdefaulttimeout(old_timeout)
1.345 +
1.346 +
1.347 +class HttpRpcServer(AbstractRpcServer):
1.348 + """Provides a simplified RPC-style interface for HTTP requests."""
1.349 +
1.350 + def _Authenticate(self):
1.351 + """Save the cookie jar after authentication."""
1.352 + super(HttpRpcServer, self)._Authenticate()
1.353 + if self.save_cookies:
1.354 + StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
1.355 + self.cookie_jar.save()
1.356 +
1.357 + def _GetOpener(self):
1.358 + """Returns an OpenerDirector that supports cookies and ignores redirects.
1.359 +
1.360 + Returns:
1.361 + A urllib2.OpenerDirector object.
1.362 + """
1.363 + opener = urllib2.OpenerDirector()
1.364 + opener.add_handler(urllib2.ProxyHandler())
1.365 + opener.add_handler(urllib2.UnknownHandler())
1.366 + opener.add_handler(urllib2.HTTPHandler())
1.367 + opener.add_handler(urllib2.HTTPDefaultErrorHandler())
1.368 + opener.add_handler(urllib2.HTTPSHandler())
1.369 + opener.add_handler(urllib2.HTTPErrorProcessor())
1.370 + if self.save_cookies:
1.371 + self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
1.372 + self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
1.373 + if os.path.exists(self.cookie_file):
1.374 + try:
1.375 + self.cookie_jar.load()
1.376 + self.authenticated = True
1.377 + StatusUpdate("Loaded authentication cookies from %s" %
1.378 + self.cookie_file)
1.379 + except (cookielib.LoadError, IOError):
1.380 + # Failed to load cookies - just ignore them.
1.381 + pass
1.382 + else:
1.383 + # Create an empty cookie file with mode 600
1.384 + fd = os.open(self.cookie_file, os.O_CREAT, 0600)
1.385 + os.close(fd)
1.386 + # Always chmod the cookie file
1.387 + os.chmod(self.cookie_file, 0600)
1.388 + else:
1.389 + # Don't save cookies across runs of update.py.
1.390 + self.cookie_jar = cookielib.CookieJar()
1.391 + opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
1.392 + return opener
1.393 +
1.394 +
1.395 +parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
1.396 +parser.add_option("-y", "--assume_yes", action="store_true",
1.397 + dest="assume_yes", default=False,
1.398 + help="Assume that the answer to yes/no questions is 'yes'.")
1.399 +# Logging
1.400 +group = parser.add_option_group("Logging options")
1.401 +group.add_option("-q", "--quiet", action="store_const", const=0,
1.402 + dest="verbose", help="Print errors only.")
1.403 +group.add_option("-v", "--verbose", action="store_const", const=2,
1.404 + dest="verbose", default=1,
1.405 + help="Print info level logs (default).")
1.406 +group.add_option("--noisy", action="store_const", const=3,
1.407 + dest="verbose", help="Print all logs.")
1.408 +# Review server
1.409 +group = parser.add_option_group("Review server options")
1.410 +group.add_option("-s", "--server", action="store", dest="server",
1.411 + default="codereview.appspot.com",
1.412 + metavar="SERVER",
1.413 + help=("The server to upload to. The format is host[:port]. "
1.414 + "Defaults to 'codereview.appspot.com'."))
1.415 +group.add_option("-e", "--email", action="store", dest="email",
1.416 + metavar="EMAIL", default=None,
1.417 + help="The username to use. Will prompt if omitted.")
1.418 +group.add_option("-H", "--host", action="store", dest="host",
1.419 + metavar="HOST", default=None,
1.420 + help="Overrides the Host header sent with all RPCs.")
1.421 +group.add_option("--no_cookies", action="store_false",
1.422 + dest="save_cookies", default=True,
1.423 + help="Do not save authentication cookies to local disk.")
1.424 +# Issue
1.425 +group = parser.add_option_group("Issue options")
1.426 +group.add_option("-d", "--description", action="store", dest="description",
1.427 + metavar="DESCRIPTION", default=None,
1.428 + help="Optional description when creating an issue.")
1.429 +group.add_option("-f", "--description_file", action="store",
1.430 + dest="description_file", metavar="DESCRIPTION_FILE",
1.431 + default=None,
1.432 + help="Optional path of a file that contains "
1.433 + "the description when creating an issue.")
1.434 +group.add_option("-r", "--reviewers", action="store", dest="reviewers",
1.435 + metavar="REVIEWERS", default=None,
1.436 + help="Add reviewers (comma separated email addresses).")
1.437 +group.add_option("--cc", action="store", dest="cc",
1.438 + metavar="CC", default=None,
1.439 + help="Add CC (comma separated email addresses).")
1.440 +# Upload options
1.441 +group = parser.add_option_group("Patch options")
1.442 +group.add_option("-m", "--message", action="store", dest="message",
1.443 + metavar="MESSAGE", default=None,
1.444 + help="A message to identify the patch. "
1.445 + "Will prompt if omitted.")
1.446 +group.add_option("-i", "--issue", type="int", action="store",
1.447 + metavar="ISSUE", default=None,
1.448 + help="Issue number to which to add. Defaults to new issue.")
1.449 +group.add_option("--download_base", action="store_true",
1.450 + dest="download_base", default=False,
1.451 + help="Base files will be downloaded by the server "
1.452 + "(side-by-side diffs may not work on files with CRs).")
1.453 +group.add_option("--rev", action="store", dest="revision",
1.454 + metavar="REV", default=None,
1.455 + help="Branch/tree/revision to diff against (used by DVCS).")
1.456 +group.add_option("--send_mail", action="store_true",
1.457 + dest="send_mail", default=False,
1.458 + help="Send notification email to reviewers.")
1.459 +
1.460 +
1.461 +def GetRpcServer(options):
1.462 + """Returns an instance of an AbstractRpcServer.
1.463 +
1.464 + Returns:
1.465 + A new AbstractRpcServer, on which RPC calls can be made.
1.466 + """
1.467 +
1.468 + rpc_server_class = HttpRpcServer
1.469 +
1.470 + def GetUserCredentials():
1.471 + """Prompts the user for a username and password."""
1.472 + email = options.email
1.473 + if email is None:
1.474 + email = GetEmail("Email (login for uploading to %s)" % options.server)
1.475 + password = getpass.getpass("Password for %s: " % email)
1.476 + return (email, password)
1.477 +
1.478 + # If this is the dev_appserver, use fake authentication.
1.479 + host = (options.host or options.server).lower()
1.480 + if host == "localhost" or host.startswith("localhost:"):
1.481 + email = options.email
1.482 + if email is None:
1.483 + email = "test@example.com"
1.484 + logging.info("Using debug user %s. Override with --email" % email)
1.485 + server = rpc_server_class(
1.486 + options.server,
1.487 + lambda: (email, "password"),
1.488 + host_override=options.host,
1.489 + extra_headers={"Cookie":
1.490 + 'dev_appserver_login="%s:False"' % email},
1.491 + save_cookies=options.save_cookies)
1.492 + # Don't try to talk to ClientLogin.
1.493 + server.authenticated = True
1.494 + return server
1.495 +
1.496 + return rpc_server_class(options.server, GetUserCredentials,
1.497 + host_override=options.host,
1.498 + save_cookies=options.save_cookies)
1.499 +
1.500 +
1.501 +def EncodeMultipartFormData(fields, files):
1.502 + """Encode form fields for multipart/form-data.
1.503 +
1.504 + Args:
1.505 + fields: A sequence of (name, value) elements for regular form fields.
1.506 + files: A sequence of (name, filename, value) elements for data to be
1.507 + uploaded as files.
1.508 + Returns:
1.509 + (content_type, body) ready for httplib.HTTP instance.
1.510 +
1.511 + Source:
1.512 + http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
1.513 + """
1.514 + BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
1.515 + CRLF = '\r\n'
1.516 + lines = []
1.517 + for (key, value) in fields:
1.518 + lines.append('--' + BOUNDARY)
1.519 + lines.append('Content-Disposition: form-data; name="%s"' % key)
1.520 + lines.append('')
1.521 + lines.append(value)
1.522 + for (key, filename, value) in files:
1.523 + lines.append('--' + BOUNDARY)
1.524 + lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
1.525 + (key, filename))
1.526 + lines.append('Content-Type: %s' % GetContentType(filename))
1.527 + lines.append('')
1.528 + lines.append(value)
1.529 + lines.append('--' + BOUNDARY + '--')
1.530 + lines.append('')
1.531 + body = CRLF.join(lines)
1.532 + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
1.533 + return content_type, body
1.534 +
1.535 +
1.536 +def GetContentType(filename):
1.537 + """Helper to guess the content-type from the filename."""
1.538 + return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
1.539 +
1.540 +
1.541 +# Use a shell for subcommands on Windows to get a PATH search.
1.542 +use_shell = sys.platform.startswith("win")
1.543 +
1.544 +def RunShellWithReturnCode(command, print_output=False,
1.545 + universal_newlines=True):
1.546 + """Executes a command and returns the output from stdout and the return code.
1.547 +
1.548 + Args:
1.549 + command: Command to execute.
1.550 + print_output: If True, the output is printed to stdout.
1.551 + If False, both stdout and stderr are ignored.
1.552 + universal_newlines: Use universal_newlines flag (default: True).
1.553 +
1.554 + Returns:
1.555 + Tuple (output, return code)
1.556 + """
1.557 + logging.info("Running %s", command)
1.558 + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1.559 + shell=use_shell, universal_newlines=universal_newlines)
1.560 + if print_output:
1.561 + output_array = []
1.562 + while True:
1.563 + line = p.stdout.readline()
1.564 + if not line:
1.565 + break
1.566 + print line.strip("\n")
1.567 + output_array.append(line)
1.568 + output = "".join(output_array)
1.569 + else:
1.570 + output = p.stdout.read()
1.571 + p.wait()
1.572 + errout = p.stderr.read()
1.573 + if print_output and errout:
1.574 + print >>sys.stderr, errout
1.575 + p.stdout.close()
1.576 + p.stderr.close()
1.577 + return output, p.returncode
1.578 +
1.579 +
1.580 +def RunShell(command, silent_ok=False, universal_newlines=True,
1.581 + print_output=False):
1.582 + data, retcode = RunShellWithReturnCode(command, print_output,
1.583 + universal_newlines)
1.584 + if retcode:
1.585 + ErrorExit("Got error status from %s:\n%s" % (command, data))
1.586 + if not silent_ok and not data:
1.587 + ErrorExit("No output from %s" % command)
1.588 + return data
1.589 +
1.590 +
1.591 +class VersionControlSystem(object):
1.592 + """Abstract base class providing an interface to the VCS."""
1.593 +
1.594 + def __init__(self, options):
1.595 + """Constructor.
1.596 +
1.597 + Args:
1.598 + options: Command line options.
1.599 + """
1.600 + self.options = options
1.601 +
1.602 + def GenerateDiff(self, args):
1.603 + """Return the current diff as a string.
1.604 +
1.605 + Args:
1.606 + args: Extra arguments to pass to the diff command.
1.607 + """
1.608 + raise NotImplementedError(
1.609 + "abstract method -- subclass %s must override" % self.__class__)
1.610 +
1.611 + def GetUnknownFiles(self):
1.612 + """Return a list of files unknown to the VCS."""
1.613 + raise NotImplementedError(
1.614 + "abstract method -- subclass %s must override" % self.__class__)
1.615 +
1.616 + def CheckForUnknownFiles(self):
1.617 + """Show an "are you sure?" prompt if there are unknown files."""
1.618 + unknown_files = self.GetUnknownFiles()
1.619 + if unknown_files:
1.620 + print "The following files are not added to version control:"
1.621 + for line in unknown_files:
1.622 + print line
1.623 + prompt = "Are you sure to continue?(y/N) "
1.624 + answer = raw_input(prompt).strip()
1.625 + if answer != "y":
1.626 + ErrorExit("User aborted")
1.627 +
1.628 + def GetBaseFile(self, filename):
1.629 + """Get the content of the upstream version of a file.
1.630 +
1.631 + Returns:
1.632 + A tuple (base_content, new_content, is_binary, status)
1.633 + base_content: The contents of the base file.
1.634 + new_content: For text files, this is empty. For binary files, this is
1.635 + the contents of the new file, since the diff output won't contain
1.636 + information to reconstruct the current file.
1.637 + is_binary: True iff the file is binary.
1.638 + status: The status of the file.
1.639 + """
1.640 +
1.641 + raise NotImplementedError(
1.642 + "abstract method -- subclass %s must override" % self.__class__)
1.643 +
1.644 +
1.645 + def GetBaseFiles(self, diff):
1.646 + """Helper that calls GetBase file for each file in the patch.
1.647 +
1.648 + Returns:
1.649 + A dictionary that maps from filename to GetBaseFile's tuple. Filenames
1.650 + are retrieved based on lines that start with "Index:" or
1.651 + "Property changes on:".
1.652 + """
1.653 + files = {}
1.654 + for line in diff.splitlines(True):
1.655 + if line.startswith('Index:') or line.startswith('Property changes on:'):
1.656 + unused, filename = line.split(':', 1)
1.657 + # On Windows if a file has property changes its filename uses '\'
1.658 + # instead of '/'.
1.659 + filename = filename.strip().replace('\\', '/')
1.660 + files[filename] = self.GetBaseFile(filename)
1.661 + return files
1.662 +
1.663 +
1.664 + def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
1.665 + files):
1.666 + """Uploads the base files (and if necessary, the current ones as well)."""
1.667 +
1.668 + def UploadFile(filename, file_id, content, is_binary, status, is_base):
1.669 + """Uploads a file to the server."""
1.670 + file_too_large = False
1.671 + if is_base:
1.672 + type = "base"
1.673 + else:
1.674 + type = "current"
1.675 + if len(content) > MAX_UPLOAD_SIZE:
1.676 + print ("Not uploading the %s file for %s because it's too large." %
1.677 + (type, filename))
1.678 + file_too_large = True
1.679 + content = ""
1.680 + checksum = md5.new(content).hexdigest()
1.681 + if options.verbose > 0 and not file_too_large:
1.682 + print "Uploading %s file for %s" % (type, filename)
1.683 + url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
1.684 + form_fields = [("filename", filename),
1.685 + ("status", status),
1.686 + ("checksum", checksum),
1.687 + ("is_binary", str(is_binary)),
1.688 + ("is_current", str(not is_base)),
1.689 + ]
1.690 + if file_too_large:
1.691 + form_fields.append(("file_too_large", "1"))
1.692 + if options.email:
1.693 + form_fields.append(("user", options.email))
1.694 + ctype, body = EncodeMultipartFormData(form_fields,
1.695 + [("data", filename, content)])
1.696 + response_body = rpc_server.Send(url, body,
1.697 + content_type=ctype)
1.698 + if not response_body.startswith("OK"):
1.699 + StatusUpdate(" --> %s" % response_body)
1.700 + sys.exit(1)
1.701 +
1.702 + patches = dict()
1.703 + [patches.setdefault(v, k) for k, v in patch_list]
1.704 + for filename in patches.keys():
1.705 + base_content, new_content, is_binary, status = files[filename]
1.706 + file_id_str = patches.get(filename)
1.707 + if file_id_str.find("nobase") != -1:
1.708 + base_content = None
1.709 + file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
1.710 + file_id = int(file_id_str)
1.711 + if base_content != None:
1.712 + UploadFile(filename, file_id, base_content, is_binary, status, True)
1.713 + if new_content != None:
1.714 + UploadFile(filename, file_id, new_content, is_binary, status, False)
1.715 +
1.716 + def IsImage(self, filename):
1.717 + """Returns true if the filename has an image extension."""
1.718 + mimetype = mimetypes.guess_type(filename)[0]
1.719 + if not mimetype:
1.720 + return False
1.721 + return mimetype.startswith("image/")
1.722 +
1.723 +
1.724 +class SubversionVCS(VersionControlSystem):
1.725 + """Implementation of the VersionControlSystem interface for Subversion."""
1.726 +
1.727 + def __init__(self, options):
1.728 + super(SubversionVCS, self).__init__(options)
1.729 + if self.options.revision:
1.730 + match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
1.731 + if not match:
1.732 + ErrorExit("Invalid Subversion revision %s." % self.options.revision)
1.733 + self.rev_start = match.group(1)
1.734 + self.rev_end = match.group(3)
1.735 + else:
1.736 + self.rev_start = self.rev_end = None
1.737 + # Cache output from "svn list -r REVNO dirname".
1.738 + # Keys: dirname, Values: 2-tuple (ouput for start rev and end rev).
1.739 + self.svnls_cache = {}
1.740 + # SVN base URL is required to fetch files deleted in an older revision.
1.741 + # Result is cached to not guess it over and over again in GetBaseFile().
1.742 + required = self.options.download_base or self.options.revision is not None
1.743 + self.svn_base = self._GuessBase(required)
1.744 +
1.745 + def GuessBase(self, required):
1.746 + """Wrapper for _GuessBase."""
1.747 + return self.svn_base
1.748 +
1.749 + def _GuessBase(self, required):
1.750 + """Returns the SVN base URL.
1.751 +
1.752 + Args:
1.753 + required: If true, exits if the url can't be guessed, otherwise None is
1.754 + returned.
1.755 + """
1.756 + info = RunShell(["svn", "info"])
1.757 + for line in info.splitlines():
1.758 + words = line.split()
1.759 + if len(words) == 2 and words[0] == "URL:":
1.760 + url = words[1]
1.761 + scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
1.762 + username, netloc = urllib.splituser(netloc)
1.763 + if username:
1.764 + logging.info("Removed username from base URL")
1.765 + if netloc.endswith("svn.python.org"):
1.766 + if netloc == "svn.python.org":
1.767 + if path.startswith("/projects/"):
1.768 + path = path[9:]
1.769 + elif netloc != "pythondev@svn.python.org":
1.770 + ErrorExit("Unrecognized Python URL: %s" % url)
1.771 + base = "http://svn.python.org/view/*checkout*%s/" % path
1.772 + logging.info("Guessed Python base = %s", base)
1.773 + elif netloc.endswith("svn.collab.net"):
1.774 + if path.startswith("/repos/"):
1.775 + path = path[6:]
1.776 + base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
1.777 + logging.info("Guessed CollabNet base = %s", base)
1.778 + elif netloc.endswith(".googlecode.com"):
1.779 + path = path + "/"
1.780 + base = urlparse.urlunparse(("http", netloc, path, params,
1.781 + query, fragment))
1.782 + logging.info("Guessed Google Code base = %s", base)
1.783 + else:
1.784 + path = path + "/"
1.785 + base = urlparse.urlunparse((scheme, netloc, path, params,
1.786 + query, fragment))
1.787 + logging.info("Guessed base = %s", base)
1.788 + return base
1.789 + if required:
1.790 + ErrorExit("Can't find URL in output from svn info")
1.791 + return None
1.792 +
1.793 + def GenerateDiff(self, args):
1.794 + cmd = ["svn", "diff"]
1.795 + if self.options.revision:
1.796 + cmd += ["-r", self.options.revision]
1.797 + cmd.extend(args)
1.798 + data = RunShell(cmd)
1.799 + count = 0
1.800 + for line in data.splitlines():
1.801 + if line.startswith("Index:") or line.startswith("Property changes on:"):
1.802 + count += 1
1.803 + logging.info(line)
1.804 + if not count:
1.805 + ErrorExit("No valid patches found in output from svn diff")
1.806 + return data
1.807 +
1.808 + def _CollapseKeywords(self, content, keyword_str):
1.809 + """Collapses SVN keywords."""
1.810 + # svn cat translates keywords but svn diff doesn't. As a result of this
1.811 + # behavior patching.PatchChunks() fails with a chunk mismatch error.
1.812 + # This part was originally written by the Review Board development team
1.813 + # who had the same problem (http://reviews.review-board.org/r/276/).
1.814 + # Mapping of keywords to known aliases
1.815 + svn_keywords = {
1.816 + # Standard keywords
1.817 + 'Date': ['Date', 'LastChangedDate'],
1.818 + 'Revision': ['Revision', 'LastChangedRevision', 'Rev'],
1.819 + 'Author': ['Author', 'LastChangedBy'],
1.820 + 'HeadURL': ['HeadURL', 'URL'],
1.821 + 'Id': ['Id'],
1.822 +
1.823 + # Aliases
1.824 + 'LastChangedDate': ['LastChangedDate', 'Date'],
1.825 + 'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
1.826 + 'LastChangedBy': ['LastChangedBy', 'Author'],
1.827 + 'URL': ['URL', 'HeadURL'],
1.828 + }
1.829 +
1.830 + def repl(m):
1.831 + if m.group(2):
1.832 + return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
1.833 + return "$%s$" % m.group(1)
1.834 + keywords = [keyword
1.835 + for name in keyword_str.split(" ")
1.836 + for keyword in svn_keywords.get(name, [])]
1.837 + return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
1.838 +
1.839 + def GetUnknownFiles(self):
1.840 + status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
1.841 + unknown_files = []
1.842 + for line in status.split("\n"):
1.843 + if line and line[0] == "?":
1.844 + unknown_files.append(line)
1.845 + return unknown_files
1.846 +
1.847 + def ReadFile(self, filename):
1.848 + """Returns the contents of a file."""
1.849 + file = open(filename, 'rb')
1.850 + result = ""
1.851 + try:
1.852 + result = file.read()
1.853 + finally:
1.854 + file.close()
1.855 + return result
1.856 +
1.857 + def GetStatus(self, filename):
1.858 + """Returns the status of a file."""
1.859 + if not self.options.revision:
1.860 + status = RunShell(["svn", "status", "--ignore-externals", filename])
1.861 + if not status:
1.862 + ErrorExit("svn status returned no output for %s" % filename)
1.863 + status_lines = status.splitlines()
1.864 + # If file is in a cl, the output will begin with
1.865 + # "\n--- Changelist 'cl_name':\n". See
1.866 + # http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
1.867 + if (len(status_lines) == 3 and
1.868 + not status_lines[0] and
1.869 + status_lines[1].startswith("--- Changelist")):
1.870 + status = status_lines[2]
1.871 + else:
1.872 + status = status_lines[0]
1.873 + # If we have a revision to diff against we need to run "svn list"
1.874 + # for the old and the new revision and compare the results to get
1.875 + # the correct status for a file.
1.876 + else:
1.877 + dirname, relfilename = os.path.split(filename)
1.878 + if dirname not in self.svnls_cache:
1.879 + cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
1.880 + out, returncode = RunShellWithReturnCode(cmd)
1.881 + if returncode:
1.882 + ErrorExit("Failed to get status for %s." % filename)
1.883 + old_files = out.splitlines()
1.884 + args = ["svn", "list"]
1.885 + if self.rev_end:
1.886 + args += ["-r", self.rev_end]
1.887 + cmd = args + [dirname or "."]
1.888 + out, returncode = RunShellWithReturnCode(cmd)
1.889 + if returncode:
1.890 + ErrorExit("Failed to run command %s" % cmd)
1.891 + self.svnls_cache[dirname] = (old_files, out.splitlines())
1.892 + old_files, new_files = self.svnls_cache[dirname]
1.893 + if relfilename in old_files and relfilename not in new_files:
1.894 + status = "D "
1.895 + elif relfilename in old_files and relfilename in new_files:
1.896 + status = "M "
1.897 + else:
1.898 + status = "A "
1.899 + return status
1.900 +
1.901 + def GetBaseFile(self, filename):
1.902 + status = self.GetStatus(filename)
1.903 + base_content = None
1.904 + new_content = None
1.905 +
1.906 + # If a file is copied its status will be "A +", which signifies
1.907 + # "addition-with-history". See "svn st" for more information. We need to
1.908 + # upload the original file or else diff parsing will fail if the file was
1.909 + # edited.
1.910 + if status[0] == "A" and status[3] != "+":
1.911 + # We'll need to upload the new content if we're adding a binary file
1.912 + # since diff's output won't contain it.
1.913 + mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
1.914 + silent_ok=True)
1.915 + base_content = ""
1.916 + is_binary = mimetype and not mimetype.startswith("text/")
1.917 + if is_binary and self.IsImage(filename):
1.918 + new_content = self.ReadFile(filename)
1.919 + elif (status[0] in ("M", "D", "R") or
1.920 + (status[0] == "A" and status[3] == "+") or # Copied file.
1.921 + (status[0] == " " and status[1] == "M")): # Property change.
1.922 + args = []
1.923 + if self.options.revision:
1.924 + url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
1.925 + else:
1.926 + # Don't change filename, it's needed later.
1.927 + url = filename
1.928 + args += ["-r", "BASE"]
1.929 + cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
1.930 + mimetype, returncode = RunShellWithReturnCode(cmd)
1.931 + if returncode:
1.932 + # File does not exist in the requested revision.
1.933 + # Reset mimetype, it contains an error message.
1.934 + mimetype = ""
1.935 + get_base = False
1.936 + is_binary = mimetype and not mimetype.startswith("text/")
1.937 + if status[0] == " ":
1.938 + # Empty base content just to force an upload.
1.939 + base_content = ""
1.940 + elif is_binary:
1.941 + if self.IsImage(filename):
1.942 + get_base = True
1.943 + if status[0] == "M":
1.944 + if not self.rev_end:
1.945 + new_content = self.ReadFile(filename)
1.946 + else:
1.947 + url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
1.948 + new_content = RunShell(["svn", "cat", url],
1.949 + universal_newlines=True, silent_ok=True)
1.950 + else:
1.951 + base_content = ""
1.952 + else:
1.953 + get_base = True
1.954 +
1.955 + if get_base:
1.956 + if is_binary:
1.957 + universal_newlines = False
1.958 + else:
1.959 + universal_newlines = True
1.960 + if self.rev_start:
1.961 + # "svn cat -r REV delete_file.txt" doesn't work. cat requires
1.962 + # the full URL with "@REV" appended instead of using "-r" option.
1.963 + url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
1.964 + base_content = RunShell(["svn", "cat", url],
1.965 + universal_newlines=universal_newlines,
1.966 + silent_ok=True)
1.967 + else:
1.968 + base_content = RunShell(["svn", "cat", filename],
1.969 + universal_newlines=universal_newlines,
1.970 + silent_ok=True)
1.971 + if not is_binary:
1.972 + args = []
1.973 + if self.rev_start:
1.974 + url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
1.975 + else:
1.976 + url = filename
1.977 + args += ["-r", "BASE"]
1.978 + cmd = ["svn"] + args + ["propget", "svn:keywords", url]
1.979 + keywords, returncode = RunShellWithReturnCode(cmd)
1.980 + if keywords and not returncode:
1.981 + base_content = self._CollapseKeywords(base_content, keywords)
1.982 + else:
1.983 + StatusUpdate("svn status returned unexpected output: %s" % status)
1.984 + sys.exit(1)
1.985 + return base_content, new_content, is_binary, status[0:5]
1.986 +
1.987 +
1.988 +class GitVCS(VersionControlSystem):
1.989 + """Implementation of the VersionControlSystem interface for Git."""
1.990 +
1.991 + def __init__(self, options):
1.992 + super(GitVCS, self).__init__(options)
1.993 + # Map of filename -> hash of base file.
1.994 + self.base_hashes = {}
1.995 +
1.996 + def GenerateDiff(self, extra_args):
1.997 + # This is more complicated than svn's GenerateDiff because we must convert
1.998 + # the diff output to include an svn-style "Index:" line as well as record
1.999 + # the hashes of the base files, so we can upload them along with our diff.
1.1000 + if self.options.revision:
1.1001 + extra_args = [self.options.revision] + extra_args
1.1002 + gitdiff = RunShell(["git", "diff", "--full-index"] + extra_args)
1.1003 + svndiff = []
1.1004 + filecount = 0
1.1005 + filename = None
1.1006 + for line in gitdiff.splitlines():
1.1007 + match = re.match(r"diff --git a/(.*) b/.*$", line)
1.1008 + if match:
1.1009 + filecount += 1
1.1010 + filename = match.group(1)
1.1011 + svndiff.append("Index: %s\n" % filename)
1.1012 + else:
1.1013 + # The "index" line in a git diff looks like this (long hashes elided):
1.1014 + # index 82c0d44..b2cee3f 100755
1.1015 + # We want to save the left hash, as that identifies the base file.
1.1016 + match = re.match(r"index (\w+)\.\.", line)
1.1017 + if match:
1.1018 + self.base_hashes[filename] = match.group(1)
1.1019 + svndiff.append(line + "\n")
1.1020 + if not filecount:
1.1021 + ErrorExit("No valid patches found in output from git diff")
1.1022 + return "".join(svndiff)
1.1023 +
1.1024 + def GetUnknownFiles(self):
1.1025 + status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
1.1026 + silent_ok=True)
1.1027 + return status.splitlines()
1.1028 +
1.1029 + def GetBaseFile(self, filename):
1.1030 + hash = self.base_hashes[filename]
1.1031 + base_content = None
1.1032 + new_content = None
1.1033 + is_binary = False
1.1034 + if hash == "0" * 40: # All-zero hash indicates no base file.
1.1035 + status = "A"
1.1036 + base_content = ""
1.1037 + else:
1.1038 + status = "M"
1.1039 + base_content, returncode = RunShellWithReturnCode(["git", "show", hash])
1.1040 + if returncode:
1.1041 + ErrorExit("Got error status from 'git show %s'" % hash)
1.1042 + return (base_content, new_content, is_binary, status)
1.1043 +
1.1044 +
1.1045 +class MercurialVCS(VersionControlSystem):
1.1046 + """Implementation of the VersionControlSystem interface for Mercurial."""
1.1047 +
1.1048 + def __init__(self, options, repo_dir):
1.1049 + super(MercurialVCS, self).__init__(options)
1.1050 + # Absolute path to repository (we can be in a subdir)
1.1051 + self.repo_dir = os.path.normpath(repo_dir)
1.1052 + # Compute the subdir
1.1053 + cwd = os.path.normpath(os.getcwd())
1.1054 + assert cwd.startswith(self.repo_dir)
1.1055 + self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
1.1056 + if self.options.revision:
1.1057 + self.base_rev = self.options.revision
1.1058 + else:
1.1059 + self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip()
1.1060 +
1.1061 + def _GetRelPath(self, filename):
1.1062 + """Get relative path of a file according to the current directory,
1.1063 + given its logical path in the repo."""
1.1064 + assert filename.startswith(self.subdir), filename
1.1065 + return filename[len(self.subdir):].lstrip(r"\/")
1.1066 +
1.1067 + def GenerateDiff(self, extra_args):
1.1068 + # If no file specified, restrict to the current subdir
1.1069 + extra_args = extra_args or ["."]
1.1070 + cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
1.1071 + data = RunShell(cmd, silent_ok=True)
1.1072 + svndiff = []
1.1073 + filecount = 0
1.1074 + for line in data.splitlines():
1.1075 + m = re.match("diff --git a/(\S+) b/(\S+)", line)
1.1076 + if m:
1.1077 + # Modify line to make it look like as it comes from svn diff.
1.1078 + # With this modification no changes on the server side are required
1.1079 + # to make upload.py work with Mercurial repos.
1.1080 + # NOTE: for proper handling of moved/copied files, we have to use
1.1081 + # the second filename.
1.1082 + filename = m.group(2)
1.1083 + svndiff.append("Index: %s" % filename)
1.1084 + svndiff.append("=" * 67)
1.1085 + filecount += 1
1.1086 + logging.info(line)
1.1087 + else:
1.1088 + svndiff.append(line)
1.1089 + if not filecount:
1.1090 + ErrorExit("No valid patches found in output from hg diff")
1.1091 + return "\n".join(svndiff) + "\n"
1.1092 +
1.1093 + def GetUnknownFiles(self):
1.1094 + """Return a list of files unknown to the VCS."""
1.1095 + args = []
1.1096 + status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
1.1097 + silent_ok=True)
1.1098 + unknown_files = []
1.1099 + for line in status.splitlines():
1.1100 + st, fn = line.split(" ", 1)
1.1101 + if st == "?":
1.1102 + unknown_files.append(fn)
1.1103 + return unknown_files
1.1104 +
1.1105 + def GetBaseFile(self, filename):
1.1106 + # "hg status" and "hg cat" both take a path relative to the current subdir
1.1107 + # rather than to the repo root, but "hg diff" has given us the full path
1.1108 + # to the repo root.
1.1109 + base_content = ""
1.1110 + new_content = None
1.1111 + is_binary = False
1.1112 + oldrelpath = relpath = self._GetRelPath(filename)
1.1113 + # "hg status -C" returns two lines for moved/copied files, one otherwise
1.1114 + out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
1.1115 + out = out.splitlines()
1.1116 + # HACK: strip error message about missing file/directory if it isn't in
1.1117 + # the working copy
1.1118 + if out[0].startswith('%s: ' % relpath):
1.1119 + out = out[1:]
1.1120 + if len(out) > 1:
1.1121 + # Moved/copied => considered as modified, use old filename to
1.1122 + # retrieve base contents
1.1123 + oldrelpath = out[1].strip()
1.1124 + status = "M"
1.1125 + else:
1.1126 + status, _ = out[0].split(' ', 1)
1.1127 + if status != "A":
1.1128 + base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
1.1129 + silent_ok=True)
1.1130 + is_binary = "\0" in base_content # Mercurial's heuristic
1.1131 + if status != "R":
1.1132 + new_content = open(relpath, "rb").read()
1.1133 + is_binary = is_binary or "\0" in new_content
1.1134 + if is_binary and base_content:
1.1135 + # Fetch again without converting newlines
1.1136 + base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
1.1137 + silent_ok=True, universal_newlines=False)
1.1138 + if not is_binary or not self.IsImage(relpath):
1.1139 + new_content = None
1.1140 + return base_content, new_content, is_binary, status
1.1141 +
1.1142 +
1.1143 +# NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
1.1144 +def SplitPatch(data):
1.1145 + """Splits a patch into separate pieces for each file.
1.1146 +
1.1147 + Args:
1.1148 + data: A string containing the output of svn diff.
1.1149 +
1.1150 + Returns:
1.1151 + A list of 2-tuple (filename, text) where text is the svn diff output
1.1152 + pertaining to filename.
1.1153 + """
1.1154 + patches = []
1.1155 + filename = None
1.1156 + diff = []
1.1157 + for line in data.splitlines(True):
1.1158 + new_filename = None
1.1159 + if line.startswith('Index:'):
1.1160 + unused, new_filename = line.split(':', 1)
1.1161 + new_filename = new_filename.strip()
1.1162 + elif line.startswith('Property changes on:'):
1.1163 + unused, temp_filename = line.split(':', 1)
1.1164 + # When a file is modified, paths use '/' between directories, however
1.1165 + # when a property is modified '\' is used on Windows. Make them the same
1.1166 + # otherwise the file shows up twice.
1.1167 + temp_filename = temp_filename.strip().replace('\\', '/')
1.1168 + if temp_filename != filename:
1.1169 + # File has property changes but no modifications, create a new diff.
1.1170 + new_filename = temp_filename
1.1171 + if new_filename:
1.1172 + if filename and diff:
1.1173 + patches.append((filename, ''.join(diff)))
1.1174 + filename = new_filename
1.1175 + diff = [line]
1.1176 + continue
1.1177 + if diff is not None:
1.1178 + diff.append(line)
1.1179 + if filename and diff:
1.1180 + patches.append((filename, ''.join(diff)))
1.1181 + return patches
1.1182 +
1.1183 +
1.1184 +def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
1.1185 + """Uploads a separate patch for each file in the diff output.
1.1186 +
1.1187 + Returns a list of [patch_key, filename] for each file.
1.1188 + """
1.1189 + patches = SplitPatch(data)
1.1190 + rv = []
1.1191 + for patch in patches:
1.1192 + if len(patch[1]) > MAX_UPLOAD_SIZE:
1.1193 + print ("Not uploading the patch for " + patch[0] +
1.1194 + " because the file is too large.")
1.1195 + continue
1.1196 + form_fields = [("filename", patch[0])]
1.1197 + if not options.download_base:
1.1198 + form_fields.append(("content_upload", "1"))
1.1199 + files = [("data", "data.diff", patch[1])]
1.1200 + ctype, body = EncodeMultipartFormData(form_fields, files)
1.1201 + url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
1.1202 + print "Uploading patch for " + patch[0]
1.1203 + response_body = rpc_server.Send(url, body, content_type=ctype)
1.1204 + lines = response_body.splitlines()
1.1205 + if not lines or lines[0] != "OK":
1.1206 + StatusUpdate(" --> %s" % response_body)
1.1207 + sys.exit(1)
1.1208 + rv.append([lines[1], patch[0]])
1.1209 + return rv
1.1210 +
1.1211 +
1.1212 +def GuessVCS(options):
1.1213 + """Helper to guess the version control system.
1.1214 +
1.1215 + This examines the current directory, guesses which VersionControlSystem
1.1216 + we're using, and returns an instance of the appropriate class. Exit with an
1.1217 + error if we can't figure it out.
1.1218 +
1.1219 + Returns:
1.1220 + A VersionControlSystem instance. Exits if the VCS can't be guessed.
1.1221 + """
1.1222 + # Mercurial has a command to get the base directory of a repository
1.1223 + # Try running it, but don't die if we don't have hg installed.
1.1224 + # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
1.1225 + try:
1.1226 + out, returncode = RunShellWithReturnCode(["hg", "root"])
1.1227 + if returncode == 0:
1.1228 + return MercurialVCS(options, out.strip())
1.1229 + except OSError, (errno, message):
1.1230 + if errno != 2: # ENOENT -- they don't have hg installed.
1.1231 + raise
1.1232 +
1.1233 + # Subversion has a .svn in all working directories.
1.1234 + if os.path.isdir('.svn'):
1.1235 + logging.info("Guessed VCS = Subversion")
1.1236 + return SubversionVCS(options)
1.1237 +
1.1238 + # Git has a command to test if you're in a git tree.
1.1239 + # Try running it, but don't die if we don't have git installed.
1.1240 + try:
1.1241 + out, returncode = RunShellWithReturnCode(["git", "rev-parse",
1.1242 + "--is-inside-work-tree"])
1.1243 + if returncode == 0:
1.1244 + return GitVCS(options)
1.1245 + except OSError, (errno, message):
1.1246 + if errno != 2: # ENOENT -- they don't have git installed.
1.1247 + raise
1.1248 +
1.1249 + ErrorExit(("Could not guess version control system. "
1.1250 + "Are you in a working copy directory?"))
1.1251 +
1.1252 +
1.1253 +def RealMain(argv, data=None):
1.1254 + """The real main function.
1.1255 +
1.1256 + Args:
1.1257 + argv: Command line arguments.
1.1258 + data: Diff contents. If None (default) the diff is generated by
1.1259 + the VersionControlSystem implementation returned by GuessVCS().
1.1260 +
1.1261 + Returns:
1.1262 + A 2-tuple (issue id, patchset id).
1.1263 + The patchset id is None if the base files are not uploaded by this
1.1264 + script (applies only to SVN checkouts).
1.1265 + """
1.1266 + logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
1.1267 + "%(lineno)s %(message)s "))
1.1268 + os.environ['LC_ALL'] = 'C'
1.1269 + options, args = parser.parse_args(argv[1:])
1.1270 + global verbosity
1.1271 + verbosity = options.verbose
1.1272 + if verbosity >= 3:
1.1273 + logging.getLogger().setLevel(logging.DEBUG)
1.1274 + elif verbosity >= 2:
1.1275 + logging.getLogger().setLevel(logging.INFO)
1.1276 + vcs = GuessVCS(options)
1.1277 + if isinstance(vcs, SubversionVCS):
1.1278 + # base field is only allowed for Subversion.
1.1279 + # Note: Fetching base files may become deprecated in future releases.
1.1280 + base = vcs.GuessBase(options.download_base)
1.1281 + else:
1.1282 + base = None
1.1283 + if not base and options.download_base:
1.1284 + options.download_base = True
1.1285 + logging.info("Enabled upload of base file")
1.1286 + if not options.assume_yes:
1.1287 + vcs.CheckForUnknownFiles()
1.1288 + if data is None:
1.1289 + data = vcs.GenerateDiff(args)
1.1290 + files = vcs.GetBaseFiles(data)
1.1291 + if verbosity >= 1:
1.1292 + print "Upload server:", options.server, "(change with -s/--server)"
1.1293 + if options.issue:
1.1294 + prompt = "Message describing this patch set: "
1.1295 + else:
1.1296 + prompt = "New issue subject: "
1.1297 + message = options.message or raw_input(prompt).strip()
1.1298 + if not message:
1.1299 + ErrorExit("A non-empty message is required")
1.1300 + rpc_server = GetRpcServer(options)
1.1301 + form_fields = [("subject", message)]
1.1302 + if base:
1.1303 + form_fields.append(("base", base))
1.1304 + if options.issue:
1.1305 + form_fields.append(("issue", str(options.issue)))
1.1306 + if options.email:
1.1307 + form_fields.append(("user", options.email))
1.1308 + if options.reviewers:
1.1309 + for reviewer in options.reviewers.split(','):
1.1310 + if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
1.1311 + ErrorExit("Invalid email address: %s" % reviewer)
1.1312 + form_fields.append(("reviewers", options.reviewers))
1.1313 + if options.cc:
1.1314 + for cc in options.cc.split(','):
1.1315 + if "@" in cc and not cc.split("@")[1].count(".") == 1:
1.1316 + ErrorExit("Invalid email address: %s" % cc)
1.1317 + form_fields.append(("cc", options.cc))
1.1318 + description = options.description
1.1319 + if options.description_file:
1.1320 + if options.description:
1.1321 + ErrorExit("Can't specify description and description_file")
1.1322 + file = open(options.description_file, 'r')
1.1323 + description = file.read()
1.1324 + file.close()
1.1325 + if description:
1.1326 + form_fields.append(("description", description))
1.1327 + # Send a hash of all the base file so the server can determine if a copy
1.1328 + # already exists in an earlier patchset.
1.1329 + base_hashes = ""
1.1330 + for file, info in files.iteritems():
1.1331 + if not info[0] is None:
1.1332 + checksum = md5.new(info[0]).hexdigest()
1.1333 + if base_hashes:
1.1334 + base_hashes += "|"
1.1335 + base_hashes += checksum + ":" + file
1.1336 + form_fields.append(("base_hashes", base_hashes))
1.1337 + # If we're uploading base files, don't send the email before the uploads, so
1.1338 + # that it contains the file status.
1.1339 + if options.send_mail and options.download_base:
1.1340 + form_fields.append(("send_mail", "1"))
1.1341 + if not options.download_base:
1.1342 + form_fields.append(("content_upload", "1"))
1.1343 + if len(data) > MAX_UPLOAD_SIZE:
1.1344 + print "Patch is large, so uploading file patches separately."
1.1345 + uploaded_diff_file = []
1.1346 + form_fields.append(("separate_patches", "1"))
1.1347 + else:
1.1348 + uploaded_diff_file = [("data", "data.diff", data)]
1.1349 + ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
1.1350 + response_body = rpc_server.Send("/upload", body, content_type=ctype)
1.1351 + patchset = None
1.1352 + if not options.download_base or not uploaded_diff_file:
1.1353 + lines = response_body.splitlines()
1.1354 + if len(lines) >= 2:
1.1355 + msg = lines[0]
1.1356 + patchset = lines[1].strip()
1.1357 + patches = [x.split(" ", 1) for x in lines[2:]]
1.1358 + else:
1.1359 + msg = response_body
1.1360 + else:
1.1361 + msg = response_body
1.1362 + StatusUpdate(msg)
1.1363 + if not response_body.startswith("Issue created.") and \
1.1364 + not response_body.startswith("Issue updated."):
1.1365 + sys.exit(0)
1.1366 + issue = msg[msg.rfind("/")+1:]
1.1367 +
1.1368 + if not uploaded_diff_file:
1.1369 + result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
1.1370 + if not options.download_base:
1.1371 + patches = result
1.1372 +
1.1373 + if not options.download_base:
1.1374 + vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
1.1375 + if options.send_mail:
1.1376 + rpc_server.Send("/" + issue + "/mail", payload="")
1.1377 + return issue, patchset
1.1378 +
1.1379 +
1.1380 +def main():
1.1381 + try:
1.1382 + RealMain(sys.argv)
1.1383 + except KeyboardInterrupt:
1.1384 + print
1.1385 + StatusUpdate("Interrupted.")
1.1386 + sys.exit(1)
1.1387 +
1.1388 +
1.1389 +if __name__ == "__main__":
1.1390 + main()
