The Tor Browser / file comparison
comparison: build/upload.py
build/upload.py
- branch
- TOR_BUG_9701
- changeset 14
- 925c144e1f1f
equal
deleted
inserted
replaced
| -1:000000000000 | 0:be153f745539 |
|---|---|
| 1 #!/usr/bin/python | |
| 2 # | |
| 3 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 4 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 5 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 6 # | |
| 7 # When run directly, this script expects the following environment variables | |
| 8 # to be set: | |
| 9 # UPLOAD_HOST : host to upload files to | |
| 10 # UPLOAD_USER : username on that host | |
| 11 # UPLOAD_PATH : path on that host to put the files in | |
| 12 # | |
| 13 # And will use the following optional environment variables if set: | |
| 14 # UPLOAD_SSH_KEY : path to a ssh private key to use | |
| 15 # UPLOAD_PORT : port to use for ssh | |
| 16 # POST_UPLOAD_CMD: a commandline to run on the remote host after uploading. | |
| 17 # UPLOAD_PATH and the full paths of all files uploaded will | |
| 18 # be appended to the commandline. | |
| 19 # | |
| 20 # All files to be uploaded should be passed as commandline arguments to this | |
| 21 # script. The script takes one other parameter, --base-path, which you can use | |
| 22 # to indicate that files should be uploaded including their paths relative | |
| 23 # to the base path. | |
| 24 | |
| 25 import sys, os | |
| 26 from optparse import OptionParser | |
| 27 from subprocess import PIPE, Popen, check_call | |
| 28 | |
| 29 def RequireEnvironmentVariable(v): | |
| 30 """Return the value of the environment variable named v, or print | |
| 31 an error and exit if it's unset (or empty).""" | |
| 32 if not v in os.environ or os.environ[v] == "": | |
| 33 print "Error: required environment variable %s not set" % v | |
| 34 sys.exit(1) | |
| 35 return os.environ[v] | |
| 36 | |
| 37 def OptionalEnvironmentVariable(v): | |
| 38 """Return the value of the environment variable named v, or None | |
| 39 if it's unset (or empty).""" | |
| 40 if v in os.environ and os.environ[v] != "": | |
| 41 return os.environ[v] | |
| 42 return None | |
| 43 | |
| 44 def FixupMsysPath(path): | |
| 45 """MSYS helpfully translates absolute pathnames in environment variables | |
| 46 and commandline arguments into Windows native paths. This sucks if you're | |
| 47 trying to pass an absolute path on a remote server. This function attempts | |
| 48 to un-mangle such paths.""" | |
| 49 if 'OSTYPE' in os.environ and os.environ['OSTYPE'] == 'msys': | |
| 50 # sort of awful, find out where our shell is (should be in msys/bin) | |
| 51 # and strip the first part of that path out of the other path | |
| 52 if 'SHELL' in os.environ: | |
| 53 sh = os.environ['SHELL'] | |
| 54 msys = sh[:sh.find('/bin')] | |
| 55 if path.startswith(msys): | |
| 56 path = path[len(msys):] | |
| 57 return path | |
| 58 | |
| 59 def WindowsPathToMsysPath(path): | |
| 60 """Translate a Windows pathname to an MSYS pathname. | |
| 61 Necessary because we call out to ssh/scp, which are MSYS binaries | |
| 62 and expect MSYS paths.""" | |
| 63 if sys.platform != 'win32': | |
| 64 return path | |
| 65 (drive, path) = os.path.splitdrive(os.path.abspath(path)) | |
| 66 return "/" + drive[0] + path.replace('\\','/') | |
| 67 | |
| 68 def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key): | |
| 69 """Given optional port and ssh key values, append valid OpenSSH | |
| 70 commandline arguments to the list cmdline if the values are not None.""" | |
| 71 if port is not None: | |
| 72 cmdline.append("-P%d" % port) | |
| 73 if ssh_key is not None: | |
| 74 # Don't interpret ~ paths - ssh can handle that on its own | |
| 75 if not ssh_key.startswith('~'): | |
| 76 ssh_key = WindowsPathToMsysPath(ssh_key) | |
| 77 cmdline.extend(["-o", "IdentityFile=%s" % ssh_key]) | |
| 78 | |
| 79 def DoSSHCommand(command, user, host, port=None, ssh_key=None): | |
| 80 """Execute command on user@host using ssh. Optionally use | |
| 81 port and ssh_key, if provided.""" | |
| 82 cmdline = ["ssh"] | |
| 83 AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) | |
| 84 cmdline.extend(["%s@%s" % (user, host), command]) | |
| 85 cmd = Popen(cmdline, stdout=PIPE) | |
| 86 retcode = cmd.wait() | |
| 87 if retcode != 0: | |
| 88 raise Exception("Command %s returned non-zero exit code: %i" % \ | |
| 89 (cmdline, retcode)) | |
| 90 return cmd.stdout.read().strip() | |
| 91 | |
| 92 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None): | |
| 93 """Upload file to user@host:remote_path using scp. Optionally use | |
| 94 port and ssh_key, if provided.""" | |
| 95 cmdline = ["scp"] | |
| 96 AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key) | |
| 97 cmdline.extend([WindowsPathToMsysPath(file), | |
| 98 "%s@%s:%s" % (user, host, remote_path)]) | |
| 99 check_call(cmdline) | |
| 100 | |
| 101 def GetRemotePath(path, local_file, base_path): | |
| 102 """Given a remote path to upload to, a full path to a local file, and an | |
| 103 optional full path that is a base path of the local file, construct the | |
| 104 full remote path to place the file in. If base_path is not None, include | |
| 105 the relative path from base_path to file.""" | |
| 106 if base_path is None or not local_file.startswith(base_path): | |
| 107 return path | |
| 108 dir = os.path.dirname(local_file) | |
| 109 # strip base_path + extra slash and make it unixy | |
| 110 dir = dir[len(base_path)+1:].replace('\\','/') | |
| 111 return path + dir | |
| 112 | |
| 113 def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None): | |
| 114 """Upload each file in the list files to user@host:path. Optionally pass | |
| 115 port and ssh_key to the ssh commands. If base_path is not None, upload | |
| 116 files including their path relative to base_path. If upload_to_temp_dir is | |
| 117 True files will be uploaded to a temporary directory on the remote server. | |
| 118 Generally, you should have a post upload command specified in these cases | |
| 119 that can move them around to their correct location(s). | |
| 120 If post_upload_command is not None, execute that command on the remote host | |
| 121 after uploading all files, passing it the upload path, and the full paths to | |
| 122 all files uploaded. | |
| 123 If verbose is True, print status updates while working.""" | |
| 124 if upload_to_temp_dir: | |
| 125 path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key) | |
| 126 if not path.endswith("/"): | |
| 127 path += "/" | |
| 128 if base_path is not None: | |
| 129 base_path = os.path.abspath(base_path) | |
| 130 remote_files = [] | |
| 131 try: | |
| 132 for file in files: | |
| 133 file = os.path.abspath(file) | |
| 134 if not os.path.isfile(file): | |
| 135 raise IOError("File not found: %s" % file) | |
| 136 # first ensure that path exists remotely | |
| 137 remote_path = GetRemotePath(path, file, base_path) | |
| 138 DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key) | |
| 139 if verbose: | |
| 140 print "Uploading " + file | |
| 141 DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key) | |
| 142 remote_files.append(remote_path + '/' + os.path.basename(file)) | |
| 143 if post_upload_command is not None: | |
| 144 if verbose: | |
| 145 print "Running post-upload command: " + post_upload_command | |
| 146 file_list = '"' + '" "'.join(remote_files) + '"' | |
| 147 DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key) | |
| 148 finally: | |
| 149 if upload_to_temp_dir: | |
| 150 DoSSHCommand("rm -rf %s" % path, user, host, port=port, | |
| 151 ssh_key=ssh_key) | |
| 152 if verbose: | |
| 153 print "Upload complete" | |
| 154 | |
| 155 if __name__ == '__main__': | |
| 156 host = RequireEnvironmentVariable('UPLOAD_HOST') | |
| 157 user = RequireEnvironmentVariable('UPLOAD_USER') | |
| 158 path = OptionalEnvironmentVariable('UPLOAD_PATH') | |
| 159 upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP') | |
| 160 port = OptionalEnvironmentVariable('UPLOAD_PORT') | |
| 161 if port is not None: | |
| 162 port = int(port) | |
| 163 key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY') | |
| 164 post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD') | |
| 165 if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir): | |
| 166 print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \ | |
| 167 "defined." | |
| 168 sys.exit(1) | |
| 169 if sys.platform == 'win32': | |
| 170 if path is not None: | |
| 171 path = FixupMsysPath(path) | |
| 172 if post_upload_command is not None: | |
| 173 post_upload_command = FixupMsysPath(post_upload_command) | |
| 174 | |
| 175 parser = OptionParser(usage="usage: %prog [options] <files>") | |
| 176 parser.add_option("-b", "--base-path", | |
| 177 action="store", dest="base_path", | |
| 178 help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.") | |
| 179 (options, args) = parser.parse_args() | |
| 180 if len(args) < 1: | |
| 181 print "You must specify at least one file to upload" | |
| 182 sys.exit(1) | |
| 183 try: | |
| 184 UploadFiles(user, host, path, args, base_path=options.base_path, | |
| 185 port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir, | |
| 186 post_upload_command=post_upload_command, | |
| 187 verbose=True) | |
| 188 except IOError, (strerror): | |
| 189 print strerror | |
| 190 sys.exit(1) | |
| 191 except Exception, (err): | |
| 192 print err | |
| 193 sys.exit(2) |
