1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/build/upload.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,193 @@
1.4 +#!/usr/bin/python
1.5 +#
1.6 +# This Source Code Form is subject to the terms of the Mozilla Public
1.7 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.8 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.9 +#
1.10 +# When run directly, this script expects the following environment variables
1.11 +# to be set:
1.12 +# UPLOAD_HOST : host to upload files to
1.13 +# UPLOAD_USER : username on that host
1.14 +# UPLOAD_PATH : path on that host to put the files in
1.15 +#
1.16 +# And will use the following optional environment variables if set:
1.17 +# UPLOAD_SSH_KEY : path to a ssh private key to use
1.18 +# UPLOAD_PORT : port to use for ssh
1.19 +# POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
1.20 +# UPLOAD_PATH and the full paths of all files uploaded will
1.21 +# be appended to the commandline.
1.22 +#
1.23 +# All files to be uploaded should be passed as commandline arguments to this
1.24 +# script. The script takes one other parameter, --base-path, which you can use
1.25 +# to indicate that files should be uploaded including their paths relative
1.26 +# to the base path.
1.27 +
1.28 +import sys, os
1.29 +from optparse import OptionParser
1.30 +from subprocess import PIPE, Popen, check_call
1.31 +
1.32 +def RequireEnvironmentVariable(v):
1.33 + """Return the value of the environment variable named v, or print
1.34 + an error and exit if it's unset (or empty)."""
1.35 + if not v in os.environ or os.environ[v] == "":
1.36 + print "Error: required environment variable %s not set" % v
1.37 + sys.exit(1)
1.38 + return os.environ[v]
1.39 +
1.40 +def OptionalEnvironmentVariable(v):
1.41 + """Return the value of the environment variable named v, or None
1.42 + if it's unset (or empty)."""
1.43 + if v in os.environ and os.environ[v] != "":
1.44 + return os.environ[v]
1.45 + return None
1.46 +
1.47 +def FixupMsysPath(path):
1.48 + """MSYS helpfully translates absolute pathnames in environment variables
1.49 + and commandline arguments into Windows native paths. This sucks if you're
1.50 + trying to pass an absolute path on a remote server. This function attempts
1.51 + to un-mangle such paths."""
1.52 + if 'OSTYPE' in os.environ and os.environ['OSTYPE'] == 'msys':
1.53 + # sort of awful, find out where our shell is (should be in msys/bin)
1.54 + # and strip the first part of that path out of the other path
1.55 + if 'SHELL' in os.environ:
1.56 + sh = os.environ['SHELL']
1.57 + msys = sh[:sh.find('/bin')]
1.58 + if path.startswith(msys):
1.59 + path = path[len(msys):]
1.60 + return path
1.61 +
1.62 +def WindowsPathToMsysPath(path):
1.63 + """Translate a Windows pathname to an MSYS pathname.
1.64 + Necessary because we call out to ssh/scp, which are MSYS binaries
1.65 + and expect MSYS paths."""
1.66 + if sys.platform != 'win32':
1.67 + return path
1.68 + (drive, path) = os.path.splitdrive(os.path.abspath(path))
1.69 + return "/" + drive[0] + path.replace('\\','/')
1.70 +
1.71 +def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key):
1.72 + """Given optional port and ssh key values, append valid OpenSSH
1.73 + commandline arguments to the list cmdline if the values are not None."""
1.74 + if port is not None:
1.75 + cmdline.append("-P%d" % port)
1.76 + if ssh_key is not None:
1.77 + # Don't interpret ~ paths - ssh can handle that on its own
1.78 + if not ssh_key.startswith('~'):
1.79 + ssh_key = WindowsPathToMsysPath(ssh_key)
1.80 + cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
1.81 +
1.82 +def DoSSHCommand(command, user, host, port=None, ssh_key=None):
1.83 + """Execute command on user@host using ssh. Optionally use
1.84 + port and ssh_key, if provided."""
1.85 + cmdline = ["ssh"]
1.86 + AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
1.87 + cmdline.extend(["%s@%s" % (user, host), command])
1.88 + cmd = Popen(cmdline, stdout=PIPE)
1.89 + retcode = cmd.wait()
1.90 + if retcode != 0:
1.91 + raise Exception("Command %s returned non-zero exit code: %i" % \
1.92 + (cmdline, retcode))
1.93 + return cmd.stdout.read().strip()
1.94 +
1.95 +def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
1.96 + """Upload file to user@host:remote_path using scp. Optionally use
1.97 + port and ssh_key, if provided."""
1.98 + cmdline = ["scp"]
1.99 + AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
1.100 + cmdline.extend([WindowsPathToMsysPath(file),
1.101 + "%s@%s:%s" % (user, host, remote_path)])
1.102 + check_call(cmdline)
1.103 +
1.104 +def GetRemotePath(path, local_file, base_path):
1.105 + """Given a remote path to upload to, a full path to a local file, and an
1.106 + optional full path that is a base path of the local file, construct the
1.107 + full remote path to place the file in. If base_path is not None, include
1.108 + the relative path from base_path to file."""
1.109 + if base_path is None or not local_file.startswith(base_path):
1.110 + return path
1.111 + dir = os.path.dirname(local_file)
1.112 + # strip base_path + extra slash and make it unixy
1.113 + dir = dir[len(base_path)+1:].replace('\\','/')
1.114 + return path + dir
1.115 +
1.116 +def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None):
1.117 + """Upload each file in the list files to user@host:path. Optionally pass
1.118 + port and ssh_key to the ssh commands. If base_path is not None, upload
1.119 + files including their path relative to base_path. If upload_to_temp_dir is
1.120 + True files will be uploaded to a temporary directory on the remote server.
1.121 + Generally, you should have a post upload command specified in these cases
1.122 + that can move them around to their correct location(s).
1.123 + If post_upload_command is not None, execute that command on the remote host
1.124 + after uploading all files, passing it the upload path, and the full paths to
1.125 + all files uploaded.
1.126 + If verbose is True, print status updates while working."""
1.127 + if upload_to_temp_dir:
1.128 + path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
1.129 + if not path.endswith("/"):
1.130 + path += "/"
1.131 + if base_path is not None:
1.132 + base_path = os.path.abspath(base_path)
1.133 + remote_files = []
1.134 + try:
1.135 + for file in files:
1.136 + file = os.path.abspath(file)
1.137 + if not os.path.isfile(file):
1.138 + raise IOError("File not found: %s" % file)
1.139 + # first ensure that path exists remotely
1.140 + remote_path = GetRemotePath(path, file, base_path)
1.141 + DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
1.142 + if verbose:
1.143 + print "Uploading " + file
1.144 + DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
1.145 + remote_files.append(remote_path + '/' + os.path.basename(file))
1.146 + if post_upload_command is not None:
1.147 + if verbose:
1.148 + print "Running post-upload command: " + post_upload_command
1.149 + file_list = '"' + '" "'.join(remote_files) + '"'
1.150 + DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
1.151 + finally:
1.152 + if upload_to_temp_dir:
1.153 + DoSSHCommand("rm -rf %s" % path, user, host, port=port,
1.154 + ssh_key=ssh_key)
1.155 + if verbose:
1.156 + print "Upload complete"
1.157 +
1.158 +if __name__ == '__main__':
1.159 + host = RequireEnvironmentVariable('UPLOAD_HOST')
1.160 + user = RequireEnvironmentVariable('UPLOAD_USER')
1.161 + path = OptionalEnvironmentVariable('UPLOAD_PATH')
1.162 + upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP')
1.163 + port = OptionalEnvironmentVariable('UPLOAD_PORT')
1.164 + if port is not None:
1.165 + port = int(port)
1.166 + key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY')
1.167 + post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD')
1.168 + if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
1.169 + print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
1.170 + "defined."
1.171 + sys.exit(1)
1.172 + if sys.platform == 'win32':
1.173 + if path is not None:
1.174 + path = FixupMsysPath(path)
1.175 + if post_upload_command is not None:
1.176 + post_upload_command = FixupMsysPath(post_upload_command)
1.177 +
1.178 + parser = OptionParser(usage="usage: %prog [options] <files>")
1.179 + parser.add_option("-b", "--base-path",
1.180 + action="store", dest="base_path",
1.181 + help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.")
1.182 + (options, args) = parser.parse_args()
1.183 + if len(args) < 1:
1.184 + print "You must specify at least one file to upload"
1.185 + sys.exit(1)
1.186 + try:
1.187 + UploadFiles(user, host, path, args, base_path=options.base_path,
1.188 + port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
1.189 + post_upload_command=post_upload_command,
1.190 + verbose=True)
1.191 + except IOError, (strerror):
1.192 + print strerror
1.193 + sys.exit(1)
1.194 + except Exception, (err):
1.195 + print err
1.196 + sys.exit(2)
