1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/tools/update-packaging/make_incremental_updates.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,542 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +import os
1.9 +import shutil
1.10 +import sha
1.11 +from os.path import join, getsize
1.12 +from stat import *
1.13 +import re
1.14 +import sys
1.15 +import getopt
1.16 +import time
1.17 +import datetime
1.18 +import bz2
1.19 +import string
1.20 +import tempfile
1.21 +
1.22 +class PatchInfo:
1.23 + """ Represents the meta-data associated with a patch
1.24 + work_dir = working dir where files are stored for this patch
1.25 + archive_files = list of files to include in this patch
1.26 + manifestv2 = set of manifest version 2 patch instructions
1.27 + manifestv3 = set of manifest version 3 patch instructions
1.28 + file_exclusion_list =
1.29 + files to exclude from this patch. names without slashes will be
1.30 + excluded anywhere in the directory hiearchy. names with slashes
1.31 + will only be excluded at that exact path
1.32 + """
1.33 + def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
1.34 + self.work_dir=work_dir
1.35 + self.archive_files=[]
1.36 + self.manifestv2=[]
1.37 + self.manifestv3=[]
1.38 + self.file_exclusion_list=file_exclusion_list
1.39 + self.path_exclusion_list=path_exclusion_list
1.40 +
1.41 + def append_add_instruction(self, filename):
1.42 + """ Appends an add instruction for this patch.
1.43 + if filename starts with distribution/extensions/.*/ this will add an
1.44 + add-if instruction that will add the file if the parent directory
1.45 + of the file exists. This was ported from
1.46 + mozilla/tools/update-packaging/common.sh's make_add_instruction.
1.47 + """
1.48 + m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
1.49 + if m:
1.50 + # Directory immediately following extensions is used for the test
1.51 + testdir = m.group(1)
1.52 + print ' add-if "'+testdir+'" "'+filename+'"'
1.53 + self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
1.54 + self.manifestv3.append('add-if "'+testdir+'" "'+filename+'"')
1.55 + else:
1.56 + print ' add "'+filename+'"'
1.57 + self.manifestv2.append('add "'+filename+'"')
1.58 + self.manifestv3.append('add "'+filename+'"')
1.59 +
1.60 + def append_add_if_not_instruction(self, filename):
1.61 + """ Appends an add-if-not instruction to the version 3 manifest for this patch.
1.62 + This was ported from mozilla/tools/update-packaging/common.sh's
1.63 + make_add_if_not_instruction.
1.64 + """
1.65 + print ' add-if-not "'+filename+'" "'+filename+'"'
1.66 + self.manifestv3.append('add-if-not "'+filename+'" "'+filename+'"')
1.67 +
1.68 + def append_patch_instruction(self, filename, patchname):
1.69 + """ Appends a patch instruction for this patch.
1.70 +
1.71 + filename = file to patch
1.72 + patchname = patchfile to apply to file
1.73 +
1.74 + if filename starts with distribution/extensions/.*/ this will add a
1.75 + patch-if instruction that will patch the file if the parent
1.76 + directory of the file exists. This was ported from
1.77 + mozilla/tools/update-packaging/common.sh's make_patch_instruction.
1.78 + """
1.79 + m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
1.80 + if m:
1.81 + testdir = m.group(1)
1.82 + print ' patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"'
1.83 + self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
1.84 + self.manifestv3.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
1.85 + else:
1.86 + print ' patch "'+patchname+'" "'+filename+'"'
1.87 + self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
1.88 + self.manifestv3.append('patch "'+patchname+'" "'+filename+'"')
1.89 +
1.90 + def append_remove_instruction(self, filename):
1.91 + """ Appends an remove instruction for this patch.
1.92 + This was ported from
1.93 + mozilla/tools/update-packaging/common.sh/make_remove_instruction
1.94 + """
1.95 + if filename.endswith("/"):
1.96 + print ' rmdir "'+filename+'"'
1.97 + self.manifestv2.append('rmdir "'+filename+'"')
1.98 + self.manifestv3.append('rmdir "'+filename+'"')
1.99 + elif filename.endswith("/*"):
1.100 + filename = filename[:-1]
1.101 + print ' rmrfdir "'+filename+'"'
1.102 + self.manifestv2.append('rmrfdir "'+filename+'"')
1.103 + self.manifestv3.append('rmrfdir "'+filename+'"')
1.104 + else:
1.105 + print ' remove "'+filename+'"'
1.106 + self.manifestv2.append('remove "'+filename+'"')
1.107 + self.manifestv3.append('remove "'+filename+'"')
1.108 +
1.109 + def create_manifest_files(self):
1.110 + """ Create the v2 manifest file in the root of the work_dir """
1.111 + manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
1.112 + manifest_file = open(manifest_file_path, "wb")
1.113 + manifest_file.writelines("type \"partial\"\n")
1.114 + manifest_file.writelines(string.join(self.manifestv2, '\n'))
1.115 + manifest_file.writelines("\n")
1.116 + manifest_file.close()
1.117 +
1.118 + bzip_file(manifest_file_path)
1.119 + self.archive_files.append('"updatev2.manifest"')
1.120 +
1.121 + """ Create the v3 manifest file in the root of the work_dir """
1.122 + manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
1.123 + manifest_file = open(manifest_file_path, "wb")
1.124 + manifest_file.writelines("type \"partial\"\n")
1.125 + manifest_file.writelines(string.join(self.manifestv3, '\n'))
1.126 + manifest_file.writelines("\n")
1.127 + manifest_file.close()
1.128 +
1.129 + bzip_file(manifest_file_path)
1.130 + self.archive_files.append('"updatev3.manifest"')
1.131 +
1.132 + def build_marfile_entry_hash(self, root_path):
1.133 + """ Iterates through the root_path, creating a MarFileEntry for each file
1.134 + and directory in that path. Excludes any filenames in the file_exclusion_list
1.135 + """
1.136 + mar_entry_hash = {}
1.137 + filename_set = set()
1.138 + dirname_set = set()
1.139 + for root, dirs, files in os.walk(root_path):
1.140 + for name in files:
1.141 + # filename is the relative path from root directory
1.142 + partial_path = root[len(root_path)+1:]
1.143 + if name not in self.file_exclusion_list:
1.144 + filename = os.path.join(partial_path, name)
1.145 + if "/"+filename not in self.path_exclusion_list:
1.146 + mar_entry_hash[filename]=MarFileEntry(root_path, filename)
1.147 + filename_set.add(filename)
1.148 +
1.149 + for name in dirs:
1.150 + # dirname is the relative path from root directory
1.151 + partial_path = root[len(root_path)+1:]
1.152 + if name not in self.file_exclusion_list:
1.153 + dirname = os.path.join(partial_path, name)
1.154 + if "/"+dirname not in self.path_exclusion_list:
1.155 + dirname = dirname+"/"
1.156 + mar_entry_hash[dirname]=MarFileEntry(root_path, dirname)
1.157 + dirname_set.add(dirname)
1.158 +
1.159 + return mar_entry_hash, filename_set, dirname_set
1.160 +
1.161 +
1.162 +class MarFileEntry:
1.163 + """Represents a file inside a Mozilla Archive Format (MAR)
1.164 + abs_path = abspath to the the file
1.165 + name = relative path within the mar. e.g.
1.166 + foo.mar/dir/bar.txt extracted into /tmp/foo:
1.167 + abs_path=/tmp/foo/dir/bar.txt
1.168 + name = dir/bar.txt
1.169 + """
1.170 + def __init__(self, root, name):
1.171 + """root = path the the top of the mar
1.172 + name = relative path within the mar"""
1.173 + self.name=name.replace("\\", "/")
1.174 + self.abs_path=os.path.join(root,name)
1.175 + self.sha_cache=None
1.176 +
1.177 + def __str__(self):
1.178 + return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
1.179 +
1.180 + def calc_file_sha_digest(self, filename):
1.181 + """ Returns sha digest of given filename"""
1.182 + file_content = open(filename, 'r').read()
1.183 + return sha.new(file_content).digest()
1.184 +
1.185 + def sha(self):
1.186 + """ Returns sha digest of file repreesnted by this _marfile_entry"""
1.187 + if not self.sha_cache:
1.188 + self.sha_cache=self.calc_file_sha_digest(self.abs_path)
1.189 + return self.sha_cache
1.190 +
1.191 +def exec_shell_cmd(cmd):
1.192 + """Execs shell cmd and raises an exception if the cmd fails"""
1.193 + if (os.system(cmd)):
1.194 + raise Exception, "cmd failed "+cmd
1.195 +
1.196 +
1.197 +def copy_file(src_file_abs_path, dst_file_abs_path):
1.198 + """ Copies src to dst creating any parent dirs required in dst first """
1.199 + dst_file_dir=os.path.dirname(dst_file_abs_path)
1.200 + if not os.path.exists(dst_file_dir):
1.201 + os.makedirs(dst_file_dir)
1.202 + # Copy the file over
1.203 + shutil.copy2(src_file_abs_path, dst_file_abs_path)
1.204 +
1.205 +def bzip_file(filename):
1.206 + """ Bzip's the file in place. The original file is replaced with a bzip'd version of itself
1.207 + assumes the path is absolute"""
1.208 + exec_shell_cmd('bzip2 -z9 "' + filename+'"')
1.209 + os.rename(filename+".bz2",filename)
1.210 +
1.211 +def bunzip_file(filename):
1.212 + """ Bzip's the file in palce. The original file is replaced with a bunzip'd version of itself.
1.213 + doesn't matter if the filename ends in .bz2 or not"""
1.214 + if not filename.endswith(".bz2"):
1.215 + os.rename(filename, filename+".bz2")
1.216 + filename=filename+".bz2"
1.217 + exec_shell_cmd('bzip2 -d "' + filename+'"')
1.218 +
1.219 +
1.220 +def extract_mar(filename, work_dir):
1.221 + """ Extracts the marfile intot he work_dir
1.222 + assumes work_dir already exists otherwise will throw osError"""
1.223 + print "Extracting "+filename+" to "+work_dir
1.224 + saved_path = os.getcwd()
1.225 + try:
1.226 + os.chdir(work_dir)
1.227 + exec_shell_cmd("mar -x "+filename)
1.228 + finally:
1.229 + os.chdir(saved_path)
1.230 +
1.231 +def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
1.232 + """ Creates the partial patch file and manifest entry for the pair of files passed in
1.233 + """
1.234 + if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
1.235 + print 'diffing "'+from_marfile_entry.name+'\"'
1.236 + #bunzip to/from
1.237 + bunzip_file(from_marfile_entry.abs_path)
1.238 + bunzip_file(to_marfile_entry.abs_path)
1.239 +
1.240 + # The patch file will be created in the working directory with the
1.241 + # name of the file in the mar + .patch
1.242 + patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch")
1.243 + patch_file_dir=os.path.dirname(patch_file_abs_path)
1.244 + if not os.path.exists(patch_file_dir):
1.245 + os.makedirs(patch_file_dir)
1.246 +
1.247 + # Create bzip'd patch file
1.248 + exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
1.249 + bzip_file(patch_file_abs_path)
1.250 +
1.251 + # Create bzip's full file
1.252 + full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name)
1.253 + shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
1.254 + bzip_file(full_file_abs_path)
1.255 +
1.256 + if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
1.257 + # Patch is smaller than file. Remove the file and add patch to manifest
1.258 + os.remove(full_file_abs_path)
1.259 + file_in_manifest_name = from_marfile_entry.name+".patch"
1.260 + file_in_manifest_abspath = patch_file_abs_path
1.261 + patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
1.262 + else:
1.263 + # File is smaller than patch. Remove the patch and add file to manifest
1.264 + os.remove(patch_file_abs_path)
1.265 + file_in_manifest_name = from_marfile_entry.name
1.266 + file_in_manifest_abspath = full_file_abs_path
1.267 + patch_info.append_add_instruction(file_in_manifest_name)
1.268 +
1.269 + shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
1.270 + patch_info.archive_files.append('"'+file_in_manifest_name+'"')
1.271 + else:
1.272 + filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
1.273 + # We've already calculated the patch for this pair of files.
1.274 + if (filename.endswith(".patch")):
1.275 + # print "skipping diff: "+from_marfile_entry.name
1.276 + # Patch was smaller than file - add patch instruction to manifest
1.277 + file_in_manifest_name = to_marfile_entry.name+'.patch';
1.278 + patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
1.279 + else:
1.280 + # File was smaller than file - add file to manifest
1.281 + file_in_manifest_name = to_marfile_entry.name
1.282 + patch_info.append_add_instruction(file_in_manifest_name)
1.283 + # Copy the pre-calculated file into our new patch work aread
1.284 + copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
1.285 + patch_info.archive_files.append('"'+file_in_manifest_name+'"')
1.286 +
1.287 +def create_add_patch_for_file(to_marfile_entry, patch_info):
1.288 + """ Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
1.289 + copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
1.290 + patch_info.append_add_instruction(to_marfile_entry.name)
1.291 + patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
1.292 +
1.293 +def create_add_if_not_patch_for_file(to_marfile_entry, patch_info):
1.294 + """ Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """
1.295 + copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
1.296 + patch_info.append_add_if_not_instruction(to_marfile_entry.name)
1.297 + patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
1.298 +
1.299 +def process_explicit_remove_files(dir_path, patch_info):
1.300 + """ Looks for a 'removed-files' file in the dir_path. If the removed-files does not exist
1.301 + this will throw. If found adds the removed-files
1.302 + found in that file to the patch_info"""
1.303 +
1.304 + # Windows and linux have this file at the root of the dir
1.305 + list_file_path = os.path.join(dir_path, "removed-files")
1.306 + prefix=""
1.307 + if not os.path.exists(list_file_path):
1.308 + # On Mac removed-files contains relative paths from Contents/MacOS/
1.309 + prefix= "Contents/MacOS"
1.310 + list_file_path = os.path.join(dir_path, prefix+"/removed-files")
1.311 +
1.312 + if (os.path.exists(list_file_path)):
1.313 + list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
1.314 +
1.315 + lines = []
1.316 + for line in list_file:
1.317 + lines.append(line.strip())
1.318 +
1.319 + lines.sort(reverse=True)
1.320 + for line in lines:
1.321 + # Exclude any blank and comment lines.
1.322 + if line and not line.startswith("#"):
1.323 + if prefix != "":
1.324 + if line.startswith("../"):
1.325 + line = line.replace("../../", "")
1.326 + line = line.replace("../", "Contents/")
1.327 + else:
1.328 + line = os.path.join(prefix,line)
1.329 + # Python on windows uses \ for path separators and the update
1.330 + # manifests expects / for path separators on all platforms.
1.331 + line = line.replace("\\", "/")
1.332 + patch_info.append_remove_instruction(line)
1.333 +
1.334 +def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list):
1.335 + """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
1.336 + # Cannocolize the paths for safey
1.337 + from_dir_path = os.path.abspath(from_dir_path)
1.338 + to_dir_path = os.path.abspath(to_dir_path)
1.339 + # Create a hashtable of the from and to directories
1.340 + from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
1.341 + to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
1.342 + # Require that the precomplete file is included in the complete update
1.343 + if "precomplete" not in to_file_set:
1.344 + raise Exception, "missing precomplete file in: "+to_dir_path
1.345 + # Create a list of the forced updates
1.346 + forced_list = forced_updates.strip().split('|')
1.347 + forced_list.append("precomplete")
1.348 +
1.349 + # Files which exist in both sets need to be patched
1.350 + patch_filenames = list(from_file_set.intersection(to_file_set))
1.351 + patch_filenames.sort(reverse=True)
1.352 + for filename in patch_filenames:
1.353 + from_marfile_entry = from_dir_hash[filename]
1.354 + to_marfile_entry = to_dir_hash[filename]
1.355 + if os.path.basename(filename) in add_if_not_list:
1.356 + # This filename is in the add if not list, explicitly add-if-not
1.357 + create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
1.358 + elif filename in forced_list:
1.359 + print 'Forcing "'+filename+'"'
1.360 + # This filename is in the forced list, explicitly add
1.361 + create_add_patch_for_file(to_dir_hash[filename], patch_info)
1.362 + else:
1.363 + if from_marfile_entry.sha() != to_marfile_entry.sha():
1.364 + # Not the same - calculate a patch
1.365 + create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
1.366 +
1.367 + # files in to_dir not in from_dir need to added
1.368 + add_filenames = list(to_file_set - from_file_set)
1.369 + add_filenames.sort(reverse=True)
1.370 + for filename in add_filenames:
1.371 + if os.path.basename(filename) in add_if_not_list:
1.372 + create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
1.373 + else:
1.374 + create_add_patch_for_file(to_dir_hash[filename], patch_info)
1.375 +
1.376 + # files in from_dir not in to_dir need to be removed
1.377 + remove_filenames = list(from_file_set - to_file_set)
1.378 + remove_filenames.sort(reverse=True)
1.379 + for filename in remove_filenames:
1.380 + patch_info.append_remove_instruction(from_dir_hash[filename].name)
1.381 +
1.382 + process_explicit_remove_files(to_dir_path, patch_info)
1.383 +
1.384 + # directories in from_dir not in to_dir need to be removed
1.385 + remove_dirnames = list(from_dir_set - to_dir_set)
1.386 + remove_dirnames.sort(reverse=True)
1.387 + for dirname in remove_dirnames:
1.388 + patch_info.append_remove_instruction(from_dir_hash[dirname].name)
1.389 +
1.390 + # Construct the Manifest files
1.391 + patch_info.create_manifest_files()
1.392 +
1.393 + # And construct the mar
1.394 + mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
1.395 + exec_shell_cmd(mar_cmd)
1.396 +
1.397 + # Copy mar to final destination
1.398 + patch_file_dir = os.path.split(patch_filename)[0]
1.399 + if not os.path.exists(patch_file_dir):
1.400 + os.makedirs(patch_file_dir)
1.401 + shutil.copy2(os.path.join(patch_info.work_dir,"output.mar"), patch_filename)
1.402 + return patch_filename
1.403 +
1.404 +def usage():
1.405 + print "-h for help"
1.406 + print "-f for patchlist_file"
1.407 +
1.408 +def get_buildid(work_dir, platform):
1.409 + """ extracts buildid from MAR
1.410 + TODO: this should handle 1.8 branch too
1.411 + """
1.412 + if platform == 'mac':
1.413 + ini = '%s/Contents/MacOS/application.ini' % work_dir
1.414 + else:
1.415 + ini = '%s/application.ini' % work_dir
1.416 + if not os.path.exists(ini):
1.417 + print 'WARNING: application.ini not found, cannot find build ID'
1.418 + return ''
1.419 + file = bz2.BZ2File(ini)
1.420 + for line in file:
1.421 + if line.find('BuildID') == 0:
1.422 + return line.strip().split('=')[1]
1.423 + print 'WARNING: cannot find build ID in application.ini'
1.424 + return ''
1.425 +
1.426 +def decode_filename(filepath):
1.427 + """ Breaks filename/dir structure into component parts based on regex
1.428 + for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
1.429 + Or linux-i686/en-US/firefox-3.0b3.complete.mar
1.430 + Returns dict with keys product, version, locale, platform, type
1.431 + """
1.432 + try:
1.433 + m = re.search(
1.434 + '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
1.435 + os.path.basename(filepath))
1.436 + return m.groupdict()
1.437 + except Exception, exc:
1.438 + try:
1.439 + m = re.search(
1.440 + '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
1.441 + filepath)
1.442 + return m.groupdict()
1.443 + except:
1.444 + raise Exception("could not parse filepath %s: %s" % (filepath, exc))
1.445 +
1.446 +def create_partial_patches(patches):
1.447 + """ Given the patches generates a set of partial patches"""
1.448 + shas = {}
1.449 +
1.450 + work_dir_root = None
1.451 + metadata = []
1.452 + try:
1.453 + work_dir_root = tempfile.mkdtemp('-fastmode', 'tmp', os.getcwd())
1.454 + print "Building patches using work dir: %s" % (work_dir_root)
1.455 +
1.456 + # Iterate through every patch set in the patch file
1.457 + patch_num = 1
1.458 + for patch in patches:
1.459 + startTime = time.time()
1.460 +
1.461 + from_filename,to_filename,patch_filename,forced_updates = patch.split(",")
1.462 + from_filename,to_filename,patch_filename = os.path.abspath(from_filename),os.path.abspath(to_filename),os.path.abspath(patch_filename)
1.463 +
1.464 + # Each patch iteration uses its own work dir
1.465 + work_dir = os.path.join(work_dir_root,str(patch_num))
1.466 + os.mkdir(work_dir)
1.467 +
1.468 + # Extract from mar into from dir
1.469 + work_dir_from = os.path.join(work_dir,"from");
1.470 + os.mkdir(work_dir_from)
1.471 + extract_mar(from_filename,work_dir_from)
1.472 + from_decoded = decode_filename(from_filename)
1.473 + from_buildid = get_buildid(work_dir_from, from_decoded['platform'])
1.474 + from_shasum = sha.sha(open(from_filename).read()).hexdigest()
1.475 + from_size = str(os.path.getsize(to_filename))
1.476 +
1.477 + # Extract to mar into to dir
1.478 + work_dir_to = os.path.join(work_dir,"to")
1.479 + os.mkdir(work_dir_to)
1.480 + extract_mar(to_filename, work_dir_to)
1.481 + to_decoded = decode_filename(from_filename)
1.482 + to_buildid = get_buildid(work_dir_to, to_decoded['platform'])
1.483 + to_shasum = sha.sha(open(to_filename).read()).hexdigest()
1.484 + to_size = str(os.path.getsize(to_filename))
1.485 +
1.486 + mar_extract_time = time.time()
1.487 +
1.488 + partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest','removed-files'],['/readme.txt']),forced_updates,['channel-prefs.js','update-settings.ini'])
1.489 + partial_buildid = to_buildid
1.490 + partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
1.491 + partial_size = str(os.path.getsize(partial_filename))
1.492 +
1.493 + metadata.append({
1.494 + 'to_filename': os.path.basename(to_filename),
1.495 + 'from_filename': os.path.basename(from_filename),
1.496 + 'partial_filename': os.path.basename(partial_filename),
1.497 + 'to_buildid':to_buildid,
1.498 + 'from_buildid':from_buildid,
1.499 + 'to_sha1sum':to_shasum,
1.500 + 'from_sha1sum':from_shasum,
1.501 + 'partial_sha1sum':partial_shasum,
1.502 + 'to_size':to_size,
1.503 + 'from_size':from_size,
1.504 + 'partial_size':partial_size,
1.505 + 'to_version':to_decoded['version'],
1.506 + 'from_version':from_decoded['version'],
1.507 + 'locale':from_decoded['locale'],
1.508 + 'platform':from_decoded['platform'],
1.509 + })
1.510 + print "done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),str(len(patches)),mar_extract_time-startTime,time.time()-mar_extract_time,time.time()-startTime)
1.511 + patch_num += 1
1.512 + return metadata
1.513 + finally:
1.514 + # If we fail or get a ctrl-c during run be sure to clean up temp dir
1.515 + if (work_dir_root and os.path.exists(work_dir_root)):
1.516 + shutil.rmtree(work_dir_root)
1.517 +
1.518 +def main(argv):
1.519 + patchlist_file = None
1.520 + try:
1.521 + opts, args = getopt.getopt(argv, "hf:", ["help", "patchlist_file="])
1.522 + for opt, arg in opts:
1.523 + if opt in ("-h", "--help"):
1.524 + usage()
1.525 + sys.exit()
1.526 + elif opt in ("-f", "--patchlist_file"):
1.527 + patchlist_file = arg
1.528 + except getopt.GetoptError:
1.529 + usage()
1.530 + sys.exit(2)
1.531 +
1.532 + if not patchlist_file:
1.533 + usage()
1.534 + sys.exit(2)
1.535 +
1.536 + patches = []
1.537 + f = open(patchlist_file, 'r')
1.538 + for line in f.readlines():
1.539 + patches.append(line)
1.540 + f.close()
1.541 + create_partial_patches(patches)
1.542 +
1.543 +if __name__ == "__main__":
1.544 + main(sys.argv[1:])
1.545 +
