michael@0: # This Source Code Form is subject to the terms of the Mozilla Public
michael@0: # License, v. 2.0. If a copy of the MPL was not distributed with this
michael@0: # file, You can obtain one at http://mozilla.org/MPL/2.0/.
michael@0: 
michael@0: import os
michael@0: import shutil
michael@0: import sha
michael@0: from os.path import join, getsize
michael@0: from stat import *
michael@0: import re
michael@0: import sys
michael@0: import getopt
michael@0: import time
michael@0: import datetime
michael@0: import bz2
michael@0: import string
michael@0: import tempfile
michael@0: 
michael@0: class PatchInfo:
michael@0:     """ Represents the meta-data associated with a patch
michael@0:         work_dir = working dir where files are stored for this patch
michael@0:         archive_files = list of files to include in this patch
michael@0:         manifestv2 = set of manifest version 2 patch instructions
michael@0:         manifestv3 = set of manifest version 3 patch instructions
michael@0:         file_exclusion_list = 
michael@0:         files to exclude from this patch. names without slashes will be
michael@0:         excluded anywhere in the directory hiearchy.   names with slashes
michael@0:         will only be excluded at that exact path
michael@0:         """
michael@0:     def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
michael@0:         self.work_dir=work_dir
michael@0:         self.archive_files=[]
michael@0:         self.manifestv2=[]
michael@0:         self.manifestv3=[]
michael@0:         self.file_exclusion_list=file_exclusion_list
michael@0:         self.path_exclusion_list=path_exclusion_list
michael@0: 
michael@0:     def append_add_instruction(self, filename):
michael@0:         """ Appends an add instruction for this patch.
michael@0:             if filename starts with distribution/extensions/.*/ this will add an
michael@0:             add-if instruction that will add the file if the parent directory
michael@0:             of the file exists.  This was ported from
michael@0:             mozilla/tools/update-packaging/common.sh's make_add_instruction.
michael@0:         """
michael@0:         m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
michael@0:         if m:
michael@0:             # Directory immediately following extensions is used for the test
michael@0:             testdir = m.group(1)
michael@0:             print '     add-if "'+testdir+'" "'+filename+'"'
michael@0:             self.manifestv2.append('add-if "'+testdir+'" "'+filename+'"')
michael@0:             self.manifestv3.append('add-if "'+testdir+'" "'+filename+'"')
michael@0:         else:
michael@0:             print '        add "'+filename+'"'
michael@0:             self.manifestv2.append('add "'+filename+'"')
michael@0:             self.manifestv3.append('add "'+filename+'"')
michael@0: 
michael@0:     def append_add_if_not_instruction(self, filename):
michael@0:         """ Appends an add-if-not instruction to the version 3 manifest for this patch.
michael@0:             This was ported from mozilla/tools/update-packaging/common.sh's
michael@0:             make_add_if_not_instruction.
michael@0:         """
michael@0:         print ' add-if-not "'+filename+'" "'+filename+'"'
michael@0:         self.manifestv3.append('add-if-not "'+filename+'" "'+filename+'"')
michael@0: 
michael@0:     def append_patch_instruction(self, filename, patchname):
michael@0:         """ Appends a patch instruction for this patch.
michael@0: 
michael@0:             filename = file to patch
michael@0:             patchname = patchfile to apply to file
michael@0: 
michael@0:             if filename starts with distribution/extensions/.*/ this will add a
michael@0:             patch-if instruction that will patch the file if the parent
michael@0:             directory of the file exists. This was ported from
michael@0:             mozilla/tools/update-packaging/common.sh's make_patch_instruction.
michael@0:         """
michael@0:         m = re.match("((?:|.*/)distribution/extensions/.*)/", filename)
michael@0:         if m:
michael@0:             testdir = m.group(1)
michael@0:             print '   patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"'
michael@0:             self.manifestv2.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
michael@0:             self.manifestv3.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
michael@0:         else:
michael@0:             print '      patch "'+patchname+'" "'+filename+'"'
michael@0:             self.manifestv2.append('patch "'+patchname+'" "'+filename+'"')
michael@0:             self.manifestv3.append('patch "'+patchname+'" "'+filename+'"')
michael@0: 
michael@0:     def append_remove_instruction(self, filename):
michael@0:         """ Appends an remove instruction for this patch.
michael@0:             This was ported from
michael@0:             mozilla/tools/update-packaging/common.sh/make_remove_instruction
michael@0:         """
michael@0:         if filename.endswith("/"):
michael@0:             print '      rmdir "'+filename+'"'
michael@0:             self.manifestv2.append('rmdir "'+filename+'"')
michael@0:             self.manifestv3.append('rmdir "'+filename+'"')
michael@0:         elif filename.endswith("/*"):
michael@0:             filename = filename[:-1]
michael@0:             print '    rmrfdir "'+filename+'"'
michael@0:             self.manifestv2.append('rmrfdir "'+filename+'"')
michael@0:             self.manifestv3.append('rmrfdir "'+filename+'"')
michael@0:         else:
michael@0:             print '     remove "'+filename+'"'
michael@0:             self.manifestv2.append('remove "'+filename+'"')
michael@0:             self.manifestv3.append('remove "'+filename+'"')
michael@0: 
michael@0:     def create_manifest_files(self):
michael@0:         """ Create the v2 manifest file in the root of the work_dir """
michael@0:         manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
michael@0:         manifest_file = open(manifest_file_path, "wb")
michael@0:         manifest_file.writelines("type \"partial\"\n")
michael@0:         manifest_file.writelines(string.join(self.manifestv2, '\n'))
michael@0:         manifest_file.writelines("\n")
michael@0:         manifest_file.close()
michael@0: 
michael@0:         bzip_file(manifest_file_path)
michael@0:         self.archive_files.append('"updatev2.manifest"')
michael@0: 
michael@0:         """ Create the v3 manifest file in the root of the work_dir """
michael@0:         manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
michael@0:         manifest_file = open(manifest_file_path, "wb")
michael@0:         manifest_file.writelines("type \"partial\"\n")
michael@0:         manifest_file.writelines(string.join(self.manifestv3, '\n'))
michael@0:         manifest_file.writelines("\n")
michael@0:         manifest_file.close()
michael@0: 
michael@0:         bzip_file(manifest_file_path)
michael@0:         self.archive_files.append('"updatev3.manifest"')
michael@0: 
michael@0:     def build_marfile_entry_hash(self, root_path):
michael@0:         """ Iterates through the root_path, creating a MarFileEntry for each file
michael@0:             and directory in that path.  Excludes any filenames in the file_exclusion_list
michael@0:         """
michael@0:         mar_entry_hash = {}
michael@0:         filename_set = set()
michael@0:         dirname_set = set()
michael@0:         for root, dirs, files in os.walk(root_path):
michael@0:             for name in files:
michael@0:                 # filename is the relative path from root directory
michael@0:                 partial_path = root[len(root_path)+1:]
michael@0:                 if name not in self.file_exclusion_list:
michael@0:                     filename = os.path.join(partial_path, name)
michael@0:                     if "/"+filename not in self.path_exclusion_list:
michael@0:                         mar_entry_hash[filename]=MarFileEntry(root_path, filename)
michael@0:                         filename_set.add(filename)
michael@0: 
michael@0:             for name in dirs:
michael@0:                 # dirname is the relative path from root directory
michael@0:                 partial_path = root[len(root_path)+1:]
michael@0:                 if name not in self.file_exclusion_list:
michael@0:                     dirname = os.path.join(partial_path, name)
michael@0:                     if "/"+dirname not in self.path_exclusion_list:
michael@0:                         dirname = dirname+"/"
michael@0:                         mar_entry_hash[dirname]=MarFileEntry(root_path, dirname)
michael@0:                         dirname_set.add(dirname)
michael@0: 
michael@0:         return mar_entry_hash, filename_set, dirname_set
michael@0:  
michael@0: 
michael@0: class MarFileEntry:
michael@0:     """Represents a file inside a Mozilla Archive Format (MAR)
michael@0:         abs_path = abspath to the the file
michael@0:         name =  relative path within the mar.  e.g.
michael@0:           foo.mar/dir/bar.txt extracted into /tmp/foo:
michael@0:             abs_path=/tmp/foo/dir/bar.txt
michael@0:             name = dir/bar.txt
michael@0:     """ 
michael@0:     def __init__(self, root, name):
michael@0:         """root = path the the top of the mar
michael@0:            name = relative path within the mar"""
michael@0:         self.name=name.replace("\\", "/")
michael@0:         self.abs_path=os.path.join(root,name)
michael@0:         self.sha_cache=None
michael@0: 
michael@0:     def __str__(self):
michael@0:         return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
michael@0: 
michael@0:     def calc_file_sha_digest(self, filename):
michael@0:         """ Returns sha digest of given filename"""
michael@0:         file_content = open(filename, 'r').read()
michael@0:         return sha.new(file_content).digest()
michael@0: 
michael@0:     def sha(self):
michael@0:         """ Returns sha digest of file repreesnted by this _marfile_entry"""
michael@0:         if not self.sha_cache:
michael@0:             self.sha_cache=self.calc_file_sha_digest(self.abs_path)
michael@0:         return self.sha_cache
michael@0: 
michael@0: def exec_shell_cmd(cmd):
michael@0:     """Execs shell cmd and raises an exception if the cmd fails"""
michael@0:     if (os.system(cmd)):
michael@0:         raise Exception, "cmd failed "+cmd
michael@0: 
michael@0: 
michael@0: def copy_file(src_file_abs_path, dst_file_abs_path):
michael@0:     """ Copies src to dst creating any parent dirs required in dst first """
michael@0:     dst_file_dir=os.path.dirname(dst_file_abs_path)
michael@0:     if not os.path.exists(dst_file_dir):
michael@0:          os.makedirs(dst_file_dir)
michael@0:     # Copy the file over
michael@0:     shutil.copy2(src_file_abs_path, dst_file_abs_path)
michael@0: 
michael@0: def bzip_file(filename):
michael@0:     """ Bzip's the file in place.  The original file is replaced with a bzip'd version of itself
michael@0:         assumes the path is absolute"""
michael@0:     exec_shell_cmd('bzip2 -z9 "' + filename+'"')
michael@0:     os.rename(filename+".bz2",filename)
michael@0: 
michael@0: def bunzip_file(filename):
michael@0:     """ Bzip's the file in palce.   The original file is replaced with a bunzip'd version of itself.
michael@0:         doesn't matter if the filename ends in .bz2 or not"""
michael@0:     if not filename.endswith(".bz2"):
michael@0:         os.rename(filename, filename+".bz2")
michael@0:         filename=filename+".bz2"
michael@0:     exec_shell_cmd('bzip2 -d "' + filename+'"') 
michael@0: 
michael@0: 
michael@0: def extract_mar(filename, work_dir): 
michael@0:     """ Extracts the marfile intot he work_dir
michael@0:         assumes work_dir already exists otherwise will throw osError"""
michael@0:     print "Extracting "+filename+" to "+work_dir
michael@0:     saved_path = os.getcwd()
michael@0:     try:
michael@0:         os.chdir(work_dir)
michael@0:         exec_shell_cmd("mar -x "+filename)
michael@0:     finally:
michael@0:         os.chdir(saved_path)
michael@0: 
michael@0: def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
michael@0:     """ Creates the partial patch file and manifest entry for the pair of files passed in
michael@0:     """
michael@0:     if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
michael@0:         print 'diffing "'+from_marfile_entry.name+'\"'
michael@0:         #bunzip to/from
michael@0:         bunzip_file(from_marfile_entry.abs_path)
michael@0:         bunzip_file(to_marfile_entry.abs_path)
michael@0: 
michael@0:         # The patch file will be created in the working directory with the
michael@0:         # name of the file in the mar + .patch
michael@0:         patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch")
michael@0:         patch_file_dir=os.path.dirname(patch_file_abs_path)
michael@0:         if not os.path.exists(patch_file_dir):
michael@0:             os.makedirs(patch_file_dir)
michael@0: 
michael@0:         # Create bzip'd patch file
michael@0:         exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
michael@0:         bzip_file(patch_file_abs_path)
michael@0: 
michael@0:         # Create bzip's full file
michael@0:         full_file_abs_path =  os.path.join(patch_info.work_dir, to_marfile_entry.name)
michael@0:         shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
michael@0:         bzip_file(full_file_abs_path)
michael@0: 
michael@0:         if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
michael@0:             # Patch is smaller than file.  Remove the file and add patch to manifest
michael@0:             os.remove(full_file_abs_path)
michael@0:             file_in_manifest_name = from_marfile_entry.name+".patch"
michael@0:             file_in_manifest_abspath = patch_file_abs_path
michael@0:             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
michael@0:         else:
michael@0:             # File is smaller than patch.  Remove the patch and add file to manifest
michael@0:             os.remove(patch_file_abs_path)
michael@0:             file_in_manifest_name = from_marfile_entry.name
michael@0:             file_in_manifest_abspath = full_file_abs_path
michael@0:             patch_info.append_add_instruction(file_in_manifest_name)
michael@0: 
michael@0:         shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
michael@0:         patch_info.archive_files.append('"'+file_in_manifest_name+'"')
michael@0:     else:
michael@0:         filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
michael@0:         # We've already calculated the patch for this pair of files.
michael@0:         if (filename.endswith(".patch")):
michael@0:             # print "skipping diff: "+from_marfile_entry.name
michael@0:             # Patch was smaller than file - add patch instruction to manifest
michael@0:             file_in_manifest_name = to_marfile_entry.name+'.patch';
michael@0:             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
michael@0:         else:
michael@0:             # File was smaller than file - add file to manifest
michael@0:             file_in_manifest_name = to_marfile_entry.name
michael@0:             patch_info.append_add_instruction(file_in_manifest_name)
michael@0:         # Copy the pre-calculated file into our new patch work aread
michael@0:         copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
michael@0:         patch_info.archive_files.append('"'+file_in_manifest_name+'"')
michael@0:  
michael@0: def create_add_patch_for_file(to_marfile_entry, patch_info):
michael@0:     """  Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
michael@0:     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
michael@0:     patch_info.append_add_instruction(to_marfile_entry.name)
michael@0:     patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
michael@0: 
michael@0: def create_add_if_not_patch_for_file(to_marfile_entry, patch_info):
michael@0:     """  Copy the file to the working dir, add the add-if-not instruction, and add it to the list of archive files """
michael@0:     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
michael@0:     patch_info.append_add_if_not_instruction(to_marfile_entry.name)
michael@0:     patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
michael@0: 
michael@0: def process_explicit_remove_files(dir_path, patch_info): 
michael@0:     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
michael@0:     this will throw.  If found adds the removed-files
michael@0:     found in that file to the patch_info"""
michael@0: 
michael@0:     # Windows and linux have this file at the root of the dir
michael@0:     list_file_path = os.path.join(dir_path, "removed-files")
michael@0:     prefix=""
michael@0:     if not os.path.exists(list_file_path):
michael@0:         # On Mac removed-files contains relative paths from Contents/MacOS/
michael@0:         prefix= "Contents/MacOS"
michael@0:         list_file_path = os.path.join(dir_path, prefix+"/removed-files")
michael@0: 
michael@0:     if (os.path.exists(list_file_path)):
michael@0:         list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
michael@0: 
michael@0:         lines = []
michael@0:         for line in list_file:
michael@0:             lines.append(line.strip())
michael@0: 
michael@0:         lines.sort(reverse=True)
michael@0:         for line in lines:
michael@0:             # Exclude any blank and comment lines.
michael@0:             if line and not line.startswith("#"):
michael@0:                 if prefix != "":
michael@0:                     if line.startswith("../"):
michael@0:                         line = line.replace("../../", "")
michael@0:                         line = line.replace("../", "Contents/")
michael@0:                     else:
michael@0:                         line = os.path.join(prefix,line)
michael@0:                 # Python on windows uses \ for path separators and the update
michael@0:                 # manifests expects / for path separators on all platforms.
michael@0:                 line = line.replace("\\", "/")
michael@0:                 patch_info.append_remove_instruction(line)
michael@0: 
michael@0: def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info, forced_updates, add_if_not_list):
michael@0:     """ Builds a partial patch by comparing the files in from_dir_path to those of to_dir_path"""
michael@0:     # Cannocolize the paths for safey
michael@0:     from_dir_path = os.path.abspath(from_dir_path)
michael@0:     to_dir_path = os.path.abspath(to_dir_path)
michael@0:     # Create a hashtable of the from  and to directories
michael@0:     from_dir_hash,from_file_set,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
michael@0:     to_dir_hash,to_file_set,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
michael@0:     # Require that the precomplete file is included in the complete update
michael@0:     if "precomplete" not in to_file_set:
michael@0:         raise Exception, "missing precomplete file in: "+to_dir_path
michael@0:     # Create a list of the forced updates 
michael@0:     forced_list = forced_updates.strip().split('|')
michael@0:     forced_list.append("precomplete")
michael@0: 
michael@0:     # Files which exist in both sets need to be patched
michael@0:     patch_filenames = list(from_file_set.intersection(to_file_set))
michael@0:     patch_filenames.sort(reverse=True)
michael@0:     for filename in patch_filenames:
michael@0:         from_marfile_entry = from_dir_hash[filename]
michael@0:         to_marfile_entry = to_dir_hash[filename]
michael@0:         if os.path.basename(filename) in add_if_not_list:
michael@0:             # This filename is in the add if not list, explicitly add-if-not
michael@0:             create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
michael@0:         elif filename in forced_list:
michael@0:             print 'Forcing "'+filename+'"'
michael@0:             # This filename is in the forced list, explicitly add
michael@0:             create_add_patch_for_file(to_dir_hash[filename], patch_info)
michael@0:         else: 
michael@0:           if from_marfile_entry.sha() != to_marfile_entry.sha():
michael@0:               # Not the same - calculate a patch
michael@0:               create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
michael@0: 
michael@0:     # files in to_dir not in from_dir need to added
michael@0:     add_filenames = list(to_file_set - from_file_set)
michael@0:     add_filenames.sort(reverse=True)
michael@0:     for filename in add_filenames:
michael@0:         if os.path.basename(filename) in add_if_not_list:
michael@0:             create_add_if_not_patch_for_file(to_dir_hash[filename], patch_info)
michael@0:         else:
michael@0:             create_add_patch_for_file(to_dir_hash[filename], patch_info)
michael@0: 
michael@0:     # files in from_dir not in to_dir need to be removed
michael@0:     remove_filenames = list(from_file_set - to_file_set)
michael@0:     remove_filenames.sort(reverse=True)
michael@0:     for filename in remove_filenames:
michael@0:         patch_info.append_remove_instruction(from_dir_hash[filename].name)
michael@0: 
michael@0:     process_explicit_remove_files(to_dir_path, patch_info)
michael@0: 
michael@0:     # directories in from_dir not in to_dir need to be removed
michael@0:     remove_dirnames = list(from_dir_set - to_dir_set)
michael@0:     remove_dirnames.sort(reverse=True)
michael@0:     for dirname in remove_dirnames:
michael@0:         patch_info.append_remove_instruction(from_dir_hash[dirname].name)
michael@0: 
michael@0:     # Construct the Manifest files
michael@0:     patch_info.create_manifest_files()
michael@0: 
michael@0:     # And construct the mar
michael@0:     mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
michael@0:     exec_shell_cmd(mar_cmd)
michael@0: 
michael@0:     # Copy mar to final destination
michael@0:     patch_file_dir = os.path.split(patch_filename)[0]
michael@0:     if not os.path.exists(patch_file_dir):
michael@0:         os.makedirs(patch_file_dir)
michael@0:     shutil.copy2(os.path.join(patch_info.work_dir,"output.mar"), patch_filename)
michael@0:     return patch_filename
michael@0: 
michael@0: def usage():
michael@0:     print "-h for help"
michael@0:     print "-f for patchlist_file"
michael@0: 
michael@0: def get_buildid(work_dir, platform):
michael@0:     """ extracts buildid from MAR
michael@0:         TODO: this should handle 1.8 branch too
michael@0:     """
michael@0:     if platform == 'mac':
michael@0:       ini = '%s/Contents/MacOS/application.ini' % work_dir
michael@0:     else:
michael@0:       ini = '%s/application.ini' % work_dir
michael@0:     if not os.path.exists(ini):
michael@0:         print 'WARNING: application.ini not found, cannot find build ID'
michael@0:         return ''
michael@0:     file = bz2.BZ2File(ini)
michael@0:     for line in file:
michael@0:       if line.find('BuildID') == 0:
michael@0:         return line.strip().split('=')[1]
michael@0:     print 'WARNING: cannot find build ID in application.ini'
michael@0:     return ''
michael@0: 
michael@0: def decode_filename(filepath):
michael@0:     """ Breaks filename/dir structure into component parts based on regex
michael@0:         for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
michael@0:         Or linux-i686/en-US/firefox-3.0b3.complete.mar
michael@0:         Returns dict with keys product, version, locale, platform, type
michael@0:     """
michael@0:     try:
michael@0:       m = re.search(
michael@0:         '(?P<product>\w+)(-)(?P<version>\w+\.\w+(\.\w+){0,2})(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
michael@0:       os.path.basename(filepath))
michael@0:       return m.groupdict()
michael@0:     except Exception, exc:
michael@0:       try:
michael@0:         m = re.search(
michael@0:           '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
michael@0:         filepath)
michael@0:         return m.groupdict()
michael@0:       except:
michael@0:         raise Exception("could not parse filepath %s: %s" % (filepath, exc))
michael@0: 
michael@0: def create_partial_patches(patches):
michael@0:     """ Given the patches generates a set of partial patches"""
michael@0:     shas = {}
michael@0: 
michael@0:     work_dir_root = None
michael@0:     metadata = []
michael@0:     try:
michael@0:         work_dir_root = tempfile.mkdtemp('-fastmode', 'tmp', os.getcwd())
michael@0:         print "Building patches using work dir: %s" % (work_dir_root)
michael@0:  
michael@0:         # Iterate through every patch set in the patch file
michael@0:         patch_num = 1
michael@0:         for patch in patches:
michael@0:             startTime = time.time()
michael@0: 
michael@0:             from_filename,to_filename,patch_filename,forced_updates = patch.split(",")
michael@0:             from_filename,to_filename,patch_filename = os.path.abspath(from_filename),os.path.abspath(to_filename),os.path.abspath(patch_filename)
michael@0: 
michael@0:             # Each patch iteration uses its own work dir
michael@0:             work_dir = os.path.join(work_dir_root,str(patch_num))
michael@0:             os.mkdir(work_dir)
michael@0: 
michael@0:             # Extract from mar into from dir
michael@0:             work_dir_from =  os.path.join(work_dir,"from");
michael@0:             os.mkdir(work_dir_from)
michael@0:             extract_mar(from_filename,work_dir_from)
michael@0:             from_decoded = decode_filename(from_filename)
michael@0:             from_buildid = get_buildid(work_dir_from, from_decoded['platform'])
michael@0:             from_shasum = sha.sha(open(from_filename).read()).hexdigest()
michael@0:             from_size = str(os.path.getsize(to_filename))
michael@0: 
michael@0:             # Extract to mar into to dir
michael@0:             work_dir_to =  os.path.join(work_dir,"to")
michael@0:             os.mkdir(work_dir_to)
michael@0:             extract_mar(to_filename, work_dir_to)
michael@0:             to_decoded = decode_filename(from_filename)
michael@0:             to_buildid = get_buildid(work_dir_to, to_decoded['platform'])
michael@0:             to_shasum = sha.sha(open(to_filename).read()).hexdigest()
michael@0:             to_size = str(os.path.getsize(to_filename))
michael@0: 
michael@0:             mar_extract_time = time.time()
michael@0: 
michael@0:             partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest','removed-files'],['/readme.txt']),forced_updates,['channel-prefs.js','update-settings.ini'])
michael@0:             partial_buildid = to_buildid
michael@0:             partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
michael@0:             partial_size = str(os.path.getsize(partial_filename))
michael@0: 
michael@0:             metadata.append({
michael@0:              'to_filename': os.path.basename(to_filename),
michael@0:              'from_filename': os.path.basename(from_filename),
michael@0:              'partial_filename': os.path.basename(partial_filename),
michael@0:              'to_buildid':to_buildid, 
michael@0:              'from_buildid':from_buildid, 
michael@0:              'to_sha1sum':to_shasum, 
michael@0:              'from_sha1sum':from_shasum, 
michael@0:              'partial_sha1sum':partial_shasum, 
michael@0:              'to_size':to_size,
michael@0:              'from_size':from_size,
michael@0:              'partial_size':partial_size,
michael@0:              'to_version':to_decoded['version'],
michael@0:              'from_version':from_decoded['version'],
michael@0:              'locale':from_decoded['locale'],
michael@0:              'platform':from_decoded['platform'],
michael@0:             })
michael@0:             print "done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),str(len(patches)),mar_extract_time-startTime,time.time()-mar_extract_time,time.time()-startTime)
michael@0:             patch_num += 1
michael@0:         return metadata
michael@0:     finally:
michael@0:         # If we fail or get a ctrl-c during run be sure to clean up temp dir
michael@0:         if (work_dir_root and os.path.exists(work_dir_root)):
michael@0:             shutil.rmtree(work_dir_root)
michael@0: 
michael@0: def main(argv):
michael@0:     patchlist_file = None
michael@0:     try:
michael@0:          opts, args = getopt.getopt(argv, "hf:", ["help", "patchlist_file="])
michael@0:          for opt, arg in opts:
michael@0:             if opt in ("-h", "--help"):
michael@0:                 usage()
michael@0:                 sys.exit()
michael@0:             elif opt in ("-f", "--patchlist_file"):
michael@0:                 patchlist_file = arg
michael@0:     except getopt.GetoptError:
michael@0:           usage()
michael@0:           sys.exit(2)
michael@0: 
michael@0:     if not patchlist_file:
michael@0:         usage()
michael@0:         sys.exit(2)
michael@0: 
michael@0:     patches = []
michael@0:     f = open(patchlist_file, 'r')
michael@0:     for line in f.readlines():
michael@0:         patches.append(line)
michael@0:     f.close()
michael@0:     create_partial_patches(patches)
michael@0: 
michael@0: if __name__ == "__main__":
michael@0:     main(sys.argv[1:])
michael@0: