1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/tools/update-packaging/make_incremental_update.sh Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,344 @@
1.4 +#!/bin/bash
1.5 +# This Source Code Form is subject to the terms of the Mozilla Public
1.6 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.8 +
1.9 +#
1.10 +# This tool generates incremental update packages for the update system.
1.11 +# Author: Darin Fisher
1.12 +#
1.13 +
1.14 +. $(dirname "$0")/common.sh
1.15 +
1.16 +# -----------------------------------------------------------------------------
1.17 +
1.18 +print_usage() {
1.19 + notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR"
1.20 + notice ""
1.21 + notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE."
1.22 + notice ""
1.23 + notice "Options:"
1.24 + notice " -h show this help text"
1.25 + notice " -f clobber this file in the installation"
1.26 + notice " Must be a path to a file to clobber in the partial update."
1.27 + notice " -q be less verbose"
1.28 + notice ""
1.29 +}
1.30 +
1.31 +check_for_forced_update() {
1.32 + force_list="$1"
1.33 + forced_file_chk="$2"
1.34 +
1.35 + local f
1.36 +
1.37 + if [ "$forced_file_chk" = "precomplete" ]; then
1.38 + ## "true" *giggle*
1.39 + return 0;
1.40 + fi
1.41 +
1.42 + if [ "${forced_file_chk##*.}" = "chk" ]
1.43 + then
1.44 + ## "true" *giggle*
1.45 + return 0;
1.46 + fi
1.47 +
1.48 + for f in $force_list; do
1.49 + #echo comparing $forced_file_chk to $f
1.50 + if [ "$forced_file_chk" = "$f" ]; then
1.51 + ## "true" *giggle*
1.52 + return 0;
1.53 + fi
1.54 +
1.55 + # If the file in the skip list ends with /*, do a prefix match.
1.56 + # This allows TorBrowser/Data/Browser/profile.default/extensions/https-everywhere@eff.org/* to be used to force all HTTPS Everywhere files to be updated.
1.57 + f_suffix=${f##*/}
1.58 + if [[ $f_suffix = "*" ]]; then
1.59 + f_prefix="${f%\/\*}";
1.60 + if [[ $forced_file_chk == $f_prefix* ]]; then
1.61 + ## 0 means "true"
1.62 + return 0;
1.63 + fi
1.64 + fi
1.65 + done
1.66 + ## 'false'... because this is bash. Oh yay!
1.67 + return 1;
1.68 +}
1.69 +
1.70 +if [ $# = 0 ]; then
1.71 + print_usage
1.72 + exit 1
1.73 +fi
1.74 +
1.75 +# The last .xpi is NoScript.
1.76 +ext_path='TorBrowser/Data/Browser/profile.default/extensions';
1.77 +# TODO: it would be better to pass this as a command line option.
1.78 +exts='https-everywhere@eff.org/* tor-launcher@torproject.org.xpi torbutton@torproject.org.xpi uriloader@pdf.js.xpi {73a6fe31-595d-460b-a920-fcc0f8843232}.xpi'
1.79 +requested_forced_updates='Contents/MacOS/TorBrowser.app/Contents/MacOS/firefox'
1.80 +for ext in $exts; do
1.81 + requested_forced_updates="$requested_forced_updates $ext_path/$ext"
1.82 +done
1.83 +
1.84 +
1.85 +# TODO: it would be better to pass this as a command line option.
1.86 +directories_to_remove='TorBrowser/Data/Browser/profile.default/extensions/https-everywhere@eff.org'
1.87 +
1.88 +while getopts "hqf:" flag
1.89 +do
1.90 + case "$flag" in
1.91 + h) print_usage; exit 0
1.92 + ;;
1.93 + q) QUIET=1
1.94 + ;;
1.95 + f) requested_forced_updates="$requested_forced_updates $OPTARG"
1.96 + ;;
1.97 + ?) print_usage; exit 1
1.98 + ;;
1.99 + esac
1.100 +done
1.101 +
1.102 +# -----------------------------------------------------------------------------
1.103 +
1.104 +let arg_start=$OPTIND-1
1.105 +shift $arg_start
1.106 +
1.107 +archive="$1"
1.108 +olddir="$2"
1.109 +newdir="$3"
1.110 +# Prevent the workdir from being inside the targetdir so it isn't included in
1.111 +# the update mar.
1.112 +if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then
1.113 + # Remove the /
1.114 + newdir=$(echo "$newdir" | sed -e 's:\/$::')
1.115 +fi
1.116 +workdir="$newdir.work"
1.117 +updatemanifestv2="$workdir/updatev2.manifest"
1.118 +updatemanifestv3="$workdir/updatev3.manifest"
1.119 +archivefiles="updatev2.manifest updatev3.manifest"
1.120 +
1.121 +mkdir -p "$workdir"
1.122 +
1.123 +# On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the
1.124 +# Info.plist so it launches the right architecture, bug 600098
1.125 +
1.126 +# Generate a list of all files in the target directory.
1.127 +pushd "$olddir"
1.128 +if test $? -ne 0 ; then
1.129 + exit 1
1.130 +fi
1.131 +
1.132 +list_files oldfiles
1.133 +list_symlinks oldsymlinks oldsymlink_targets
1.134 +list_dirs olddirs
1.135 +
1.136 +popd
1.137 +
1.138 +pushd "$newdir"
1.139 +if test $? -ne 0 ; then
1.140 + exit 1
1.141 +fi
1.142 +
1.143 +if [ ! -f "precomplete" ]; then
1.144 + notice "precomplete file is missing!"
1.145 + exit 1
1.146 +fi
1.147 +
1.148 +list_dirs newdirs
1.149 +list_files newfiles
1.150 +list_symlinks newsymlinks newsymlink_targets
1.151 +
1.152 +popd
1.153 +
1.154 +# Add the type of update to the beginning of the update manifests.
1.155 +notice ""
1.156 +notice "Adding type instruction to update manifests"
1.157 +> "$updatemanifestv2"
1.158 +> "$updatemanifestv3"
1.159 +notice " type partial"
1.160 +echo "type \"partial\"" >> "$updatemanifestv2"
1.161 +echo "type \"partial\"" >> "$updatemanifestv3"
1.162 +
1.163 +# If removal of any old, existing directories is desired, emit the appropriate
1.164 +# rmrfdir commands.
1.165 +notice ""
1.166 +notice "Adding directory removal instructions to update manifests"
1.167 +for dir_to_remove in $directories_to_remove; do
1.168 + # rmrfdir requires a trailing slash, so add one if missing.
1.169 + if ! [[ "$dir_to_remove" =~ /$ ]]; then
1.170 + dir_to_remove="${dir_to_remove}/"
1.171 + fi
1.172 + echo "rmrfdir \"$dir_to_remove\"" >> "$updatemanifestv2"
1.173 + echo "rmrfdir \"$dir_to_remove\"" >> "$updatemanifestv3"
1.174 +done
1.175 +
1.176 +notice ""
1.177 +notice "Adding file patch and add instructions to update manifests"
1.178 +
1.179 +num_oldfiles=${#oldfiles[*]}
1.180 +remove_array=
1.181 +num_removes=0
1.182 +
1.183 +for ((i=0; $i<$num_oldfiles; i=$i+1)); do
1.184 + f="${oldfiles[$i]}"
1.185 +
1.186 + # This file is created by Talkback, so we can ignore it
1.187 + if [ "$f" = "readme.txt" ]; then
1.188 + continue 1
1.189 + fi
1.190 +
1.191 + # removed-files is excluded by make_incremental_updates.py so it is excluded
1.192 + # here for consistency.
1.193 + if [ "`basename "$f"`" = "removed-files" ]; then
1.194 + continue 1
1.195 + fi
1.196 +
1.197 + # If this file exists in the new directory as well, then check if it differs.
1.198 + if [ -f "$newdir/$f" ]; then
1.199 +
1.200 + if check_for_add_if_not_update "$f"; then
1.201 + # The full workdir may not exist yet, so create it if necessary.
1.202 + mkdir -p `dirname "$workdir/$f"`
1.203 + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
1.204 + copy_perm "$newdir/$f" "$workdir/$f"
1.205 + make_add_if_not_instruction "$f" "$updatemanifestv3"
1.206 + archivefiles="$archivefiles \"$f\""
1.207 + continue 1
1.208 + fi
1.209 +
1.210 + if check_for_forced_update "$requested_forced_updates" "$f"; then
1.211 + # The full workdir may not exist yet, so create it if necessary.
1.212 + mkdir -p `dirname "$workdir/$f"`
1.213 + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
1.214 + copy_perm "$newdir/$f" "$workdir/$f"
1.215 + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
1.216 + archivefiles="$archivefiles \"$f\""
1.217 + continue 1
1.218 + fi
1.219 +
1.220 + if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then
1.221 + # Compute both the compressed binary diff and the compressed file, and
1.222 + # compare the sizes. Then choose the smaller of the two to package.
1.223 + dir=$(dirname "$workdir/$f")
1.224 + mkdir -p "$dir"
1.225 + notice "diffing \"$f\""
1.226 + $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
1.227 + $BZIP2 -z9 "$workdir/$f.patch"
1.228 + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
1.229 + copy_perm "$newdir/$f" "$workdir/$f"
1.230 + patchfile="$workdir/$f.patch.bz2"
1.231 + patchsize=$(get_file_size "$patchfile")
1.232 + fullsize=$(get_file_size "$workdir/$f")
1.233 +
1.234 + if [ $patchsize -lt $fullsize ]; then
1.235 + make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
1.236 + mv -f "$patchfile" "$workdir/$f.patch"
1.237 + rm -f "$workdir/$f"
1.238 + archivefiles="$archivefiles \"$f.patch\""
1.239 + else
1.240 + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
1.241 + rm -f "$patchfile"
1.242 + archivefiles="$archivefiles \"$f\""
1.243 + fi
1.244 + fi
1.245 + else
1.246 + # remove instructions are added after add / patch instructions for
1.247 + # consistency with make_incremental_updates.py
1.248 + remove_array[$num_removes]=$f
1.249 + (( num_removes++ ))
1.250 + fi
1.251 +done
1.252 +
1.253 +# Remove and re-add symlinks
1.254 +notice ""
1.255 +notice "Adding symlink remove/add instructions to update manifests"
1.256 +num_oldsymlinks=${#oldsymlinks[*]}
1.257 +for ((i=0; $i<$num_oldsymlinks; i=$i+1)); do
1.258 + link="${oldsymlinks[$i]}"
1.259 + verbose_notice " remove: $link"
1.260 + echo "remove \"$link\"" >> "$updatemanifestv2"
1.261 + echo "remove \"$link\"" >> "$updatemanifestv3"
1.262 +done
1.263 +
1.264 +num_newsymlinks=${#newsymlinks[*]}
1.265 +for ((i=0; $i<$num_newsymlinks; i=$i+1)); do
1.266 + link="${newsymlinks[$i]}"
1.267 + target="${newsymlink_targets[$i]}"
1.268 + make_addsymlink_instruction "$link" "$target" "$updatemanifestv2" "$updatemanifestv3"
1.269 +done
1.270 +
1.271 +# Newly added files
1.272 +notice ""
1.273 +notice "Adding file add instructions to update manifests"
1.274 +num_newfiles=${#newfiles[*]}
1.275 +
1.276 +for ((i=0; $i<$num_newfiles; i=$i+1)); do
1.277 + f="${newfiles[$i]}"
1.278 +
1.279 + # removed-files is excluded by make_incremental_updates.py so it is excluded
1.280 + # here for consistency.
1.281 + if [ "`basename "$f"`" = "removed-files" ]; then
1.282 + continue 1
1.283 + fi
1.284 +
1.285 + # If we've already tested this file, then skip it
1.286 + for ((j=0; $j<$num_oldfiles; j=$j+1)); do
1.287 + if [ "$f" = "${oldfiles[j]}" ]; then
1.288 + continue 2
1.289 + fi
1.290 + done
1.291 +
1.292 + dir=$(dirname "$workdir/$f")
1.293 + mkdir -p "$dir"
1.294 +
1.295 + $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
1.296 + copy_perm "$newdir/$f" "$workdir/$f"
1.297 +
1.298 + if check_for_add_if_not_update "$f"; then
1.299 + make_add_if_not_instruction "$f" "$updatemanifestv3"
1.300 + else
1.301 + make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3"
1.302 + fi
1.303 +
1.304 +
1.305 + archivefiles="$archivefiles \"$f\""
1.306 +done
1.307 +
1.308 +notice ""
1.309 +notice "Adding file remove instructions to update manifests"
1.310 +for ((i=0; $i<$num_removes; i=$i+1)); do
1.311 + f="${remove_array[$i]}"
1.312 + notice " remove \"$f\""
1.313 + echo "remove \"$f\"" >> "$updatemanifestv2"
1.314 + echo "remove \"$f\"" >> "$updatemanifestv3"
1.315 +done
1.316 +
1.317 +# Add remove instructions for any dead files.
1.318 +notice ""
1.319 +notice "Adding file and directory remove instructions from file 'removed-files'"
1.320 +append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3"
1.321 +
1.322 +notice ""
1.323 +notice "Adding directory remove instructions for directories that no longer exist"
1.324 +num_olddirs=${#olddirs[*]}
1.325 +
1.326 +for ((i=0; $i<$num_olddirs; i=$i+1)); do
1.327 + f="${olddirs[$i]}"
1.328 + # If this dir doesn't exist in the new directory remove it.
1.329 + if [ ! -d "$newdir/$f" ]; then
1.330 + notice " rmdir $f/"
1.331 + echo "rmdir \"$f/\"" >> "$updatemanifestv2"
1.332 + echo "rmdir \"$f/\"" >> "$updatemanifestv3"
1.333 + fi
1.334 +done
1.335 +
1.336 +$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
1.337 +$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
1.338 +
1.339 +eval "$MAR -C \"$workdir\" -c output.mar $archivefiles"
1.340 +mv -f "$workdir/output.mar" "$archive"
1.341 +
1.342 +# cleanup
1.343 +rm -fr "$workdir"
1.344 +
1.345 +notice ""
1.346 +notice "Finished"
1.347 +notice ""
