The Tor Browser / file comparison
comparison: tools/update-packaging/make_incremental_update.sh
tools/update-packaging/make_incremental_update.sh
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
equal
deleted
inserted
replaced
| -1:000000000000 | 0:56c0a51a20c8 |
|---|---|
| 1 #!/bin/bash | |
| 2 # This Source Code Form is subject to the terms of the Mozilla Public | |
| 3 # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| 4 # file, You can obtain one at http://mozilla.org/MPL/2.0/. | |
| 5 | |
| 6 # | |
| 7 # This tool generates incremental update packages for the update system. | |
| 8 # Author: Darin Fisher | |
| 9 # | |
| 10 | |
| 11 . $(dirname "$0")/common.sh | |
| 12 | |
| 13 # ----------------------------------------------------------------------------- | |
| 14 | |
| 15 print_usage() { | |
| 16 notice "Usage: $(basename $0) [OPTIONS] ARCHIVE FROMDIR TODIR" | |
| 17 notice "" | |
| 18 notice "The differences between FROMDIR and TODIR will be stored in ARCHIVE." | |
| 19 notice "" | |
| 20 notice "Options:" | |
| 21 notice " -h show this help text" | |
| 22 notice " -f clobber this file in the installation" | |
| 23 notice " Must be a path to a file to clobber in the partial update." | |
| 24 notice " -q be less verbose" | |
| 25 notice "" | |
| 26 } | |
| 27 | |
| 28 check_for_forced_update() { | |
| 29 force_list="$1" | |
| 30 forced_file_chk="$2" | |
| 31 | |
| 32 local f | |
| 33 | |
| 34 if [ "$forced_file_chk" = "precomplete" ]; then | |
| 35 ## "true" *giggle* | |
| 36 return 0; | |
| 37 fi | |
| 38 | |
| 39 if [ "${forced_file_chk##*.}" = "chk" ] | |
| 40 then | |
| 41 ## "true" *giggle* | |
| 42 return 0; | |
| 43 fi | |
| 44 | |
| 45 for f in $force_list; do | |
| 46 #echo comparing $forced_file_chk to $f | |
| 47 if [ "$forced_file_chk" = "$f" ]; then | |
| 48 ## "true" *giggle* | |
| 49 return 0; | |
| 50 fi | |
| 51 | |
| 52 # If the file in the skip list ends with /*, do a prefix match. | |
| 53 # This allows TorBrowser/Data/Browser/profile.default/extensions/https-everywhere@eff.org/* to be used to force all HTTPS Everywhere files to be updated. | |
| 54 f_suffix=${f##*/} | |
| 55 if [[ $f_suffix = "*" ]]; then | |
| 56 f_prefix="${f%\/\*}"; | |
| 57 if [[ $forced_file_chk == $f_prefix* ]]; then | |
| 58 ## 0 means "true" | |
| 59 return 0; | |
| 60 fi | |
| 61 fi | |
| 62 done | |
| 63 ## 'false'... because this is bash. Oh yay! | |
| 64 return 1; | |
| 65 } | |
| 66 | |
| 67 if [ $# = 0 ]; then | |
| 68 print_usage | |
| 69 exit 1 | |
| 70 fi | |
| 71 | |
| 72 # The last .xpi is NoScript. | |
| 73 ext_path='TorBrowser/Data/Browser/profile.default/extensions'; | |
| 74 # TODO: it would be better to pass this as a command line option. | |
| 75 exts='https-everywhere@eff.org/* tor-launcher@torproject.org.xpi torbutton@torproject.org.xpi uriloader@pdf.js.xpi {73a6fe31-595d-460b-a920-fcc0f8843232}.xpi' | |
| 76 requested_forced_updates='Contents/MacOS/TorBrowser.app/Contents/MacOS/firefox' | |
| 77 for ext in $exts; do | |
| 78 requested_forced_updates="$requested_forced_updates $ext_path/$ext" | |
| 79 done | |
| 80 | |
| 81 | |
| 82 # TODO: it would be better to pass this as a command line option. | |
| 83 directories_to_remove='TorBrowser/Data/Browser/profile.default/extensions/https-everywhere@eff.org' | |
| 84 | |
| 85 while getopts "hqf:" flag | |
| 86 do | |
| 87 case "$flag" in | |
| 88 h) print_usage; exit 0 | |
| 89 ;; | |
| 90 q) QUIET=1 | |
| 91 ;; | |
| 92 f) requested_forced_updates="$requested_forced_updates $OPTARG" | |
| 93 ;; | |
| 94 ?) print_usage; exit 1 | |
| 95 ;; | |
| 96 esac | |
| 97 done | |
| 98 | |
| 99 # ----------------------------------------------------------------------------- | |
| 100 | |
| 101 let arg_start=$OPTIND-1 | |
| 102 shift $arg_start | |
| 103 | |
| 104 archive="$1" | |
| 105 olddir="$2" | |
| 106 newdir="$3" | |
| 107 # Prevent the workdir from being inside the targetdir so it isn't included in | |
| 108 # the update mar. | |
| 109 if [ $(echo "$newdir" | grep -c '\/$') = 1 ]; then | |
| 110 # Remove the / | |
| 111 newdir=$(echo "$newdir" | sed -e 's:\/$::') | |
| 112 fi | |
| 113 workdir="$newdir.work" | |
| 114 updatemanifestv2="$workdir/updatev2.manifest" | |
| 115 updatemanifestv3="$workdir/updatev3.manifest" | |
| 116 archivefiles="updatev2.manifest updatev3.manifest" | |
| 117 | |
| 118 mkdir -p "$workdir" | |
| 119 | |
| 120 # On Mac, the precomplete file added by Bug 386760 will cause OS X to reload the | |
| 121 # Info.plist so it launches the right architecture, bug 600098 | |
| 122 | |
| 123 # Generate a list of all files in the target directory. | |
| 124 pushd "$olddir" | |
| 125 if test $? -ne 0 ; then | |
| 126 exit 1 | |
| 127 fi | |
| 128 | |
| 129 list_files oldfiles | |
| 130 list_symlinks oldsymlinks oldsymlink_targets | |
| 131 list_dirs olddirs | |
| 132 | |
| 133 popd | |
| 134 | |
| 135 pushd "$newdir" | |
| 136 if test $? -ne 0 ; then | |
| 137 exit 1 | |
| 138 fi | |
| 139 | |
| 140 if [ ! -f "precomplete" ]; then | |
| 141 notice "precomplete file is missing!" | |
| 142 exit 1 | |
| 143 fi | |
| 144 | |
| 145 list_dirs newdirs | |
| 146 list_files newfiles | |
| 147 list_symlinks newsymlinks newsymlink_targets | |
| 148 | |
| 149 popd | |
| 150 | |
| 151 # Add the type of update to the beginning of the update manifests. | |
| 152 notice "" | |
| 153 notice "Adding type instruction to update manifests" | |
| 154 > "$updatemanifestv2" | |
| 155 > "$updatemanifestv3" | |
| 156 notice " type partial" | |
| 157 echo "type \"partial\"" >> "$updatemanifestv2" | |
| 158 echo "type \"partial\"" >> "$updatemanifestv3" | |
| 159 | |
| 160 # If removal of any old, existing directories is desired, emit the appropriate | |
| 161 # rmrfdir commands. | |
| 162 notice "" | |
| 163 notice "Adding directory removal instructions to update manifests" | |
| 164 for dir_to_remove in $directories_to_remove; do | |
| 165 # rmrfdir requires a trailing slash, so add one if missing. | |
| 166 if ! [[ "$dir_to_remove" =~ /$ ]]; then | |
| 167 dir_to_remove="${dir_to_remove}/" | |
| 168 fi | |
| 169 echo "rmrfdir \"$dir_to_remove\"" >> "$updatemanifestv2" | |
| 170 echo "rmrfdir \"$dir_to_remove\"" >> "$updatemanifestv3" | |
| 171 done | |
| 172 | |
| 173 notice "" | |
| 174 notice "Adding file patch and add instructions to update manifests" | |
| 175 | |
| 176 num_oldfiles=${#oldfiles[*]} | |
| 177 remove_array= | |
| 178 num_removes=0 | |
| 179 | |
| 180 for ((i=0; $i<$num_oldfiles; i=$i+1)); do | |
| 181 f="${oldfiles[$i]}" | |
| 182 | |
| 183 # This file is created by Talkback, so we can ignore it | |
| 184 if [ "$f" = "readme.txt" ]; then | |
| 185 continue 1 | |
| 186 fi | |
| 187 | |
| 188 # removed-files is excluded by make_incremental_updates.py so it is excluded | |
| 189 # here for consistency. | |
| 190 if [ "`basename "$f"`" = "removed-files" ]; then | |
| 191 continue 1 | |
| 192 fi | |
| 193 | |
| 194 # If this file exists in the new directory as well, then check if it differs. | |
| 195 if [ -f "$newdir/$f" ]; then | |
| 196 | |
| 197 if check_for_add_if_not_update "$f"; then | |
| 198 # The full workdir may not exist yet, so create it if necessary. | |
| 199 mkdir -p `dirname "$workdir/$f"` | |
| 200 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" | |
| 201 copy_perm "$newdir/$f" "$workdir/$f" | |
| 202 make_add_if_not_instruction "$f" "$updatemanifestv3" | |
| 203 archivefiles="$archivefiles \"$f\"" | |
| 204 continue 1 | |
| 205 fi | |
| 206 | |
| 207 if check_for_forced_update "$requested_forced_updates" "$f"; then | |
| 208 # The full workdir may not exist yet, so create it if necessary. | |
| 209 mkdir -p `dirname "$workdir/$f"` | |
| 210 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" | |
| 211 copy_perm "$newdir/$f" "$workdir/$f" | |
| 212 make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1 | |
| 213 archivefiles="$archivefiles \"$f\"" | |
| 214 continue 1 | |
| 215 fi | |
| 216 | |
| 217 if ! diff "$olddir/$f" "$newdir/$f" > /dev/null; then | |
| 218 # Compute both the compressed binary diff and the compressed file, and | |
| 219 # compare the sizes. Then choose the smaller of the two to package. | |
| 220 dir=$(dirname "$workdir/$f") | |
| 221 mkdir -p "$dir" | |
| 222 notice "diffing \"$f\"" | |
| 223 $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch" | |
| 224 $BZIP2 -z9 "$workdir/$f.patch" | |
| 225 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" | |
| 226 copy_perm "$newdir/$f" "$workdir/$f" | |
| 227 patchfile="$workdir/$f.patch.bz2" | |
| 228 patchsize=$(get_file_size "$patchfile") | |
| 229 fullsize=$(get_file_size "$workdir/$f") | |
| 230 | |
| 231 if [ $patchsize -lt $fullsize ]; then | |
| 232 make_patch_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" | |
| 233 mv -f "$patchfile" "$workdir/$f.patch" | |
| 234 rm -f "$workdir/$f" | |
| 235 archivefiles="$archivefiles \"$f.patch\"" | |
| 236 else | |
| 237 make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" | |
| 238 rm -f "$patchfile" | |
| 239 archivefiles="$archivefiles \"$f\"" | |
| 240 fi | |
| 241 fi | |
| 242 else | |
| 243 # remove instructions are added after add / patch instructions for | |
| 244 # consistency with make_incremental_updates.py | |
| 245 remove_array[$num_removes]=$f | |
| 246 (( num_removes++ )) | |
| 247 fi | |
| 248 done | |
| 249 | |
| 250 # Remove and re-add symlinks | |
| 251 notice "" | |
| 252 notice "Adding symlink remove/add instructions to update manifests" | |
| 253 num_oldsymlinks=${#oldsymlinks[*]} | |
| 254 for ((i=0; $i<$num_oldsymlinks; i=$i+1)); do | |
| 255 link="${oldsymlinks[$i]}" | |
| 256 verbose_notice " remove: $link" | |
| 257 echo "remove \"$link\"" >> "$updatemanifestv2" | |
| 258 echo "remove \"$link\"" >> "$updatemanifestv3" | |
| 259 done | |
| 260 | |
| 261 num_newsymlinks=${#newsymlinks[*]} | |
| 262 for ((i=0; $i<$num_newsymlinks; i=$i+1)); do | |
| 263 link="${newsymlinks[$i]}" | |
| 264 target="${newsymlink_targets[$i]}" | |
| 265 make_addsymlink_instruction "$link" "$target" "$updatemanifestv2" "$updatemanifestv3" | |
| 266 done | |
| 267 | |
| 268 # Newly added files | |
| 269 notice "" | |
| 270 notice "Adding file add instructions to update manifests" | |
| 271 num_newfiles=${#newfiles[*]} | |
| 272 | |
| 273 for ((i=0; $i<$num_newfiles; i=$i+1)); do | |
| 274 f="${newfiles[$i]}" | |
| 275 | |
| 276 # removed-files is excluded by make_incremental_updates.py so it is excluded | |
| 277 # here for consistency. | |
| 278 if [ "`basename "$f"`" = "removed-files" ]; then | |
| 279 continue 1 | |
| 280 fi | |
| 281 | |
| 282 # If we've already tested this file, then skip it | |
| 283 for ((j=0; $j<$num_oldfiles; j=$j+1)); do | |
| 284 if [ "$f" = "${oldfiles[j]}" ]; then | |
| 285 continue 2 | |
| 286 fi | |
| 287 done | |
| 288 | |
| 289 dir=$(dirname "$workdir/$f") | |
| 290 mkdir -p "$dir" | |
| 291 | |
| 292 $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f" | |
| 293 copy_perm "$newdir/$f" "$workdir/$f" | |
| 294 | |
| 295 if check_for_add_if_not_update "$f"; then | |
| 296 make_add_if_not_instruction "$f" "$updatemanifestv3" | |
| 297 else | |
| 298 make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" | |
| 299 fi | |
| 300 | |
| 301 | |
| 302 archivefiles="$archivefiles \"$f\"" | |
| 303 done | |
| 304 | |
| 305 notice "" | |
| 306 notice "Adding file remove instructions to update manifests" | |
| 307 for ((i=0; $i<$num_removes; i=$i+1)); do | |
| 308 f="${remove_array[$i]}" | |
| 309 notice " remove \"$f\"" | |
| 310 echo "remove \"$f\"" >> "$updatemanifestv2" | |
| 311 echo "remove \"$f\"" >> "$updatemanifestv3" | |
| 312 done | |
| 313 | |
| 314 # Add remove instructions for any dead files. | |
| 315 notice "" | |
| 316 notice "Adding file and directory remove instructions from file 'removed-files'" | |
| 317 append_remove_instructions "$newdir" "$updatemanifestv2" "$updatemanifestv3" | |
| 318 | |
| 319 notice "" | |
| 320 notice "Adding directory remove instructions for directories that no longer exist" | |
| 321 num_olddirs=${#olddirs[*]} | |
| 322 | |
| 323 for ((i=0; $i<$num_olddirs; i=$i+1)); do | |
| 324 f="${olddirs[$i]}" | |
| 325 # If this dir doesn't exist in the new directory remove it. | |
| 326 if [ ! -d "$newdir/$f" ]; then | |
| 327 notice " rmdir $f/" | |
| 328 echo "rmdir \"$f/\"" >> "$updatemanifestv2" | |
| 329 echo "rmdir \"$f/\"" >> "$updatemanifestv3" | |
| 330 fi | |
| 331 done | |
| 332 | |
| 333 $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2" | |
| 334 $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3" | |
| 335 | |
| 336 eval "$MAR -C \"$workdir\" -c output.mar $archivefiles" | |
| 337 mv -f "$workdir/output.mar" "$archive" | |
| 338 | |
| 339 # cleanup | |
| 340 rm -fr "$workdir" | |
| 341 | |
| 342 notice "" | |
| 343 notice "Finished" | |
| 344 notice "" |
