config/MozZipFile.py@97036ab72558 (annotated)
config/MozZipFile.py
Tue, 06 Jan 2015 21:39:09 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Tue, 06 Jan 2015 21:39:09 +0100
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
- permissions
- -rw-r--r--
Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.
| michael@0 | 1 | # This Source Code Form is subject to the terms of the Mozilla Public |
| michael@0 | 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this |
| michael@0 | 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| michael@0 | 4 | |
| michael@0 | 5 | import os |
| michael@0 | 6 | import time |
| michael@0 | 7 | import zipfile |
| michael@0 | 8 | |
| michael@0 | 9 | from mozbuild.util import lock_file |
| michael@0 | 10 | |
| michael@0 | 11 | |
| michael@0 | 12 | class ZipFile(zipfile.ZipFile): |
| michael@0 | 13 | """ Class with methods to open, read, write, close, list zip files. |
| michael@0 | 14 | |
| michael@0 | 15 | Subclassing zipfile.ZipFile to allow for overwriting of existing |
| michael@0 | 16 | entries, though only for writestr, not for write. |
| michael@0 | 17 | """ |
| michael@0 | 18 | def __init__(self, file, mode="r", compression=zipfile.ZIP_STORED, |
| michael@0 | 19 | lock = False): |
| michael@0 | 20 | if lock: |
| michael@0 | 21 | assert isinstance(file, basestring) |
| michael@0 | 22 | self.lockfile = lock_file(file + '.lck') |
| michael@0 | 23 | else: |
| michael@0 | 24 | self.lockfile = None |
| michael@0 | 25 | |
| michael@0 | 26 | if mode == 'a' and lock: |
| michael@0 | 27 | # appending to a file which doesn't exist fails, but we can't check |
| michael@0 | 28 | # existence util we hold the lock |
| michael@0 | 29 | if (not os.path.isfile(file)) or os.path.getsize(file) == 0: |
| michael@0 | 30 | mode = 'w' |
| michael@0 | 31 | |
| michael@0 | 32 | zipfile.ZipFile.__init__(self, file, mode, compression) |
| michael@0 | 33 | self._remove = [] |
| michael@0 | 34 | self.end = self.fp.tell() |
| michael@0 | 35 | self.debug = 0 |
| michael@0 | 36 | |
| michael@0 | 37 | def writestr(self, zinfo_or_arcname, bytes): |
| michael@0 | 38 | """Write contents into the archive. |
| michael@0 | 39 | |
| michael@0 | 40 | The contents is the argument 'bytes', 'zinfo_or_arcname' is either |
| michael@0 | 41 | a ZipInfo instance or the name of the file in the archive. |
| michael@0 | 42 | This method is overloaded to allow overwriting existing entries. |
| michael@0 | 43 | """ |
| michael@0 | 44 | if not isinstance(zinfo_or_arcname, zipfile.ZipInfo): |
| michael@0 | 45 | zinfo = zipfile.ZipInfo(filename=zinfo_or_arcname, |
| michael@0 | 46 | date_time=time.localtime(time.time())) |
| michael@0 | 47 | zinfo.compress_type = self.compression |
| michael@0 | 48 | # Add some standard UNIX file access permissions (-rw-r--r--). |
| michael@0 | 49 | zinfo.external_attr = (0x81a4 & 0xFFFF) << 16L |
| michael@0 | 50 | else: |
| michael@0 | 51 | zinfo = zinfo_or_arcname |
| michael@0 | 52 | |
| michael@0 | 53 | # Now to the point why we overwrote this in the first place, |
| michael@0 | 54 | # remember the entry numbers if we already had this entry. |
| michael@0 | 55 | # Optimizations: |
| michael@0 | 56 | # If the entry to overwrite is the last one, just reuse that. |
| michael@0 | 57 | # If we store uncompressed and the new content has the same size |
| michael@0 | 58 | # as the old, reuse the existing entry. |
| michael@0 | 59 | |
| michael@0 | 60 | doSeek = False # store if we need to seek to the eof after overwriting |
| michael@0 | 61 | if self.NameToInfo.has_key(zinfo.filename): |
| michael@0 | 62 | # Find the last ZipInfo with our name. |
| michael@0 | 63 | # Last, because that's catching multiple overwrites |
| michael@0 | 64 | i = len(self.filelist) |
| michael@0 | 65 | while i > 0: |
| michael@0 | 66 | i -= 1 |
| michael@0 | 67 | if self.filelist[i].filename == zinfo.filename: |
| michael@0 | 68 | break |
| michael@0 | 69 | zi = self.filelist[i] |
| michael@0 | 70 | if ((zinfo.compress_type == zipfile.ZIP_STORED |
| michael@0 | 71 | and zi.compress_size == len(bytes)) |
| michael@0 | 72 | or (i + 1) == len(self.filelist)): |
| michael@0 | 73 | # make sure we're allowed to write, otherwise done by writestr below |
| michael@0 | 74 | self._writecheck(zi) |
| michael@0 | 75 | # overwrite existing entry |
| michael@0 | 76 | self.fp.seek(zi.header_offset) |
| michael@0 | 77 | if (i + 1) == len(self.filelist): |
| michael@0 | 78 | # this is the last item in the file, just truncate |
| michael@0 | 79 | self.fp.truncate() |
| michael@0 | 80 | else: |
| michael@0 | 81 | # we need to move to the end of the file afterwards again |
| michael@0 | 82 | doSeek = True |
| michael@0 | 83 | # unhook the current zipinfo, the writestr of our superclass |
| michael@0 | 84 | # will add a new one |
| michael@0 | 85 | self.filelist.pop(i) |
| michael@0 | 86 | self.NameToInfo.pop(zinfo.filename) |
| michael@0 | 87 | else: |
| michael@0 | 88 | # Couldn't optimize, sadly, just remember the old entry for removal |
| michael@0 | 89 | self._remove.append(self.filelist.pop(i)) |
| michael@0 | 90 | zipfile.ZipFile.writestr(self, zinfo, bytes) |
| michael@0 | 91 | self.filelist.sort(lambda l, r: cmp(l.header_offset, r.header_offset)) |
| michael@0 | 92 | if doSeek: |
| michael@0 | 93 | self.fp.seek(self.end) |
| michael@0 | 94 | self.end = self.fp.tell() |
| michael@0 | 95 | |
| michael@0 | 96 | def close(self): |
| michael@0 | 97 | """Close the file, and for mode "w" and "a" write the ending |
| michael@0 | 98 | records. |
| michael@0 | 99 | |
| michael@0 | 100 | Overwritten to compact overwritten entries. |
| michael@0 | 101 | """ |
| michael@0 | 102 | if not self._remove: |
| michael@0 | 103 | # we don't have anything special to do, let's just call base |
| michael@0 | 104 | r = zipfile.ZipFile.close(self) |
| michael@0 | 105 | self.lockfile = None |
| michael@0 | 106 | return r |
| michael@0 | 107 | |
| michael@0 | 108 | if self.fp.mode != 'r+b': |
| michael@0 | 109 | # adjust file mode if we originally just wrote, now we rewrite |
| michael@0 | 110 | self.fp.close() |
| michael@0 | 111 | self.fp = open(self.filename, 'r+b') |
| michael@0 | 112 | all = map(lambda zi: (zi, True), self.filelist) + \ |
| michael@0 | 113 | map(lambda zi: (zi, False), self._remove) |
| michael@0 | 114 | all.sort(lambda l, r: cmp(l[0].header_offset, r[0].header_offset)) |
| michael@0 | 115 | # empty _remove for multiple closes |
| michael@0 | 116 | self._remove = [] |
| michael@0 | 117 | |
| michael@0 | 118 | lengths = [all[i+1][0].header_offset - all[i][0].header_offset |
| michael@0 | 119 | for i in xrange(len(all)-1)] |
| michael@0 | 120 | lengths.append(self.end - all[-1][0].header_offset) |
| michael@0 | 121 | to_pos = 0 |
| michael@0 | 122 | for (zi, keep), length in zip(all, lengths): |
| michael@0 | 123 | if not keep: |
| michael@0 | 124 | continue |
| michael@0 | 125 | oldoff = zi.header_offset |
| michael@0 | 126 | # python <= 2.4 has file_offset |
| michael@0 | 127 | if hasattr(zi, 'file_offset'): |
| michael@0 | 128 | zi.file_offset = zi.file_offset + to_pos - oldoff |
| michael@0 | 129 | zi.header_offset = to_pos |
| michael@0 | 130 | self.fp.seek(oldoff) |
| michael@0 | 131 | content = self.fp.read(length) |
| michael@0 | 132 | self.fp.seek(to_pos) |
| michael@0 | 133 | self.fp.write(content) |
| michael@0 | 134 | to_pos += length |
| michael@0 | 135 | self.fp.truncate() |
| michael@0 | 136 | zipfile.ZipFile.close(self) |
| michael@0 | 137 | self.lockfile = None |
