1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/config/tests/unitMozZipFile.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,201 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +import unittest
1.9 +
1.10 +import shutil
1.11 +import os
1.12 +import re
1.13 +import sys
1.14 +import random
1.15 +import copy
1.16 +from string import letters
1.17 +
1.18 +'''
1.19 +Test case infrastructure for MozZipFile.
1.20 +
1.21 +This isn't really a unit test, but a test case generator and runner.
1.22 +For a given set of files, lengths, and number of writes, we create
1.23 +a testcase for every combination of the three. There are some
1.24 +symmetries used to reduce the number of test cases, the first file
1.25 +written is always the first file, the second is either the first or
1.26 +the second, the third is one of the first three. That is, if we
1.27 +had 4 files, but only three writes, the fourth file would never even
1.28 +get tried.
1.29 +
1.30 +The content written to the jars is pseudorandom with a fixed seed.
1.31 +'''
1.32 +
1.33 +if not __file__:
1.34 + __file__ = sys.argv[0]
1.35 +sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
1.36 +
1.37 +from MozZipFile import ZipFile
1.38 +import zipfile
1.39 +
1.40 +leafs = (
1.41 + 'firstdir/oneleaf',
1.42 + 'seconddir/twoleaf',
1.43 + 'thirddir/with/sub/threeleaf')
1.44 +_lengths = map(lambda n: n * 64, [16, 64, 80])
1.45 +lengths = 3
1.46 +writes = 5
1.47 +
1.48 +def givenlength(i):
1.49 + '''Return a length given in the _lengths array to allow manual
1.50 + tuning of which lengths of zip entries to use.
1.51 + '''
1.52 + return _lengths[i]
1.53 +
1.54 +
1.55 +def prod(*iterables):
1.56 + ''''Tensor product of a list of iterables.
1.57 +
1.58 + This generator returns lists of items, one of each given
1.59 + iterable. It iterates over all possible combinations.
1.60 + '''
1.61 + for item in iterables[0]:
1.62 + if len(iterables) == 1:
1.63 + yield [item]
1.64 + else:
1.65 + for others in prod(*iterables[1:]):
1.66 + yield [item] + others
1.67 +
1.68 +
1.69 +def getid(descs):
1.70 + 'Convert a list of ints to a string.'
1.71 + return reduce(lambda x,y: x+'{0}{1}'.format(*tuple(y)), descs,'')
1.72 +
1.73 +
1.74 +def getContent(length):
1.75 + 'Get pseudo random content of given length.'
1.76 + rv = [None] * length
1.77 + for i in xrange(length):
1.78 + rv[i] = random.choice(letters)
1.79 + return ''.join(rv)
1.80 +
1.81 +
1.82 +def createWriter(sizer, *items):
1.83 + 'Helper method to fill in tests, one set of writes, one for each item'
1.84 + locitems = copy.deepcopy(items)
1.85 + for item in locitems:
1.86 + item['length'] = sizer(item.pop('length', 0))
1.87 + def helper(self):
1.88 + mode = 'w'
1.89 + if os.path.isfile(self.f):
1.90 + mode = 'a'
1.91 + zf = ZipFile(self.f, mode, self.compression)
1.92 + for item in locitems:
1.93 + self._write(zf, **item)
1.94 + zf = None
1.95 + pass
1.96 + return helper
1.97 +
1.98 +def createTester(name, *writes):
1.99 + '''Helper method to fill in tests, calls into a list of write
1.100 + helper methods.
1.101 + '''
1.102 + _writes = copy.copy(writes)
1.103 + def tester(self):
1.104 + for w in _writes:
1.105 + getattr(self, w)()
1.106 + self._verifyZip()
1.107 + pass
1.108 + # unit tests get confused if the method name isn't test...
1.109 + tester.__name__ = name
1.110 + return tester
1.111 +
1.112 +class TestExtensiveStored(unittest.TestCase):
1.113 + '''Unit tests for MozZipFile
1.114 +
1.115 + The testcase are actually populated by code following the class
1.116 + definition.
1.117 + '''
1.118 +
1.119 + stage = "mozzipfilestage"
1.120 + compression = zipfile.ZIP_STORED
1.121 +
1.122 + def leaf(self, *leafs):
1.123 + return os.path.join(self.stage, *leafs)
1.124 + def setUp(self):
1.125 + if os.path.exists(self.stage):
1.126 + shutil.rmtree(self.stage)
1.127 + os.mkdir(self.stage)
1.128 + self.f = self.leaf('test.jar')
1.129 + self.ref = {}
1.130 + self.seed = 0
1.131 +
1.132 + def tearDown(self):
1.133 + self.f = None
1.134 + self.ref = None
1.135 +
1.136 + def _verifyZip(self):
1.137 + zf = zipfile.ZipFile(self.f)
1.138 + badEntry = zf.testzip()
1.139 + self.failIf(badEntry, badEntry)
1.140 + zlist = zf.namelist()
1.141 + zlist.sort()
1.142 + vlist = self.ref.keys()
1.143 + vlist.sort()
1.144 + self.assertEqual(zlist, vlist)
1.145 + for leaf, content in self.ref.iteritems():
1.146 + zcontent = zf.read(leaf)
1.147 + self.assertEqual(content, zcontent)
1.148 +
1.149 + def _write(self, zf, seed=None, leaf=0, length=0):
1.150 + if seed is None:
1.151 + seed = self.seed
1.152 + self.seed += 1
1.153 + random.seed(seed)
1.154 + leaf = leafs[leaf]
1.155 + content = getContent(length)
1.156 + self.ref[leaf] = content
1.157 + zf.writestr(leaf, content)
1.158 + dir = os.path.dirname(self.leaf('stage', leaf))
1.159 + if not os.path.isdir(dir):
1.160 + os.makedirs(dir)
1.161 + open(self.leaf('stage', leaf), 'w').write(content)
1.162 +
1.163 +# all leafs in all lengths
1.164 +atomics = list(prod(xrange(len(leafs)), xrange(lengths)))
1.165 +
1.166 +# populate TestExtensiveStore with testcases
1.167 +for w in xrange(writes):
1.168 + # Don't iterate over all files for the the first n passes,
1.169 + # those are redundant as long as w < lengths.
1.170 + # There are symmetries in the trailing end, too, but I don't know
1.171 + # how to reduce those out right now.
1.172 + nonatomics = [list(prod(range(min(i,len(leafs))), xrange(lengths)))
1.173 + for i in xrange(1, w+1)] + [atomics]
1.174 + for descs in prod(*nonatomics):
1.175 + suffix = getid(descs)
1.176 + dicts = [dict(leaf=leaf, length=length) for leaf, length in descs]
1.177 + setattr(TestExtensiveStored, '_write' + suffix,
1.178 + createWriter(givenlength, *dicts))
1.179 + setattr(TestExtensiveStored, 'test' + suffix,
1.180 + createTester('test' + suffix, '_write' + suffix))
1.181 +
1.182 +# now create another round of tests, with two writing passes
1.183 +# first, write all file combinations into the jar, close it,
1.184 +# and then write all atomics again.
1.185 +# This should catch more or less all artifacts generated
1.186 +# by the final ordering step when closing the jar.
1.187 +files = [list(prod([i], xrange(lengths))) for i in xrange(len(leafs))]
1.188 +allfiles = reduce(lambda l,r:l+r,
1.189 + [list(prod(*files[:(i+1)])) for i in xrange(len(leafs))])
1.190 +
1.191 +for first in allfiles:
1.192 + testbasename = 'test{0}_'.format(getid(first))
1.193 + test = [None, '_write' + getid(first), None]
1.194 + for second in atomics:
1.195 + test[0] = testbasename + getid([second])
1.196 + test[2] = '_write' + getid([second])
1.197 + setattr(TestExtensiveStored, test[0], createTester(*test))
1.198 +
1.199 +class TestExtensiveDeflated(TestExtensiveStored):
1.200 + 'Test all that has been tested with ZIP_STORED with DEFLATED, too.'
1.201 + compression = zipfile.ZIP_DEFLATED
1.202 +
1.203 +if __name__ == '__main__':
1.204 + unittest.main()
