1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/python/mozbuild/mozpack/test/test_copier.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,464 @@
1.4 +# This Source Code Form is subject to the terms of the Mozilla Public
1.5 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.6 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.7 +
1.8 +from mozpack.copier import (
1.9 + FileCopier,
1.10 + FilePurger,
1.11 + FileRegistry,
1.12 + Jarrer,
1.13 +)
1.14 +from mozpack.files import (
1.15 + GeneratedFile,
1.16 + ExistingFile,
1.17 +)
1.18 +from mozpack.mozjar import JarReader
1.19 +import mozpack.path
1.20 +import unittest
1.21 +import mozunit
1.22 +import os
1.23 +import stat
1.24 +from mozpack.errors import ErrorMessage
1.25 +from mozpack.test.test_files import (
1.26 + MockDest,
1.27 + MatchTestTemplate,
1.28 + TestWithTmpDir,
1.29 +)
1.30 +
1.31 +
1.32 +class TestFileRegistry(MatchTestTemplate, unittest.TestCase):
1.33 + def add(self, path):
1.34 + self.registry.add(path, GeneratedFile(path))
1.35 +
1.36 + def do_check(self, pattern, result):
1.37 + self.checked = True
1.38 + if result:
1.39 + self.assertTrue(self.registry.contains(pattern))
1.40 + else:
1.41 + self.assertFalse(self.registry.contains(pattern))
1.42 + self.assertEqual(self.registry.match(pattern), result)
1.43 +
1.44 + def test_file_registry(self):
1.45 + self.registry = FileRegistry()
1.46 + self.registry.add('foo', GeneratedFile('foo'))
1.47 + bar = GeneratedFile('bar')
1.48 + self.registry.add('bar', bar)
1.49 + self.assertEqual(self.registry.paths(), ['foo', 'bar'])
1.50 + self.assertEqual(self.registry['bar'], bar)
1.51 +
1.52 + self.assertRaises(ErrorMessage, self.registry.add, 'foo',
1.53 + GeneratedFile('foo2'))
1.54 +
1.55 + self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
1.56 +
1.57 + self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
1.58 + GeneratedFile('foobar'))
1.59 + self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
1.60 + GeneratedFile('foobar'))
1.61 +
1.62 + self.assertEqual(self.registry.paths(), ['foo', 'bar'])
1.63 +
1.64 + self.registry.remove('foo')
1.65 + self.assertEqual(self.registry.paths(), ['bar'])
1.66 + self.registry.remove('bar')
1.67 + self.assertEqual(self.registry.paths(), [])
1.68 +
1.69 + self.prepare_match_test()
1.70 + self.do_match_test()
1.71 + self.assertTrue(self.checked)
1.72 + self.assertEqual(self.registry.paths(), [
1.73 + 'bar',
1.74 + 'foo/bar',
1.75 + 'foo/baz',
1.76 + 'foo/qux/1',
1.77 + 'foo/qux/bar',
1.78 + 'foo/qux/2/test',
1.79 + 'foo/qux/2/test2',
1.80 + ])
1.81 +
1.82 + self.registry.remove('foo/qux')
1.83 + self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
1.84 +
1.85 + self.registry.add('foo/qux', GeneratedFile('fooqux'))
1.86 + self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz',
1.87 + 'foo/qux'])
1.88 + self.registry.remove('foo/b*')
1.89 + self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
1.90 +
1.91 + self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
1.92 + self.assertEqual(len(self.registry), 2)
1.93 +
1.94 + self.add('foo/.foo')
1.95 + self.assertTrue(self.registry.contains('foo/.foo'))
1.96 +
1.97 + def test_registry_paths(self):
1.98 + self.registry = FileRegistry()
1.99 +
1.100 + # Can't add a file if it requires a directory in place of a
1.101 + # file we also require.
1.102 + self.registry.add('foo', GeneratedFile('foo'))
1.103 + self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
1.104 + GeneratedFile('foobar'))
1.105 +
1.106 + # Can't add a file if we already have a directory there.
1.107 + self.registry.add('bar/baz', GeneratedFile('barbaz'))
1.108 + self.assertRaises(ErrorMessage, self.registry.add, 'bar',
1.109 + GeneratedFile('bar'))
1.110 +
1.111 + # Bump the count of things that require bar/ to 2.
1.112 + self.registry.add('bar/zot', GeneratedFile('barzot'))
1.113 + self.assertRaises(ErrorMessage, self.registry.add, 'bar',
1.114 + GeneratedFile('bar'))
1.115 +
1.116 + # Drop the count of things that require bar/ to 1.
1.117 + self.registry.remove('bar/baz')
1.118 + self.assertRaises(ErrorMessage, self.registry.add, 'bar',
1.119 + GeneratedFile('bar'))
1.120 +
1.121 + # Drop the count of things that require bar/ to 0.
1.122 + self.registry.remove('bar/zot')
1.123 + self.registry.add('bar/zot', GeneratedFile('barzot'))
1.124 +
1.125 + def test_required_directories(self):
1.126 + self.registry = FileRegistry()
1.127 +
1.128 + self.registry.add('foo', GeneratedFile('foo'))
1.129 + self.assertEqual(self.registry.required_directories(), set())
1.130 +
1.131 + self.registry.add('bar/baz', GeneratedFile('barbaz'))
1.132 + self.assertEqual(self.registry.required_directories(), {'bar'})
1.133 +
1.134 + self.registry.add('bar/zot', GeneratedFile('barzot'))
1.135 + self.assertEqual(self.registry.required_directories(), {'bar'})
1.136 +
1.137 + self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
1.138 + self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'})
1.139 +
1.140 + self.registry.remove('bar/zap/zot')
1.141 + self.assertEqual(self.registry.required_directories(), {'bar'})
1.142 +
1.143 + self.registry.remove('bar/baz')
1.144 + self.assertEqual(self.registry.required_directories(), {'bar'})
1.145 +
1.146 + self.registry.remove('bar/zot')
1.147 + self.assertEqual(self.registry.required_directories(), set())
1.148 +
1.149 + self.registry.add('x/y/z', GeneratedFile('xyz'))
1.150 + self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
1.151 +
1.152 +
1.153 +class TestFileCopier(TestWithTmpDir):
1.154 + def all_dirs(self, base):
1.155 + all_dirs = set()
1.156 + for root, dirs, files in os.walk(base):
1.157 + if not dirs:
1.158 + all_dirs.add(mozpack.path.relpath(root, base))
1.159 + return all_dirs
1.160 +
1.161 + def all_files(self, base):
1.162 + all_files = set()
1.163 + for root, dirs, files in os.walk(base):
1.164 + for f in files:
1.165 + all_files.add(
1.166 + mozpack.path.join(mozpack.path.relpath(root, base), f))
1.167 + return all_files
1.168 +
1.169 + def test_file_copier(self):
1.170 + copier = FileCopier()
1.171 + copier.add('foo/bar', GeneratedFile('foobar'))
1.172 + copier.add('foo/qux', GeneratedFile('fooqux'))
1.173 + copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
1.174 + copier.add('bar', GeneratedFile('bar'))
1.175 + copier.add('qux/foo', GeneratedFile('quxfoo'))
1.176 + copier.add('qux/bar', GeneratedFile(''))
1.177 +
1.178 + result = copier.copy(self.tmpdir)
1.179 + self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
1.180 + self.assertEqual(self.all_dirs(self.tmpdir),
1.181 + set(['foo/deep/nested/directory', 'qux']))
1.182 +
1.183 + self.assertEqual(result.updated_files, set(self.tmppath(p) for p in
1.184 + self.all_files(self.tmpdir)))
1.185 + self.assertEqual(result.existing_files, set())
1.186 + self.assertEqual(result.removed_files, set())
1.187 + self.assertEqual(result.removed_directories, set())
1.188 +
1.189 + copier.remove('foo')
1.190 + copier.add('test', GeneratedFile('test'))
1.191 + result = copier.copy(self.tmpdir)
1.192 + self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
1.193 + self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
1.194 + self.assertEqual(result.removed_files, set(self.tmppath(p) for p in
1.195 + ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))
1.196 +
1.197 + def test_symlink_directory_replaced(self):
1.198 + """Directory symlinks in destination are replaced if they need to be
1.199 + real directories."""
1.200 + if not self.symlink_supported:
1.201 + return
1.202 +
1.203 + dest = self.tmppath('dest')
1.204 +
1.205 + copier = FileCopier()
1.206 + copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))
1.207 +
1.208 + os.makedirs(self.tmppath('dest/foo'))
1.209 + dummy = self.tmppath('dummy')
1.210 + os.mkdir(dummy)
1.211 + link = self.tmppath('dest/foo/bar')
1.212 + os.symlink(dummy, link)
1.213 +
1.214 + result = copier.copy(dest)
1.215 +
1.216 + st = os.lstat(link)
1.217 + self.assertFalse(stat.S_ISLNK(st.st_mode))
1.218 + self.assertTrue(stat.S_ISDIR(st.st_mode))
1.219 +
1.220 + self.assertEqual(self.all_files(dest), set(copier.paths()))
1.221 +
1.222 + self.assertEqual(result.removed_directories, set())
1.223 + self.assertEqual(len(result.updated_files), 1)
1.224 +
1.225 + def test_remove_unaccounted_directory_symlinks(self):
1.226 + """Directory symlinks in destination that are not in the way are
1.227 + deleted according to remove_unaccounted and
1.228 + remove_all_directory_symlinks.
1.229 + """
1.230 + if not self.symlink_supported:
1.231 + return
1.232 +
1.233 + dest = self.tmppath('dest')
1.234 +
1.235 + copier = FileCopier()
1.236 + copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))
1.237 +
1.238 + os.makedirs(self.tmppath('dest/foo'))
1.239 + dummy = self.tmppath('dummy')
1.240 + os.mkdir(dummy)
1.241 +
1.242 + os.mkdir(self.tmppath('dest/zot'))
1.243 + link = self.tmppath('dest/zot/zap')
1.244 + os.symlink(dummy, link)
1.245 +
1.246 + # If not remove_unaccounted but remove_empty_directories, then
1.247 + # the symlinked directory remains (as does its containing
1.248 + # directory).
1.249 + result = copier.copy(dest, remove_unaccounted=False,
1.250 + remove_empty_directories=True,
1.251 + remove_all_directory_symlinks=False)
1.252 +
1.253 + st = os.lstat(link)
1.254 + self.assertTrue(stat.S_ISLNK(st.st_mode))
1.255 + self.assertFalse(stat.S_ISDIR(st.st_mode))
1.256 +
1.257 + self.assertEqual(self.all_files(dest), set(copier.paths()))
1.258 + self.assertEqual(self.all_dirs(dest), set(['foo/bar']))
1.259 +
1.260 + self.assertEqual(result.removed_directories, set())
1.261 + self.assertEqual(len(result.updated_files), 1)
1.262 +
1.263 + # If remove_unaccounted but not remove_empty_directories, then
1.264 + # only the symlinked directory is removed.
1.265 + result = copier.copy(dest, remove_unaccounted=True,
1.266 + remove_empty_directories=False,
1.267 + remove_all_directory_symlinks=False)
1.268 +
1.269 + st = os.lstat(self.tmppath('dest/zot'))
1.270 + self.assertFalse(stat.S_ISLNK(st.st_mode))
1.271 + self.assertTrue(stat.S_ISDIR(st.st_mode))
1.272 +
1.273 + self.assertEqual(result.removed_files, set([link]))
1.274 + self.assertEqual(result.removed_directories, set())
1.275 +
1.276 + self.assertEqual(self.all_files(dest), set(copier.paths()))
1.277 + self.assertEqual(self.all_dirs(dest), set(['foo/bar', 'zot']))
1.278 +
1.279 + # If remove_unaccounted and remove_empty_directories, then
1.280 + # both the symlink and its containing directory are removed.
1.281 + link = self.tmppath('dest/zot/zap')
1.282 + os.symlink(dummy, link)
1.283 +
1.284 + result = copier.copy(dest, remove_unaccounted=True,
1.285 + remove_empty_directories=True,
1.286 + remove_all_directory_symlinks=False)
1.287 +
1.288 + self.assertEqual(result.removed_files, set([link]))
1.289 + self.assertEqual(result.removed_directories, set([self.tmppath('dest/zot')]))
1.290 +
1.291 + self.assertEqual(self.all_files(dest), set(copier.paths()))
1.292 + self.assertEqual(self.all_dirs(dest), set(['foo/bar']))
1.293 +
1.294 + def test_permissions(self):
1.295 + """Ensure files without write permission can be deleted."""
1.296 + with open(self.tmppath('dummy'), 'a'):
1.297 + pass
1.298 +
1.299 + p = self.tmppath('no_perms')
1.300 + with open(p, 'a'):
1.301 + pass
1.302 +
1.303 + # Make file and directory unwritable. Reminder: making a directory
1.304 + # unwritable prevents modifications (including deletes) from the list
1.305 + # of files in that directory.
1.306 + os.chmod(p, 0400)
1.307 + os.chmod(self.tmpdir, 0400)
1.308 +
1.309 + copier = FileCopier()
1.310 + copier.add('dummy', GeneratedFile('content'))
1.311 + result = copier.copy(self.tmpdir)
1.312 + self.assertEqual(result.removed_files_count, 1)
1.313 + self.assertFalse(os.path.exists(p))
1.314 +
1.315 + def test_no_remove(self):
1.316 + copier = FileCopier()
1.317 + copier.add('foo', GeneratedFile('foo'))
1.318 +
1.319 + with open(self.tmppath('bar'), 'a'):
1.320 + pass
1.321 +
1.322 + os.mkdir(self.tmppath('emptydir'))
1.323 + d = self.tmppath('populateddir')
1.324 + os.mkdir(d)
1.325 +
1.326 + with open(self.tmppath('populateddir/foo'), 'a'):
1.327 + pass
1.328 +
1.329 + result = copier.copy(self.tmpdir, remove_unaccounted=False)
1.330 +
1.331 + self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar',
1.332 + 'populateddir/foo']))
1.333 + self.assertEqual(self.all_dirs(self.tmpdir), set(['populateddir']))
1.334 + self.assertEqual(result.removed_files, set())
1.335 + self.assertEqual(result.removed_directories,
1.336 + set([self.tmppath('emptydir')]))
1.337 +
1.338 + def test_no_remove_empty_directories(self):
1.339 + copier = FileCopier()
1.340 + copier.add('foo', GeneratedFile('foo'))
1.341 +
1.342 + with open(self.tmppath('bar'), 'a'):
1.343 + pass
1.344 +
1.345 + os.mkdir(self.tmppath('emptydir'))
1.346 + d = self.tmppath('populateddir')
1.347 + os.mkdir(d)
1.348 +
1.349 + with open(self.tmppath('populateddir/foo'), 'a'):
1.350 + pass
1.351 +
1.352 + result = copier.copy(self.tmpdir, remove_unaccounted=False,
1.353 + remove_empty_directories=False)
1.354 +
1.355 + self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar',
1.356 + 'populateddir/foo']))
1.357 + self.assertEqual(self.all_dirs(self.tmpdir), set(['emptydir',
1.358 + 'populateddir']))
1.359 + self.assertEqual(result.removed_files, set())
1.360 + self.assertEqual(result.removed_directories, set())
1.361 +
1.362 + def test_optional_exists_creates_unneeded_directory(self):
1.363 + """Demonstrate that a directory not strictly required, but specified
1.364 + as the path to an optional file, will be unnecessarily created.
1.365 +
1.366 + This behaviour is wrong; fixing it is tracked by Bug 972432;
1.367 + and this test exists to guard against unexpected changes in
1.368 + behaviour.
1.369 + """
1.370 +
1.371 + dest = self.tmppath('dest')
1.372 +
1.373 + copier = FileCopier()
1.374 + copier.add('foo/bar', ExistingFile(required=False))
1.375 +
1.376 + result = copier.copy(dest)
1.377 +
1.378 + st = os.lstat(self.tmppath('dest/foo'))
1.379 + self.assertFalse(stat.S_ISLNK(st.st_mode))
1.380 + self.assertTrue(stat.S_ISDIR(st.st_mode))
1.381 +
1.382 + # What's worse, we have no record that dest was created.
1.383 + self.assertEquals(len(result.updated_files), 0)
1.384 +
1.385 + # But we do have an erroneous record of an optional file
1.386 + # existing when it does not.
1.387 + self.assertIn(self.tmppath('dest/foo/bar'), result.existing_files)
1.388 +
1.389 +
1.390 +class TestFilePurger(TestWithTmpDir):
1.391 + def test_file_purger(self):
1.392 + existing = os.path.join(self.tmpdir, 'existing')
1.393 + extra = os.path.join(self.tmpdir, 'extra')
1.394 + empty_dir = os.path.join(self.tmpdir, 'dir')
1.395 +
1.396 + with open(existing, 'a'):
1.397 + pass
1.398 +
1.399 + with open(extra, 'a'):
1.400 + pass
1.401 +
1.402 + os.mkdir(empty_dir)
1.403 + with open(os.path.join(empty_dir, 'foo'), 'a'):
1.404 + pass
1.405 +
1.406 + self.assertTrue(os.path.exists(existing))
1.407 + self.assertTrue(os.path.exists(extra))
1.408 +
1.409 + purger = FilePurger()
1.410 + purger.add('existing')
1.411 + result = purger.purge(self.tmpdir)
1.412 + self.assertEqual(result.removed_files, set(self.tmppath(p) for p in
1.413 + ('extra', 'dir/foo')))
1.414 + self.assertEqual(result.removed_files_count, 2)
1.415 + self.assertEqual(result.removed_directories_count, 1)
1.416 +
1.417 + self.assertTrue(os.path.exists(existing))
1.418 + self.assertFalse(os.path.exists(extra))
1.419 + self.assertFalse(os.path.exists(empty_dir))
1.420 +
1.421 +
1.422 +class TestJarrer(unittest.TestCase):
1.423 + def check_jar(self, dest, copier):
1.424 + jar = JarReader(fileobj=dest)
1.425 + self.assertEqual([f.filename for f in jar], copier.paths())
1.426 + for f in jar:
1.427 + self.assertEqual(f.uncompressed_data.read(),
1.428 + copier[f.filename].content)
1.429 +
1.430 + def test_jarrer(self):
1.431 + copier = Jarrer()
1.432 + copier.add('foo/bar', GeneratedFile('foobar'))
1.433 + copier.add('foo/qux', GeneratedFile('fooqux'))
1.434 + copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
1.435 + copier.add('bar', GeneratedFile('bar'))
1.436 + copier.add('qux/foo', GeneratedFile('quxfoo'))
1.437 + copier.add('qux/bar', GeneratedFile(''))
1.438 +
1.439 + dest = MockDest()
1.440 + copier.copy(dest)
1.441 + self.check_jar(dest, copier)
1.442 +
1.443 + copier.remove('foo')
1.444 + copier.add('test', GeneratedFile('test'))
1.445 + copier.copy(dest)
1.446 + self.check_jar(dest, copier)
1.447 +
1.448 + copier.remove('test')
1.449 + copier.add('test', GeneratedFile('replaced-content'))
1.450 + copier.copy(dest)
1.451 + self.check_jar(dest, copier)
1.452 +
1.453 + copier.copy(dest)
1.454 + self.check_jar(dest, copier)
1.455 +
1.456 + preloaded = ['qux/bar', 'bar']
1.457 + copier.preload(preloaded)
1.458 + copier.copy(dest)
1.459 +
1.460 + dest.seek(0)
1.461 + jar = JarReader(fileobj=dest)
1.462 + self.assertEqual([f.filename for f in jar], preloaded +
1.463 + [p for p in copier.paths() if not p in preloaded])
1.464 + self.assertEqual(jar.last_preloaded, preloaded[-1])
1.465 +
1.466 +if __name__ == '__main__':
1.467 + mozunit.main()
