1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/toolkit/crashreporter/tools/unit-symbolstore.py Wed Dec 31 06:09:35 2014 +0100
1.3 @@ -0,0 +1,305 @@
1.4 +#!/usr/bin/env python
1.5 +# This Source Code Form is subject to the terms of the Mozilla Public
1.6 +# License, v. 2.0. If a copy of the MPL was not distributed with this
1.7 +# file, You can obtain one at http://mozilla.org/MPL/2.0/.
1.8 +
1.9 +import os, tempfile, unittest, shutil, struct, platform, subprocess, multiprocessing.dummy
1.10 +import mock
1.11 +from mock import patch
1.12 +import symbolstore
1.13 +
1.14 +# Some simple functions to mock out files that the platform-specific dumpers will accept.
1.15 +# dump_syms itself will not be run (we mock that call out), but we can't override
1.16 +# the ShouldProcessFile method since we actually want to test that.
1.17 +def write_elf(filename):
1.18 + open(filename, "wb").write(struct.pack("<7B45x", 0x7f, ord("E"), ord("L"), ord("F"), 1, 1, 1))
1.19 +
1.20 +def write_macho(filename):
1.21 + open(filename, "wb").write(struct.pack("<I28x", 0xfeedface))
1.22 +
1.23 +def write_pdb(filename):
1.24 + open(filename, "w").write("aaa")
1.25 + # write out a fake DLL too
1.26 + open(os.path.splitext(filename)[0] + ".dll", "w").write("aaa")
1.27 +
1.28 +writer = {'Windows': write_pdb,
1.29 + 'Microsoft': write_pdb,
1.30 + 'Linux': write_elf,
1.31 + 'Sunos5': write_elf,
1.32 + 'Darwin': write_macho}[platform.system()]
1.33 +extension = {'Windows': ".pdb",
1.34 + 'Microsoft': ".pdb",
1.35 + 'Linux': ".so",
1.36 + 'Sunos5': ".so",
1.37 + 'Darwin': ".dylib"}[platform.system()]
1.38 +
1.39 +def add_extension(files):
1.40 + return [f + extension for f in files]
1.41 +
1.42 +class HelperMixin(object):
1.43 + """
1.44 + Test that passing filenames to exclude from processing works.
1.45 + """
1.46 + def setUp(self):
1.47 + self.test_dir = tempfile.mkdtemp()
1.48 + if not self.test_dir.endswith("/"):
1.49 + self.test_dir += "/"
1.50 + symbolstore.srcdirRepoInfo = {}
1.51 + symbolstore.vcsFileInfoCache = {}
1.52 +
1.53 + def tearDown(self):
1.54 + shutil.rmtree(self.test_dir)
1.55 + symbolstore.srcdirRepoInfo = {}
1.56 + symbolstore.vcsFileInfoCache = {}
1.57 +
1.58 + def add_test_files(self, files):
1.59 + for f in files:
1.60 + f = os.path.join(self.test_dir, f)
1.61 + d = os.path.dirname(f)
1.62 + if d and not os.path.exists(d):
1.63 + os.makedirs(d)
1.64 + writer(f)
1.65 +
1.66 +class TestExclude(HelperMixin, unittest.TestCase):
1.67 + def test_exclude_wildcard(self):
1.68 + """
1.69 + Test that using an exclude list with a wildcard pattern works.
1.70 + """
1.71 + processed = []
1.72 + def mock_process_file(filenames):
1.73 + for filename in filenames:
1.74 + processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
1.75 + return True
1.76 + self.add_test_files(add_extension(["foo", "bar", "abc/xyz", "abc/fooxyz", "def/asdf", "def/xyzfoo"]))
1.77 + d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
1.78 + symbol_path="symbol_path",
1.79 + exclude=["*foo*"])
1.80 + d.ProcessFiles = mock_process_file
1.81 + d.Process(self.test_dir)
1.82 + d.Finish(stop_pool=False)
1.83 + processed.sort()
1.84 + expected = add_extension(["bar", "abc/xyz", "def/asdf"])
1.85 + expected.sort()
1.86 + self.assertEqual(processed, expected)
1.87 +
1.88 + def test_exclude_filenames(self):
1.89 + """
1.90 + Test that excluding a filename without a wildcard works.
1.91 + """
1.92 + processed = []
1.93 + def mock_process_file(filenames):
1.94 + for filename in filenames:
1.95 + processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
1.96 + return True
1.97 + self.add_test_files(add_extension(["foo", "bar", "abc/foo", "abc/bar", "def/foo", "def/bar"]))
1.98 + d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
1.99 + symbol_path="symbol_path",
1.100 + exclude=add_extension(["foo"]))
1.101 + d.ProcessFiles = mock_process_file
1.102 + d.Process(self.test_dir)
1.103 + d.Finish(stop_pool=False)
1.104 + processed.sort()
1.105 + expected = add_extension(["bar", "abc/bar", "def/bar"])
1.106 + expected.sort()
1.107 + self.assertEqual(processed, expected)
1.108 +
1.109 +def popen_factory(stdouts):
1.110 + """
1.111 + Generate a class that can mock subprocess.Popen. |stdouts| is an iterable that
1.112 + should return an iterable for the stdout of each process in turn.
1.113 + """
1.114 + class mock_popen(object):
1.115 + def __init__(self, args, *args_rest, **kwargs):
1.116 + self.stdout = stdouts.next()
1.117 +
1.118 + def wait(self):
1.119 + return 0
1.120 + return mock_popen
1.121 +
1.122 +def mock_dump_syms(module_id, filename):
1.123 + return ["MODULE os x86 %s %s" % (module_id, filename),
1.124 + "FILE 0 foo.c",
1.125 + "PUBLIC xyz 123"]
1.126 +
1.127 +class TestCopyDebugUniversal(HelperMixin, unittest.TestCase):
1.128 + """
1.129 + Test that CopyDebug does the right thing when dumping multiple architectures.
1.130 + """
1.131 + def setUp(self):
1.132 + HelperMixin.setUp(self)
1.133 + self.symbol_dir = tempfile.mkdtemp()
1.134 + self._subprocess_call = subprocess.call
1.135 + subprocess.call = self.mock_call
1.136 + self._subprocess_popen = subprocess.Popen
1.137 + subprocess.Popen = popen_factory(self.next_mock_stdout())
1.138 + self.stdouts = []
1.139 + self._shutil_rmtree = shutil.rmtree
1.140 + shutil.rmtree = self.mock_rmtree
1.141 +
1.142 + def tearDown(self):
1.143 + HelperMixin.tearDown(self)
1.144 + shutil.rmtree = self._shutil_rmtree
1.145 + shutil.rmtree(self.symbol_dir)
1.146 + subprocess.call = self._subprocess_call
1.147 + subprocess.Popen = self._subprocess_popen
1.148 +
1.149 + def mock_rmtree(self, path):
1.150 + pass
1.151 +
1.152 + def mock_call(self, args, **kwargs):
1.153 + if args[0].endswith("dsymutil"):
1.154 + filename = args[-1]
1.155 + os.makedirs(filename + ".dSYM")
1.156 + return 0
1.157 +
1.158 + def next_mock_stdout(self):
1.159 + if not self.stdouts:
1.160 + yield iter([])
1.161 + for s in self.stdouts:
1.162 + yield iter(s)
1.163 +
1.164 + def test_copy_debug_universal(self):
1.165 + """
1.166 + Test that dumping symbols for multiple architectures only copies debug symbols once
1.167 + per file.
1.168 + """
1.169 + copied = []
1.170 + def mock_copy_debug(filename, debug_file, guid):
1.171 + copied.append(filename[len(self.symbol_dir):] if filename.startswith(self.symbol_dir) else filename)
1.172 + self.add_test_files(add_extension(["foo"]))
1.173 + self.stdouts.append(mock_dump_syms("X" * 33, add_extension(["foo"])[0]))
1.174 + self.stdouts.append(mock_dump_syms("Y" * 33, add_extension(["foo"])[0]))
1.175 + d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
1.176 + symbol_path=self.symbol_dir,
1.177 + copy_debug=True,
1.178 + archs="abc xyz")
1.179 + d.CopyDebug = mock_copy_debug
1.180 + d.Process(self.test_dir)
1.181 + d.Finish(stop_pool=False)
1.182 + self.assertEqual(1, len(copied))
1.183 +
1.184 +class TestGetVCSFilename(HelperMixin, unittest.TestCase):
1.185 + def setUp(self):
1.186 + HelperMixin.setUp(self)
1.187 +
1.188 + def tearDown(self):
1.189 + HelperMixin.tearDown(self)
1.190 +
1.191 + @patch("subprocess.Popen")
1.192 + def testVCSFilenameHg(self, mock_Popen):
1.193 + # mock calls to `hg parent` and `hg showconfig paths.default`
1.194 + mock_communicate = mock_Popen.return_value.communicate
1.195 + mock_communicate.side_effect = [("abcd1234", ""),
1.196 + ("http://example.com/repo", "")]
1.197 + os.mkdir(os.path.join(self.test_dir, ".hg"))
1.198 + filename = os.path.join(self.test_dir, "foo.c")
1.199 + self.assertEqual("hg:example.com/repo:foo.c:abcd1234",
1.200 + symbolstore.GetVCSFilename(filename, [self.test_dir])[0])
1.201 +
1.202 + @patch("subprocess.Popen")
1.203 + def testVCSFilenameHgMultiple(self, mock_Popen):
1.204 + # mock calls to `hg parent` and `hg showconfig paths.default`
1.205 + mock_communicate = mock_Popen.return_value.communicate
1.206 + mock_communicate.side_effect = [("abcd1234", ""),
1.207 + ("http://example.com/repo", ""),
1.208 + ("0987ffff", ""),
1.209 + ("http://example.com/other", "")]
1.210 + srcdir1 = os.path.join(self.test_dir, "one")
1.211 + srcdir2 = os.path.join(self.test_dir, "two")
1.212 + os.makedirs(os.path.join(srcdir1, ".hg"))
1.213 + os.makedirs(os.path.join(srcdir2, ".hg"))
1.214 + filename1 = os.path.join(srcdir1, "foo.c")
1.215 + filename2 = os.path.join(srcdir2, "bar.c")
1.216 + self.assertEqual("hg:example.com/repo:foo.c:abcd1234",
1.217 + symbolstore.GetVCSFilename(filename1, [srcdir1, srcdir2])[0])
1.218 + self.assertEqual("hg:example.com/other:bar.c:0987ffff",
1.219 + symbolstore.GetVCSFilename(filename2, [srcdir1, srcdir2])[0])
1.220 +
1.221 +class TestRepoManifest(HelperMixin, unittest.TestCase):
1.222 + def testRepoManifest(self):
1.223 + manifest = os.path.join(self.test_dir, "sources.xml")
1.224 + open(manifest, "w").write("""<?xml version="1.0" encoding="UTF-8"?>
1.225 +<manifest>
1.226 +<remote fetch="http://example.com/foo/" name="foo"/>
1.227 +<remote fetch="git://example.com/bar/" name="bar"/>
1.228 +<default remote="bar"/>
1.229 +<project name="projects/one" revision="abcd1234"/>
1.230 +<project name="projects/two" path="projects/another" revision="ffffffff" remote="foo"/>
1.231 +<project name="something_else" revision="00000000" remote="bar"/>
1.232 +</manifest>
1.233 +""")
1.234 + # Use a source file from each of the three projects
1.235 + file1 = os.path.join(self.test_dir, "projects", "one", "src1.c")
1.236 + file2 = os.path.join(self.test_dir, "projects", "another", "src2.c")
1.237 + file3 = os.path.join(self.test_dir, "something_else", "src3.c")
1.238 + d = symbolstore.Dumper("dump_syms", "symbol_path",
1.239 + repo_manifest=manifest)
1.240 + self.assertEqual("git:example.com/bar/projects/one:src1.c:abcd1234",
1.241 + symbolstore.GetVCSFilename(file1, d.srcdirs)[0])
1.242 + self.assertEqual("git:example.com/foo/projects/two:src2.c:ffffffff",
1.243 + symbolstore.GetVCSFilename(file2, d.srcdirs)[0])
1.244 + self.assertEqual("git:example.com/bar/something_else:src3.c:00000000",
1.245 + symbolstore.GetVCSFilename(file3, d.srcdirs)[0])
1.246 +
1.247 +if platform.system() in ("Windows", "Microsoft"):
1.248 + class TestSourceServer(HelperMixin, unittest.TestCase):
1.249 + @patch("subprocess.call")
1.250 + @patch("subprocess.Popen")
1.251 + def test_HGSERVER(self, mock_Popen, mock_call):
1.252 + """
1.253 + Test that HGSERVER gets set correctly in the source server index.
1.254 + """
1.255 + symbolpath = os.path.join(self.test_dir, "symbols")
1.256 + os.makedirs(symbolpath)
1.257 + srcdir = os.path.join(self.test_dir, "srcdir")
1.258 + os.makedirs(os.path.join(srcdir, ".hg"))
1.259 + sourcefile = os.path.join(srcdir, "foo.c")
1.260 + test_files = add_extension(["foo"])
1.261 + self.add_test_files(test_files)
1.262 + # srcsrv needs PDBSTR_PATH set
1.263 + os.environ["PDBSTR_PATH"] = "pdbstr"
1.264 + # mock calls to `dump_syms`, `hg parent` and
1.265 + # `hg showconfig paths.default`
1.266 + mock_Popen.return_value.stdout = iter([
1.267 + "MODULE os x86 %s %s" % ("X" * 33, test_files[0]),
1.268 + "FILE 0 %s" % sourcefile,
1.269 + "PUBLIC xyz 123"
1.270 + ])
1.271 + mock_communicate = mock_Popen.return_value.communicate
1.272 + mock_communicate.side_effect = [("abcd1234", ""),
1.273 + ("http://example.com/repo", ""),
1.274 + ]
1.275 + # And mock the call to pdbstr to capture the srcsrv stream data.
1.276 + global srcsrv_stream
1.277 + srcsrv_stream = None
1.278 + def mock_pdbstr(args, cwd="", **kwargs):
1.279 + for arg in args:
1.280 + if arg.startswith("-i:"):
1.281 + global srcsrv_stream
1.282 + srcsrv_stream = open(os.path.join(cwd, arg[3:]), 'r').read()
1.283 + return 0
1.284 + mock_call.side_effect = mock_pdbstr
1.285 + d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
1.286 + symbol_path=symbolpath,
1.287 + srcdirs=[srcdir],
1.288 + vcsinfo=True,
1.289 + srcsrv=True,
1.290 + copy_debug=True)
1.291 + # stub out CopyDebug
1.292 + d.CopyDebug = lambda *args: True
1.293 + d.Process(self.test_dir)
1.294 + d.Finish(stop_pool=False)
1.295 + self.assertNotEqual(srcsrv_stream, None)
1.296 + hgserver = [x.rstrip() for x in srcsrv_stream.splitlines() if x.startswith("HGSERVER=")]
1.297 + self.assertEqual(len(hgserver), 1)
1.298 + self.assertEqual(hgserver[0].split("=")[1], "http://example.com/repo")
1.299 +
1.300 +if __name__ == '__main__':
1.301 + # use the multiprocessing.dummy module to use threading wrappers so
1.302 + # that our mocking/module-patching works
1.303 + symbolstore.Dumper.GlobalInit(module=multiprocessing.dummy)
1.304 +
1.305 + unittest.main()
1.306 +
1.307 + symbolstore.Dumper.pool.close()
1.308 + symbolstore.Dumper.pool.join()
