toolkit/crashreporter/tools/unit-symbolstore.py@6474c204b198
toolkit/crashreporter/tools/unit-symbolstore.py
Wed, 31 Dec 2014 06:09:35 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Wed, 31 Dec 2014 06:09:35 +0100
- changeset 0
- 6474c204b198
- permissions
- -rw-r--r--
Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.
1 #!/usr/bin/env python
2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this
4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
6 import os, tempfile, unittest, shutil, struct, platform, subprocess, multiprocessing.dummy
7 import mock
8 from mock import patch
9 import symbolstore
11 # Some simple functions to mock out files that the platform-specific dumpers will accept.
12 # dump_syms itself will not be run (we mock that call out), but we can't override
13 # the ShouldProcessFile method since we actually want to test that.
14 def write_elf(filename):
15 open(filename, "wb").write(struct.pack("<7B45x", 0x7f, ord("E"), ord("L"), ord("F"), 1, 1, 1))
17 def write_macho(filename):
18 open(filename, "wb").write(struct.pack("<I28x", 0xfeedface))
20 def write_pdb(filename):
21 open(filename, "w").write("aaa")
22 # write out a fake DLL too
23 open(os.path.splitext(filename)[0] + ".dll", "w").write("aaa")
25 writer = {'Windows': write_pdb,
26 'Microsoft': write_pdb,
27 'Linux': write_elf,
28 'Sunos5': write_elf,
29 'Darwin': write_macho}[platform.system()]
30 extension = {'Windows': ".pdb",
31 'Microsoft': ".pdb",
32 'Linux': ".so",
33 'Sunos5': ".so",
34 'Darwin': ".dylib"}[platform.system()]
36 def add_extension(files):
37 return [f + extension for f in files]
39 class HelperMixin(object):
40 """
41 Test that passing filenames to exclude from processing works.
42 """
43 def setUp(self):
44 self.test_dir = tempfile.mkdtemp()
45 if not self.test_dir.endswith("/"):
46 self.test_dir += "/"
47 symbolstore.srcdirRepoInfo = {}
48 symbolstore.vcsFileInfoCache = {}
50 def tearDown(self):
51 shutil.rmtree(self.test_dir)
52 symbolstore.srcdirRepoInfo = {}
53 symbolstore.vcsFileInfoCache = {}
55 def add_test_files(self, files):
56 for f in files:
57 f = os.path.join(self.test_dir, f)
58 d = os.path.dirname(f)
59 if d and not os.path.exists(d):
60 os.makedirs(d)
61 writer(f)
63 class TestExclude(HelperMixin, unittest.TestCase):
64 def test_exclude_wildcard(self):
65 """
66 Test that using an exclude list with a wildcard pattern works.
67 """
68 processed = []
69 def mock_process_file(filenames):
70 for filename in filenames:
71 processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
72 return True
73 self.add_test_files(add_extension(["foo", "bar", "abc/xyz", "abc/fooxyz", "def/asdf", "def/xyzfoo"]))
74 d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
75 symbol_path="symbol_path",
76 exclude=["*foo*"])
77 d.ProcessFiles = mock_process_file
78 d.Process(self.test_dir)
79 d.Finish(stop_pool=False)
80 processed.sort()
81 expected = add_extension(["bar", "abc/xyz", "def/asdf"])
82 expected.sort()
83 self.assertEqual(processed, expected)
85 def test_exclude_filenames(self):
86 """
87 Test that excluding a filename without a wildcard works.
88 """
89 processed = []
90 def mock_process_file(filenames):
91 for filename in filenames:
92 processed.append((filename[len(self.test_dir):] if filename.startswith(self.test_dir) else filename).replace('\\', '/'))
93 return True
94 self.add_test_files(add_extension(["foo", "bar", "abc/foo", "abc/bar", "def/foo", "def/bar"]))
95 d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
96 symbol_path="symbol_path",
97 exclude=add_extension(["foo"]))
98 d.ProcessFiles = mock_process_file
99 d.Process(self.test_dir)
100 d.Finish(stop_pool=False)
101 processed.sort()
102 expected = add_extension(["bar", "abc/bar", "def/bar"])
103 expected.sort()
104 self.assertEqual(processed, expected)
106 def popen_factory(stdouts):
107 """
108 Generate a class that can mock subprocess.Popen. |stdouts| is an iterable that
109 should return an iterable for the stdout of each process in turn.
110 """
111 class mock_popen(object):
112 def __init__(self, args, *args_rest, **kwargs):
113 self.stdout = stdouts.next()
115 def wait(self):
116 return 0
117 return mock_popen
119 def mock_dump_syms(module_id, filename):
120 return ["MODULE os x86 %s %s" % (module_id, filename),
121 "FILE 0 foo.c",
122 "PUBLIC xyz 123"]
124 class TestCopyDebugUniversal(HelperMixin, unittest.TestCase):
125 """
126 Test that CopyDebug does the right thing when dumping multiple architectures.
127 """
128 def setUp(self):
129 HelperMixin.setUp(self)
130 self.symbol_dir = tempfile.mkdtemp()
131 self._subprocess_call = subprocess.call
132 subprocess.call = self.mock_call
133 self._subprocess_popen = subprocess.Popen
134 subprocess.Popen = popen_factory(self.next_mock_stdout())
135 self.stdouts = []
136 self._shutil_rmtree = shutil.rmtree
137 shutil.rmtree = self.mock_rmtree
139 def tearDown(self):
140 HelperMixin.tearDown(self)
141 shutil.rmtree = self._shutil_rmtree
142 shutil.rmtree(self.symbol_dir)
143 subprocess.call = self._subprocess_call
144 subprocess.Popen = self._subprocess_popen
146 def mock_rmtree(self, path):
147 pass
149 def mock_call(self, args, **kwargs):
150 if args[0].endswith("dsymutil"):
151 filename = args[-1]
152 os.makedirs(filename + ".dSYM")
153 return 0
155 def next_mock_stdout(self):
156 if not self.stdouts:
157 yield iter([])
158 for s in self.stdouts:
159 yield iter(s)
161 def test_copy_debug_universal(self):
162 """
163 Test that dumping symbols for multiple architectures only copies debug symbols once
164 per file.
165 """
166 copied = []
167 def mock_copy_debug(filename, debug_file, guid):
168 copied.append(filename[len(self.symbol_dir):] if filename.startswith(self.symbol_dir) else filename)
169 self.add_test_files(add_extension(["foo"]))
170 self.stdouts.append(mock_dump_syms("X" * 33, add_extension(["foo"])[0]))
171 self.stdouts.append(mock_dump_syms("Y" * 33, add_extension(["foo"])[0]))
172 d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
173 symbol_path=self.symbol_dir,
174 copy_debug=True,
175 archs="abc xyz")
176 d.CopyDebug = mock_copy_debug
177 d.Process(self.test_dir)
178 d.Finish(stop_pool=False)
179 self.assertEqual(1, len(copied))
181 class TestGetVCSFilename(HelperMixin, unittest.TestCase):
182 def setUp(self):
183 HelperMixin.setUp(self)
185 def tearDown(self):
186 HelperMixin.tearDown(self)
188 @patch("subprocess.Popen")
189 def testVCSFilenameHg(self, mock_Popen):
190 # mock calls to `hg parent` and `hg showconfig paths.default`
191 mock_communicate = mock_Popen.return_value.communicate
192 mock_communicate.side_effect = [("abcd1234", ""),
193 ("http://example.com/repo", "")]
194 os.mkdir(os.path.join(self.test_dir, ".hg"))
195 filename = os.path.join(self.test_dir, "foo.c")
196 self.assertEqual("hg:example.com/repo:foo.c:abcd1234",
197 symbolstore.GetVCSFilename(filename, [self.test_dir])[0])
199 @patch("subprocess.Popen")
200 def testVCSFilenameHgMultiple(self, mock_Popen):
201 # mock calls to `hg parent` and `hg showconfig paths.default`
202 mock_communicate = mock_Popen.return_value.communicate
203 mock_communicate.side_effect = [("abcd1234", ""),
204 ("http://example.com/repo", ""),
205 ("0987ffff", ""),
206 ("http://example.com/other", "")]
207 srcdir1 = os.path.join(self.test_dir, "one")
208 srcdir2 = os.path.join(self.test_dir, "two")
209 os.makedirs(os.path.join(srcdir1, ".hg"))
210 os.makedirs(os.path.join(srcdir2, ".hg"))
211 filename1 = os.path.join(srcdir1, "foo.c")
212 filename2 = os.path.join(srcdir2, "bar.c")
213 self.assertEqual("hg:example.com/repo:foo.c:abcd1234",
214 symbolstore.GetVCSFilename(filename1, [srcdir1, srcdir2])[0])
215 self.assertEqual("hg:example.com/other:bar.c:0987ffff",
216 symbolstore.GetVCSFilename(filename2, [srcdir1, srcdir2])[0])
218 class TestRepoManifest(HelperMixin, unittest.TestCase):
219 def testRepoManifest(self):
220 manifest = os.path.join(self.test_dir, "sources.xml")
221 open(manifest, "w").write("""<?xml version="1.0" encoding="UTF-8"?>
222 <manifest>
223 <remote fetch="http://example.com/foo/" name="foo"/>
224 <remote fetch="git://example.com/bar/" name="bar"/>
225 <default remote="bar"/>
226 <project name="projects/one" revision="abcd1234"/>
227 <project name="projects/two" path="projects/another" revision="ffffffff" remote="foo"/>
228 <project name="something_else" revision="00000000" remote="bar"/>
229 </manifest>
230 """)
231 # Use a source file from each of the three projects
232 file1 = os.path.join(self.test_dir, "projects", "one", "src1.c")
233 file2 = os.path.join(self.test_dir, "projects", "another", "src2.c")
234 file3 = os.path.join(self.test_dir, "something_else", "src3.c")
235 d = symbolstore.Dumper("dump_syms", "symbol_path",
236 repo_manifest=manifest)
237 self.assertEqual("git:example.com/bar/projects/one:src1.c:abcd1234",
238 symbolstore.GetVCSFilename(file1, d.srcdirs)[0])
239 self.assertEqual("git:example.com/foo/projects/two:src2.c:ffffffff",
240 symbolstore.GetVCSFilename(file2, d.srcdirs)[0])
241 self.assertEqual("git:example.com/bar/something_else:src3.c:00000000",
242 symbolstore.GetVCSFilename(file3, d.srcdirs)[0])
244 if platform.system() in ("Windows", "Microsoft"):
245 class TestSourceServer(HelperMixin, unittest.TestCase):
246 @patch("subprocess.call")
247 @patch("subprocess.Popen")
248 def test_HGSERVER(self, mock_Popen, mock_call):
249 """
250 Test that HGSERVER gets set correctly in the source server index.
251 """
252 symbolpath = os.path.join(self.test_dir, "symbols")
253 os.makedirs(symbolpath)
254 srcdir = os.path.join(self.test_dir, "srcdir")
255 os.makedirs(os.path.join(srcdir, ".hg"))
256 sourcefile = os.path.join(srcdir, "foo.c")
257 test_files = add_extension(["foo"])
258 self.add_test_files(test_files)
259 # srcsrv needs PDBSTR_PATH set
260 os.environ["PDBSTR_PATH"] = "pdbstr"
261 # mock calls to `dump_syms`, `hg parent` and
262 # `hg showconfig paths.default`
263 mock_Popen.return_value.stdout = iter([
264 "MODULE os x86 %s %s" % ("X" * 33, test_files[0]),
265 "FILE 0 %s" % sourcefile,
266 "PUBLIC xyz 123"
267 ])
268 mock_communicate = mock_Popen.return_value.communicate
269 mock_communicate.side_effect = [("abcd1234", ""),
270 ("http://example.com/repo", ""),
271 ]
272 # And mock the call to pdbstr to capture the srcsrv stream data.
273 global srcsrv_stream
274 srcsrv_stream = None
275 def mock_pdbstr(args, cwd="", **kwargs):
276 for arg in args:
277 if arg.startswith("-i:"):
278 global srcsrv_stream
279 srcsrv_stream = open(os.path.join(cwd, arg[3:]), 'r').read()
280 return 0
281 mock_call.side_effect = mock_pdbstr
282 d = symbolstore.GetPlatformSpecificDumper(dump_syms="dump_syms",
283 symbol_path=symbolpath,
284 srcdirs=[srcdir],
285 vcsinfo=True,
286 srcsrv=True,
287 copy_debug=True)
288 # stub out CopyDebug
289 d.CopyDebug = lambda *args: True
290 d.Process(self.test_dir)
291 d.Finish(stop_pool=False)
292 self.assertNotEqual(srcsrv_stream, None)
293 hgserver = [x.rstrip() for x in srcsrv_stream.splitlines() if x.startswith("HGSERVER=")]
294 self.assertEqual(len(hgserver), 1)
295 self.assertEqual(hgserver[0].split("=")[1], "http://example.com/repo")
297 if __name__ == '__main__':
298 # use the multiprocessing.dummy module to use threading wrappers so
299 # that our mocking/module-patching works
300 symbolstore.Dumper.GlobalInit(module=multiprocessing.dummy)
302 unittest.main()
304 symbolstore.Dumper.pool.close()
305 symbolstore.Dumper.pool.join()
