|
1 # This Source Code Form is subject to the terms of the Mozilla Public |
|
2 # License, v. 2.0. If a copy of the MPL was not distributed with this |
|
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
|
4 |
|
5 import errno |
|
6 import os |
|
7 import platform |
|
8 import shutil |
|
9 import stat |
|
10 import subprocess |
|
11 import uuid |
|
12 import mozbuild.makeutil as makeutil |
|
13 from mozbuild.preprocessor import Preprocessor |
|
14 from mozbuild.util import FileAvoidWrite |
|
15 from mozpack.executables import ( |
|
16 is_executable, |
|
17 may_strip, |
|
18 strip, |
|
19 may_elfhack, |
|
20 elfhack, |
|
21 ) |
|
22 from mozpack.chrome.manifest import ManifestEntry |
|
23 from io import BytesIO |
|
24 from mozpack.errors import ( |
|
25 ErrorMessage, |
|
26 errors, |
|
27 ) |
|
28 from mozpack.mozjar import JarReader |
|
29 import mozpack.path |
|
30 from collections import OrderedDict |
|
31 from jsmin import JavascriptMinify |
|
32 from tempfile import ( |
|
33 mkstemp, |
|
34 NamedTemporaryFile, |
|
35 ) |
|
36 |
|
37 |
|
38 class Dest(object): |
|
39 ''' |
|
40 Helper interface for BaseFile.copy. The interface works as follows: |
|
41 - read() and write() can be used to sequentially read/write from the |
|
42 underlying file. |
|
43 - a call to read() after a write() will re-open the underlying file and |
|
44 read from it. |
|
45 - a call to write() after a read() will re-open the underlying file, |
|
46 emptying it, and write to it. |
|
47 ''' |
|
48 def __init__(self, path): |
|
49 self.path = path |
|
50 self.mode = None |
|
51 |
|
52 @property |
|
53 def name(self): |
|
54 return self.path |
|
55 |
|
56 def read(self, length=-1): |
|
57 if self.mode != 'r': |
|
58 self.file = open(self.path, 'rb') |
|
59 self.mode = 'r' |
|
60 return self.file.read(length) |
|
61 |
|
62 def write(self, data): |
|
63 if self.mode != 'w': |
|
64 self.file = open(self.path, 'wb') |
|
65 self.mode = 'w' |
|
66 return self.file.write(data) |
|
67 |
|
68 def exists(self): |
|
69 return os.path.exists(self.path) |
|
70 |
|
71 def close(self): |
|
72 if self.mode: |
|
73 self.mode = None |
|
74 self.file.close() |
|
75 |
|
76 |
|
77 class BaseFile(object): |
|
78 ''' |
|
79 Base interface and helper for file copying. Derived class may implement |
|
80 their own copy function, or rely on BaseFile.copy using the open() member |
|
81 function and/or the path property. |
|
82 ''' |
|
83 @staticmethod |
|
84 def is_older(first, second): |
|
85 ''' |
|
86 Compares the modification time of two files, and returns whether the |
|
87 ``first`` file is older than the ``second`` file. |
|
88 ''' |
|
89 # os.path.getmtime returns a result in seconds with precision up to |
|
90 # the microsecond. But microsecond is too precise because |
|
91 # shutil.copystat only copies milliseconds, and seconds is not |
|
92 # enough precision. |
|
93 return int(os.path.getmtime(first) * 1000) \ |
|
94 <= int(os.path.getmtime(second) * 1000) |
|
95 |
|
96 @staticmethod |
|
97 def any_newer(dest, inputs): |
|
98 ''' |
|
99 Compares the modification time of ``dest`` to multiple input files, and |
|
100 returns whether any of the ``inputs`` is newer (has a later mtime) than |
|
101 ``dest``. |
|
102 ''' |
|
103 # os.path.getmtime returns a result in seconds with precision up to |
|
104 # the microsecond. But microsecond is too precise because |
|
105 # shutil.copystat only copies milliseconds, and seconds is not |
|
106 # enough precision. |
|
107 dest_mtime = int(os.path.getmtime(dest) * 1000) |
|
108 for input in inputs: |
|
109 if dest_mtime < int(os.path.getmtime(input) * 1000): |
|
110 return True |
|
111 return False |
|
112 |
|
113 def copy(self, dest, skip_if_older=True): |
|
114 ''' |
|
115 Copy the BaseFile content to the destination given as a string or a |
|
116 Dest instance. Avoids replacing existing files if the BaseFile content |
|
117 matches that of the destination, or in case of plain files, if the |
|
118 destination is newer than the original file. This latter behaviour is |
|
119 disabled when skip_if_older is False. |
|
120 Returns whether a copy was actually performed (True) or not (False). |
|
121 ''' |
|
122 if isinstance(dest, basestring): |
|
123 dest = Dest(dest) |
|
124 else: |
|
125 assert isinstance(dest, Dest) |
|
126 |
|
127 can_skip_content_check = False |
|
128 if not dest.exists(): |
|
129 can_skip_content_check = True |
|
130 elif getattr(self, 'path', None) and getattr(dest, 'path', None): |
|
131 if skip_if_older and BaseFile.is_older(self.path, dest.path): |
|
132 return False |
|
133 elif os.path.getsize(self.path) != os.path.getsize(dest.path): |
|
134 can_skip_content_check = True |
|
135 |
|
136 if can_skip_content_check: |
|
137 if getattr(self, 'path', None) and getattr(dest, 'path', None): |
|
138 shutil.copy2(self.path, dest.path) |
|
139 else: |
|
140 # Ensure the file is always created |
|
141 if not dest.exists(): |
|
142 dest.write('') |
|
143 shutil.copyfileobj(self.open(), dest) |
|
144 return True |
|
145 |
|
146 src = self.open() |
|
147 copy_content = '' |
|
148 while True: |
|
149 dest_content = dest.read(32768) |
|
150 src_content = src.read(32768) |
|
151 copy_content += src_content |
|
152 if len(dest_content) == len(src_content) == 0: |
|
153 break |
|
154 # If the read content differs between origin and destination, |
|
155 # write what was read up to now, and copy the remainder. |
|
156 if dest_content != src_content: |
|
157 dest.write(copy_content) |
|
158 shutil.copyfileobj(src, dest) |
|
159 break |
|
160 if hasattr(self, 'path') and hasattr(dest, 'path'): |
|
161 shutil.copystat(self.path, dest.path) |
|
162 return True |
|
163 |
|
164 def open(self): |
|
165 ''' |
|
166 Return a file-like object allowing to read() the content of the |
|
167 associated file. This is meant to be overloaded in subclasses to return |
|
168 a custom file-like object. |
|
169 ''' |
|
170 assert self.path is not None |
|
171 return open(self.path, 'rb') |
|
172 |
|
173 @property |
|
174 def mode(self): |
|
175 ''' |
|
176 Return the file's unix mode, or None if it has no meaning. |
|
177 ''' |
|
178 return None |
|
179 |
|
180 |
|
181 class File(BaseFile): |
|
182 ''' |
|
183 File class for plain files. |
|
184 ''' |
|
185 def __init__(self, path): |
|
186 self.path = path |
|
187 |
|
188 @property |
|
189 def mode(self): |
|
190 ''' |
|
191 Return the file's unix mode, as returned by os.stat().st_mode. |
|
192 ''' |
|
193 if platform.system() == 'Windows': |
|
194 return None |
|
195 assert self.path is not None |
|
196 return os.stat(self.path).st_mode |
|
197 |
|
198 class ExecutableFile(File): |
|
199 ''' |
|
200 File class for executable and library files on OS/2, OS/X and ELF systems. |
|
201 (see mozpack.executables.is_executable documentation). |
|
202 ''' |
|
203 def copy(self, dest, skip_if_older=True): |
|
204 real_dest = dest |
|
205 if not isinstance(dest, basestring): |
|
206 fd, dest = mkstemp() |
|
207 os.close(fd) |
|
208 os.remove(dest) |
|
209 assert isinstance(dest, basestring) |
|
210 # If File.copy didn't actually copy because dest is newer, check the |
|
211 # file sizes. If dest is smaller, it means it is already stripped and |
|
212 # elfhacked, so we can skip. |
|
213 if not File.copy(self, dest, skip_if_older) and \ |
|
214 os.path.getsize(self.path) > os.path.getsize(dest): |
|
215 return False |
|
216 try: |
|
217 if may_strip(dest): |
|
218 strip(dest) |
|
219 if may_elfhack(dest): |
|
220 elfhack(dest) |
|
221 except ErrorMessage: |
|
222 os.remove(dest) |
|
223 raise |
|
224 |
|
225 if real_dest != dest: |
|
226 f = File(dest) |
|
227 ret = f.copy(real_dest, skip_if_older) |
|
228 os.remove(dest) |
|
229 return ret |
|
230 return True |
|
231 |
|
232 |
|
233 class AbsoluteSymlinkFile(File): |
|
234 '''File class that is copied by symlinking (if available). |
|
235 |
|
236 This class only works if the target path is absolute. |
|
237 ''' |
|
238 |
|
239 def __init__(self, path): |
|
240 if not os.path.isabs(path): |
|
241 raise ValueError('Symlink target not absolute: %s' % path) |
|
242 |
|
243 File.__init__(self, path) |
|
244 |
|
245 def copy(self, dest, skip_if_older=True): |
|
246 assert isinstance(dest, basestring) |
|
247 |
|
248 # The logic in this function is complicated by the fact that symlinks |
|
249 # aren't universally supported. So, where symlinks aren't supported, we |
|
250 # fall back to file copying. Keep in mind that symlink support is |
|
251 # per-filesystem, not per-OS. |
|
252 |
|
253 # Handle the simple case where symlinks are definitely not supported by |
|
254 # falling back to file copy. |
|
255 if not hasattr(os, 'symlink'): |
|
256 return File.copy(self, dest, skip_if_older=skip_if_older) |
|
257 |
|
258 # Always verify the symlink target path exists. |
|
259 if not os.path.exists(self.path): |
|
260 raise ErrorMessage('Symlink target path does not exist: %s' % self.path) |
|
261 |
|
262 st = None |
|
263 |
|
264 try: |
|
265 st = os.lstat(dest) |
|
266 except OSError as ose: |
|
267 if ose.errno != errno.ENOENT: |
|
268 raise |
|
269 |
|
270 # If the dest is a symlink pointing to us, we have nothing to do. |
|
271 # If it's the wrong symlink, the filesystem must support symlinks, |
|
272 # so we replace with a proper symlink. |
|
273 if st and stat.S_ISLNK(st.st_mode): |
|
274 link = os.readlink(dest) |
|
275 if link == self.path: |
|
276 return False |
|
277 |
|
278 os.remove(dest) |
|
279 os.symlink(self.path, dest) |
|
280 return True |
|
281 |
|
282 # If the destination doesn't exist, we try to create a symlink. If that |
|
283 # fails, we fall back to copy code. |
|
284 if not st: |
|
285 try: |
|
286 os.symlink(self.path, dest) |
|
287 return True |
|
288 except OSError: |
|
289 return File.copy(self, dest, skip_if_older=skip_if_older) |
|
290 |
|
291 # Now the complicated part. If the destination exists, we could be |
|
292 # replacing a file with a symlink. Or, the filesystem may not support |
|
293 # symlinks. We want to minimize I/O overhead for performance reasons, |
|
294 # so we keep the existing destination file around as long as possible. |
|
295 # A lot of the system calls would be eliminated if we cached whether |
|
296 # symlinks are supported. However, even if we performed a single |
|
297 # up-front test of whether the root of the destination directory |
|
298 # supports symlinks, there's no guarantee that all operations for that |
|
299 # dest (or source) would be on the same filesystem and would support |
|
300 # symlinks. |
|
301 # |
|
302 # Our strategy is to attempt to create a new symlink with a random |
|
303 # name. If that fails, we fall back to copy mode. If that works, we |
|
304 # remove the old destination and move the newly-created symlink into |
|
305 # its place. |
|
306 |
|
307 temp_dest = os.path.join(os.path.dirname(dest), str(uuid.uuid4())) |
|
308 try: |
|
309 os.symlink(self.path, temp_dest) |
|
310 # TODO Figure out exactly how symlink creation fails and only trap |
|
311 # that. |
|
312 except EnvironmentError: |
|
313 return File.copy(self, dest, skip_if_older=skip_if_older) |
|
314 |
|
315 # If removing the original file fails, don't forget to clean up the |
|
316 # temporary symlink. |
|
317 try: |
|
318 os.remove(dest) |
|
319 except EnvironmentError: |
|
320 os.remove(temp_dest) |
|
321 raise |
|
322 |
|
323 os.rename(temp_dest, dest) |
|
324 return True |
|
325 |
|
326 |
|
327 class ExistingFile(BaseFile): |
|
328 ''' |
|
329 File class that represents a file that may exist but whose content comes |
|
330 from elsewhere. |
|
331 |
|
332 This purpose of this class is to account for files that are installed via |
|
333 external means. It is typically only used in manifests or in registries to |
|
334 account for files. |
|
335 |
|
336 When asked to copy, this class does nothing because nothing is known about |
|
337 the source file/data. |
|
338 |
|
339 Instances of this class come in two flavors: required and optional. If an |
|
340 existing file is required, it must exist during copy() or an error is |
|
341 raised. |
|
342 ''' |
|
343 def __init__(self, required): |
|
344 self.required = required |
|
345 |
|
346 def copy(self, dest, skip_if_older=True): |
|
347 if isinstance(dest, basestring): |
|
348 dest = Dest(dest) |
|
349 else: |
|
350 assert isinstance(dest, Dest) |
|
351 |
|
352 if not self.required: |
|
353 return |
|
354 |
|
355 if not dest.exists(): |
|
356 errors.fatal("Required existing file doesn't exist: %s" % |
|
357 dest.path) |
|
358 |
|
359 |
|
360 class PreprocessedFile(BaseFile): |
|
361 ''' |
|
362 File class for a file that is preprocessed. PreprocessedFile.copy() runs |
|
363 the preprocessor on the file to create the output. |
|
364 ''' |
|
365 def __init__(self, path, depfile_path, marker, defines, extra_depends=None): |
|
366 self.path = path |
|
367 self.depfile = depfile_path |
|
368 self.marker = marker |
|
369 self.defines = defines |
|
370 self.extra_depends = list(extra_depends or []) |
|
371 |
|
372 def copy(self, dest, skip_if_older=True): |
|
373 ''' |
|
374 Invokes the preprocessor to create the destination file. |
|
375 ''' |
|
376 if isinstance(dest, basestring): |
|
377 dest = Dest(dest) |
|
378 else: |
|
379 assert isinstance(dest, Dest) |
|
380 |
|
381 # We have to account for the case where the destination exists and is a |
|
382 # symlink to something. Since we know the preprocessor is certainly not |
|
383 # going to create a symlink, we can just remove the existing one. If the |
|
384 # destination is not a symlink, we leave it alone, since we're going to |
|
385 # overwrite its contents anyway. |
|
386 # If symlinks aren't supported at all, we can skip this step. |
|
387 if hasattr(os, 'symlink'): |
|
388 if os.path.islink(dest.path): |
|
389 os.remove(dest.path) |
|
390 |
|
391 pp_deps = set(self.extra_depends) |
|
392 |
|
393 # If a dependency file was specified, and it exists, add any |
|
394 # dependencies from that file to our list. |
|
395 if self.depfile and os.path.exists(self.depfile): |
|
396 target = mozpack.path.normpath(dest.name) |
|
397 with open(self.depfile, 'rb') as fileobj: |
|
398 for rule in makeutil.read_dep_makefile(fileobj): |
|
399 if target in rule.targets(): |
|
400 pp_deps.update(rule.dependencies()) |
|
401 |
|
402 skip = False |
|
403 if dest.exists() and skip_if_older: |
|
404 # If a dependency file was specified, and it doesn't exist, |
|
405 # assume that the preprocessor needs to be rerun. That will |
|
406 # regenerate the dependency file. |
|
407 if self.depfile and not os.path.exists(self.depfile): |
|
408 skip = False |
|
409 else: |
|
410 skip = not BaseFile.any_newer(dest.path, pp_deps) |
|
411 |
|
412 if skip: |
|
413 return False |
|
414 |
|
415 deps_out = None |
|
416 if self.depfile: |
|
417 deps_out = FileAvoidWrite(self.depfile) |
|
418 pp = Preprocessor(defines=self.defines, marker=self.marker) |
|
419 |
|
420 with open(self.path, 'rU') as input: |
|
421 pp.processFile(input=input, output=dest, depfile=deps_out) |
|
422 |
|
423 dest.close() |
|
424 if self.depfile: |
|
425 deps_out.close() |
|
426 |
|
427 return True |
|
428 |
|
429 |
|
430 class GeneratedFile(BaseFile): |
|
431 ''' |
|
432 File class for content with no previous existence on the filesystem. |
|
433 ''' |
|
434 def __init__(self, content): |
|
435 self.content = content |
|
436 |
|
437 def open(self): |
|
438 return BytesIO(self.content) |
|
439 |
|
440 |
|
441 class DeflatedFile(BaseFile): |
|
442 ''' |
|
443 File class for members of a jar archive. DeflatedFile.copy() effectively |
|
444 extracts the file from the jar archive. |
|
445 ''' |
|
446 def __init__(self, file): |
|
447 from mozpack.mozjar import JarFileReader |
|
448 assert isinstance(file, JarFileReader) |
|
449 self.file = file |
|
450 |
|
451 def open(self): |
|
452 self.file.seek(0) |
|
453 return self.file |
|
454 |
|
455 |
|
456 class XPTFile(GeneratedFile): |
|
457 ''' |
|
458 File class for a linked XPT file. It takes several XPT files as input |
|
459 (using the add() and remove() member functions), and links them at copy() |
|
460 time. |
|
461 ''' |
|
462 def __init__(self): |
|
463 self._files = set() |
|
464 |
|
465 def add(self, xpt): |
|
466 ''' |
|
467 Add the given XPT file (as a BaseFile instance) to the list of XPTs |
|
468 to link. |
|
469 ''' |
|
470 assert isinstance(xpt, BaseFile) |
|
471 self._files.add(xpt) |
|
472 |
|
473 def remove(self, xpt): |
|
474 ''' |
|
475 Remove the given XPT file (as a BaseFile instance) from the list of |
|
476 XPTs to link. |
|
477 ''' |
|
478 assert isinstance(xpt, BaseFile) |
|
479 self._files.remove(xpt) |
|
480 |
|
481 def copy(self, dest, skip_if_older=True): |
|
482 ''' |
|
483 Link the registered XPTs and place the resulting linked XPT at the |
|
484 destination given as a string or a Dest instance. Avoids an expensive |
|
485 XPT linking if the interfaces in an existing destination match those of |
|
486 the individual XPTs to link. |
|
487 skip_if_older is ignored. |
|
488 ''' |
|
489 if isinstance(dest, basestring): |
|
490 dest = Dest(dest) |
|
491 assert isinstance(dest, Dest) |
|
492 |
|
493 from xpt import xpt_link, Typelib, Interface |
|
494 all_typelibs = [Typelib.read(f.open()) for f in self._files] |
|
495 if dest.exists(): |
|
496 # Typelib.read() needs to seek(), so use a BytesIO for dest |
|
497 # content. |
|
498 dest_interfaces = \ |
|
499 dict((i.name, i) |
|
500 for i in Typelib.read(BytesIO(dest.read())).interfaces |
|
501 if i.iid != Interface.UNRESOLVED_IID) |
|
502 identical = True |
|
503 for f in self._files: |
|
504 typelib = Typelib.read(f.open()) |
|
505 for i in typelib.interfaces: |
|
506 if i.iid != Interface.UNRESOLVED_IID and \ |
|
507 not (i.name in dest_interfaces and |
|
508 i == dest_interfaces[i.name]): |
|
509 identical = False |
|
510 break |
|
511 if identical: |
|
512 return False |
|
513 s = BytesIO() |
|
514 xpt_link(all_typelibs).write(s) |
|
515 dest.write(s.getvalue()) |
|
516 return True |
|
517 |
|
518 def open(self): |
|
519 raise RuntimeError("Unsupported") |
|
520 |
|
521 def isempty(self): |
|
522 ''' |
|
523 Return whether there are XPT files to link. |
|
524 ''' |
|
525 return len(self._files) == 0 |
|
526 |
|
527 |
|
528 class ManifestFile(BaseFile): |
|
529 ''' |
|
530 File class for a manifest file. It takes individual manifest entries (using |
|
531 the add() and remove() member functions), and adjusts them to be relative |
|
532 to the base path for the manifest, given at creation. |
|
533 Example: |
|
534 There is a manifest entry "content webapprt webapprt/content/" relative |
|
535 to "webapprt/chrome". When packaging, the entry will be stored in |
|
536 jar:webapprt/omni.ja!/chrome/chrome.manifest, which means the entry |
|
537 will have to be relative to "chrome" instead of "webapprt/chrome". This |
|
538 doesn't really matter when serializing the entry, since this base path |
|
539 is not written out, but it matters when moving the entry at the same |
|
540 time, e.g. to jar:webapprt/omni.ja!/chrome.manifest, which we don't do |
|
541 currently but could in the future. |
|
542 ''' |
|
543 def __init__(self, base, entries=None): |
|
544 self._entries = entries if entries else [] |
|
545 self._base = base |
|
546 |
|
547 def add(self, entry): |
|
548 ''' |
|
549 Add the given entry to the manifest. Entries are rebased at open() time |
|
550 instead of add() time so that they can be more easily remove()d. |
|
551 ''' |
|
552 assert isinstance(entry, ManifestEntry) |
|
553 self._entries.append(entry) |
|
554 |
|
555 def remove(self, entry): |
|
556 ''' |
|
557 Remove the given entry from the manifest. |
|
558 ''' |
|
559 assert isinstance(entry, ManifestEntry) |
|
560 self._entries.remove(entry) |
|
561 |
|
562 def open(self): |
|
563 ''' |
|
564 Return a file-like object allowing to read() the serialized content of |
|
565 the manifest. |
|
566 ''' |
|
567 return BytesIO(''.join('%s\n' % e.rebase(self._base) |
|
568 for e in self._entries)) |
|
569 |
|
570 def __iter__(self): |
|
571 ''' |
|
572 Iterate over entries in the manifest file. |
|
573 ''' |
|
574 return iter(self._entries) |
|
575 |
|
576 def isempty(self): |
|
577 ''' |
|
578 Return whether there are manifest entries to write |
|
579 ''' |
|
580 return len(self._entries) == 0 |
|
581 |
|
582 |
|
583 class MinifiedProperties(BaseFile): |
|
584 ''' |
|
585 File class for minified properties. This wraps around a BaseFile instance, |
|
586 and removes lines starting with a # from its content. |
|
587 ''' |
|
588 def __init__(self, file): |
|
589 assert isinstance(file, BaseFile) |
|
590 self._file = file |
|
591 |
|
592 def open(self): |
|
593 ''' |
|
594 Return a file-like object allowing to read() the minified content of |
|
595 the properties file. |
|
596 ''' |
|
597 return BytesIO(''.join(l for l in self._file.open().readlines() |
|
598 if not l.startswith('#'))) |
|
599 |
|
600 |
|
601 class MinifiedJavaScript(BaseFile): |
|
602 ''' |
|
603 File class for minifying JavaScript files. |
|
604 ''' |
|
605 def __init__(self, file, verify_command=None): |
|
606 assert isinstance(file, BaseFile) |
|
607 self._file = file |
|
608 self._verify_command = verify_command |
|
609 |
|
610 def open(self): |
|
611 output = BytesIO() |
|
612 minify = JavascriptMinify(self._file.open(), output) |
|
613 minify.minify() |
|
614 output.seek(0) |
|
615 |
|
616 if not self._verify_command: |
|
617 return output |
|
618 |
|
619 input_source = self._file.open().read() |
|
620 output_source = output.getvalue() |
|
621 |
|
622 with NamedTemporaryFile() as fh1, NamedTemporaryFile() as fh2: |
|
623 fh1.write(input_source) |
|
624 fh2.write(output_source) |
|
625 fh1.flush() |
|
626 fh2.flush() |
|
627 |
|
628 try: |
|
629 args = list(self._verify_command) |
|
630 args.extend([fh1.name, fh2.name]) |
|
631 subprocess.check_output(args, stderr=subprocess.STDOUT) |
|
632 except subprocess.CalledProcessError as e: |
|
633 errors.warn('JS minification verification failed for %s:' % |
|
634 (getattr(self._file, 'path', '<unknown>'))) |
|
635 # Prefix each line with "Warning:" so mozharness doesn't |
|
636 # think these error messages are real errors. |
|
637 for line in e.output.splitlines(): |
|
638 errors.warn(line) |
|
639 |
|
640 return self._file.open() |
|
641 |
|
642 return output |
|
643 |
|
644 |
|
645 class BaseFinder(object): |
|
646 def __init__(self, base, minify=False, minify_js=False, |
|
647 minify_js_verify_command=None): |
|
648 ''' |
|
649 Initializes the instance with a reference base directory. |
|
650 |
|
651 The optional minify argument specifies whether minification of code |
|
652 should occur. minify_js is an additional option to control minification |
|
653 of JavaScript. It requires minify to be True. |
|
654 |
|
655 minify_js_verify_command can be used to optionally verify the results |
|
656 of JavaScript minification. If defined, it is expected to be an iterable |
|
657 that will constitute the first arguments to a called process which will |
|
658 receive the filenames of the original and minified JavaScript files. |
|
659 The invoked process can then verify the results. If minification is |
|
660 rejected, the process exits with a non-0 exit code and the original |
|
661 JavaScript source is used. An example value for this argument is |
|
662 ('/path/to/js', '/path/to/verify/script.js'). |
|
663 ''' |
|
664 if minify_js and not minify: |
|
665 raise ValueError('minify_js requires minify.') |
|
666 |
|
667 self.base = base |
|
668 self._minify = minify |
|
669 self._minify_js = minify_js |
|
670 self._minify_js_verify_command = minify_js_verify_command |
|
671 |
|
672 def find(self, pattern): |
|
673 ''' |
|
674 Yield path, BaseFile_instance pairs for all files under the base |
|
675 directory and its subdirectories that match the given pattern. See the |
|
676 mozpack.path.match documentation for a description of the handled |
|
677 patterns. |
|
678 ''' |
|
679 while pattern.startswith('/'): |
|
680 pattern = pattern[1:] |
|
681 for p, f in self._find(pattern): |
|
682 yield p, self._minify_file(p, f) |
|
683 |
|
684 def __iter__(self): |
|
685 ''' |
|
686 Iterates over all files under the base directory (excluding files |
|
687 starting with a '.' and files at any level under a directory starting |
|
688 with a '.'). |
|
689 for path, file in finder: |
|
690 ... |
|
691 ''' |
|
692 return self.find('') |
|
693 |
|
694 def __contains__(self, pattern): |
|
695 raise RuntimeError("'in' operator forbidden for %s. Use contains()." % |
|
696 self.__class__.__name__) |
|
697 |
|
698 def contains(self, pattern): |
|
699 ''' |
|
700 Return whether some files under the base directory match the given |
|
701 pattern. See the mozpack.path.match documentation for a description of |
|
702 the handled patterns. |
|
703 ''' |
|
704 return any(self.find(pattern)) |
|
705 |
|
706 def _minify_file(self, path, file): |
|
707 ''' |
|
708 Return an appropriate MinifiedSomething wrapper for the given BaseFile |
|
709 instance (file), according to the file type (determined by the given |
|
710 path), if the FileFinder was created with minification enabled. |
|
711 Otherwise, just return the given BaseFile instance. |
|
712 ''' |
|
713 if not self._minify or isinstance(file, ExecutableFile): |
|
714 return file |
|
715 |
|
716 if path.endswith('.properties'): |
|
717 return MinifiedProperties(file) |
|
718 |
|
719 if self._minify_js and path.endswith(('.js', '.jsm')): |
|
720 return MinifiedJavaScript(file, self._minify_js_verify_command) |
|
721 |
|
722 return file |
|
723 |
|
724 |
|
725 class FileFinder(BaseFinder): |
|
726 ''' |
|
727 Helper to get appropriate BaseFile instances from the file system. |
|
728 ''' |
|
729 def __init__(self, base, find_executables=True, ignore=(), **kargs): |
|
730 ''' |
|
731 Create a FileFinder for files under the given base directory. |
|
732 |
|
733 The find_executables argument determines whether the finder needs to |
|
734 try to guess whether files are executables. Disabling this guessing |
|
735 when not necessary can speed up the finder significantly. |
|
736 |
|
737 ``ignore`` accepts an iterable of patterns to ignore. Entries are |
|
738 strings that match paths relative to ``base`` using |
|
739 ``mozpack.path.match()``. This means if an entry corresponds |
|
740 to a directory, all files under that directory will be ignored. If |
|
741 an entry corresponds to a file, that particular file will be ignored. |
|
742 ''' |
|
743 BaseFinder.__init__(self, base, **kargs) |
|
744 self.find_executables = find_executables |
|
745 self.ignore = ignore |
|
746 |
|
747 def _find(self, pattern): |
|
748 ''' |
|
749 Actual implementation of FileFinder.find(), dispatching to specialized |
|
750 member functions depending on what kind of pattern was given. |
|
751 Note all files with a name starting with a '.' are ignored when |
|
752 scanning directories, but are not ignored when explicitely requested. |
|
753 ''' |
|
754 if '*' in pattern: |
|
755 return self._find_glob('', mozpack.path.split(pattern)) |
|
756 elif os.path.isdir(os.path.join(self.base, pattern)): |
|
757 return self._find_dir(pattern) |
|
758 else: |
|
759 return self._find_file(pattern) |
|
760 |
|
761 def _find_dir(self, path): |
|
762 ''' |
|
763 Actual implementation of FileFinder.find() when the given pattern |
|
764 corresponds to an existing directory under the base directory. |
|
765 Ignores file names starting with a '.' under the given path. If the |
|
766 path itself has leafs starting with a '.', they are not ignored. |
|
767 ''' |
|
768 for p in self.ignore: |
|
769 if mozpack.path.match(path, p): |
|
770 return |
|
771 |
|
772 # The sorted makes the output idempotent. Otherwise, we are |
|
773 # likely dependent on filesystem implementation details, such as |
|
774 # inode ordering. |
|
775 for p in sorted(os.listdir(os.path.join(self.base, path))): |
|
776 if p.startswith('.'): |
|
777 continue |
|
778 for p_, f in self._find(mozpack.path.join(path, p)): |
|
779 yield p_, f |
|
780 |
|
781 def _find_file(self, path): |
|
782 ''' |
|
783 Actual implementation of FileFinder.find() when the given pattern |
|
784 corresponds to an existing file under the base directory. |
|
785 ''' |
|
786 srcpath = os.path.join(self.base, path) |
|
787 if not os.path.exists(srcpath): |
|
788 return |
|
789 |
|
790 for p in self.ignore: |
|
791 if mozpack.path.match(path, p): |
|
792 return |
|
793 |
|
794 if self.find_executables and is_executable(srcpath): |
|
795 yield path, ExecutableFile(srcpath) |
|
796 else: |
|
797 yield path, File(srcpath) |
|
798 |
|
799 def _find_glob(self, base, pattern): |
|
800 ''' |
|
801 Actual implementation of FileFinder.find() when the given pattern |
|
802 contains globbing patterns ('*' or '**'). This is meant to be an |
|
803 equivalent of: |
|
804 for p, f in self: |
|
805 if mozpack.path.match(p, pattern): |
|
806 yield p, f |
|
807 but avoids scanning the entire tree. |
|
808 ''' |
|
809 if not pattern: |
|
810 for p, f in self._find(base): |
|
811 yield p, f |
|
812 elif pattern[0] == '**': |
|
813 for p, f in self._find(base): |
|
814 if mozpack.path.match(p, mozpack.path.join(*pattern)): |
|
815 yield p, f |
|
816 elif '*' in pattern[0]: |
|
817 if not os.path.exists(os.path.join(self.base, base)): |
|
818 return |
|
819 |
|
820 for p in self.ignore: |
|
821 if mozpack.path.match(base, p): |
|
822 return |
|
823 |
|
824 # See above comment w.r.t. sorted() and idempotent behavior. |
|
825 for p in sorted(os.listdir(os.path.join(self.base, base))): |
|
826 if p.startswith('.') and not pattern[0].startswith('.'): |
|
827 continue |
|
828 if mozpack.path.match(p, pattern[0]): |
|
829 for p_, f in self._find_glob(mozpack.path.join(base, p), |
|
830 pattern[1:]): |
|
831 yield p_, f |
|
832 else: |
|
833 for p, f in self._find_glob(mozpack.path.join(base, pattern[0]), |
|
834 pattern[1:]): |
|
835 yield p, f |
|
836 |
|
837 |
|
838 class JarFinder(BaseFinder): |
|
839 ''' |
|
840 Helper to get appropriate DeflatedFile instances from a JarReader. |
|
841 ''' |
|
842 def __init__(self, base, reader, **kargs): |
|
843 ''' |
|
844 Create a JarFinder for files in the given JarReader. The base argument |
|
845 is used as an indication of the Jar file location. |
|
846 ''' |
|
847 assert isinstance(reader, JarReader) |
|
848 BaseFinder.__init__(self, base, **kargs) |
|
849 self._files = OrderedDict((f.filename, f) for f in reader) |
|
850 |
|
851 def _find(self, pattern): |
|
852 ''' |
|
853 Actual implementation of JarFinder.find(), dispatching to specialized |
|
854 member functions depending on what kind of pattern was given. |
|
855 ''' |
|
856 if '*' in pattern: |
|
857 for p in self._files: |
|
858 if mozpack.path.match(p, pattern): |
|
859 yield p, DeflatedFile(self._files[p]) |
|
860 elif pattern == '': |
|
861 for p in self._files: |
|
862 yield p, DeflatedFile(self._files[p]) |
|
863 elif pattern in self._files: |
|
864 yield pattern, DeflatedFile(self._files[pattern]) |
|
865 else: |
|
866 for p in self._files: |
|
867 if mozpack.path.basedir(p, [pattern]) == pattern: |
|
868 yield p, DeflatedFile(self._files[p]) |