config/configobj.py@97036ab72558
config/configobj.py
Tue, 06 Jan 2015 21:39:09 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Tue, 06 Jan 2015 21:39:09 +0100
- branch
- TOR_BUG_9701
- changeset 8
- 97036ab72558
- permissions
- -rw-r--r--
Conditionally force memory storage according to privacy.thirdparty.isolate;
This solves Tor bug #9701, complying with disk avoidance documented in
https://www.torproject.org/projects/torbrowser/design/#disk-avoidance.
1 # configobj.py
2 # A config file reader/writer that supports nested sections in config files.
3 # Copyright (C) 2005-2006 Michael Foord, Nicola Larosa
4 # E-mail: fuzzyman AT voidspace DOT org DOT uk
5 # nico AT tekNico DOT net
7 # ConfigObj 4
8 # http://www.voidspace.org.uk/python/configobj.html
10 # Released subject to the BSD License
11 # Please see http://www.voidspace.org.uk/python/license.shtml
13 # Scripts maintained at http://www.voidspace.org.uk/python/index.shtml
14 # For information about bugfixes, updates and support, please join the
15 # ConfigObj mailing list:
16 # http://lists.sourceforge.net/lists/listinfo/configobj-develop
17 # Comments, suggestions and bug reports welcome.
19 from __future__ import generators
21 import sys
22 INTP_VER = sys.version_info[:2]
23 if INTP_VER < (2, 2):
24 raise RuntimeError("Python v.2.2 or later needed")
26 import os, re
27 compiler = None
28 try:
29 import compiler
30 except ImportError:
31 # for IronPython
32 pass
33 from types import StringTypes
34 from warnings import warn
35 try:
36 from codecs import BOM_UTF8, BOM_UTF16, BOM_UTF16_BE, BOM_UTF16_LE
37 except ImportError:
38 # Python 2.2 does not have these
39 # UTF-8
40 BOM_UTF8 = '\xef\xbb\xbf'
41 # UTF-16, little endian
42 BOM_UTF16_LE = '\xff\xfe'
43 # UTF-16, big endian
44 BOM_UTF16_BE = '\xfe\xff'
45 if sys.byteorder == 'little':
46 # UTF-16, native endianness
47 BOM_UTF16 = BOM_UTF16_LE
48 else:
49 # UTF-16, native endianness
50 BOM_UTF16 = BOM_UTF16_BE
52 # A dictionary mapping BOM to
53 # the encoding to decode with, and what to set the
54 # encoding attribute to.
55 BOMS = {
56 BOM_UTF8: ('utf_8', None),
57 BOM_UTF16_BE: ('utf16_be', 'utf_16'),
58 BOM_UTF16_LE: ('utf16_le', 'utf_16'),
59 BOM_UTF16: ('utf_16', 'utf_16'),
60 }
61 # All legal variants of the BOM codecs.
62 # TODO: the list of aliases is not meant to be exhaustive, is there a
63 # better way ?
64 BOM_LIST = {
65 'utf_16': 'utf_16',
66 'u16': 'utf_16',
67 'utf16': 'utf_16',
68 'utf-16': 'utf_16',
69 'utf16_be': 'utf16_be',
70 'utf_16_be': 'utf16_be',
71 'utf-16be': 'utf16_be',
72 'utf16_le': 'utf16_le',
73 'utf_16_le': 'utf16_le',
74 'utf-16le': 'utf16_le',
75 'utf_8': 'utf_8',
76 'u8': 'utf_8',
77 'utf': 'utf_8',
78 'utf8': 'utf_8',
79 'utf-8': 'utf_8',
80 }
82 # Map of encodings to the BOM to write.
83 BOM_SET = {
84 'utf_8': BOM_UTF8,
85 'utf_16': BOM_UTF16,
86 'utf16_be': BOM_UTF16_BE,
87 'utf16_le': BOM_UTF16_LE,
88 None: BOM_UTF8
89 }
91 try:
92 from validate import VdtMissingValue
93 except ImportError:
94 VdtMissingValue = None
96 try:
97 enumerate
98 except NameError:
99 def enumerate(obj):
100 """enumerate for Python 2.2."""
101 i = -1
102 for item in obj:
103 i += 1
104 yield i, item
106 try:
107 True, False
108 except NameError:
109 True, False = 1, 0
112 __version__ = '4.4.0'
114 __revision__ = '$Id: configobj.py,v 3.5 2007/07/02 18:20:24 benjamin%smedbergs.us Exp $'
116 __docformat__ = "restructuredtext en"
118 __all__ = (
119 '__version__',
120 'DEFAULT_INDENT_TYPE',
121 'DEFAULT_INTERPOLATION',
122 'ConfigObjError',
123 'NestingError',
124 'ParseError',
125 'DuplicateError',
126 'ConfigspecError',
127 'ConfigObj',
128 'SimpleVal',
129 'InterpolationError',
130 'InterpolationLoopError',
131 'MissingInterpolationOption',
132 'RepeatSectionError',
133 'UnreprError',
134 'UnknownType',
135 '__docformat__',
136 'flatten_errors',
137 )
139 DEFAULT_INTERPOLATION = 'configparser'
140 DEFAULT_INDENT_TYPE = ' '
141 MAX_INTERPOL_DEPTH = 10
143 OPTION_DEFAULTS = {
144 'interpolation': True,
145 'raise_errors': False,
146 'list_values': True,
147 'create_empty': False,
148 'file_error': False,
149 'configspec': None,
150 'stringify': True,
151 # option may be set to one of ('', ' ', '\t')
152 'indent_type': None,
153 'encoding': None,
154 'default_encoding': None,
155 'unrepr': False,
156 'write_empty_values': False,
157 }
160 def getObj(s):
161 s = "a=" + s
162 if compiler is None:
163 raise ImportError('compiler module not available')
164 p = compiler.parse(s)
165 return p.getChildren()[1].getChildren()[0].getChildren()[1]
167 class UnknownType(Exception):
168 pass
170 class Builder:
172 def build(self, o):
173 m = getattr(self, 'build_' + o.__class__.__name__, None)
174 if m is None:
175 raise UnknownType(o.__class__.__name__)
176 return m(o)
178 def build_List(self, o):
179 return map(self.build, o.getChildren())
181 def build_Const(self, o):
182 return o.value
184 def build_Dict(self, o):
185 d = {}
186 i = iter(map(self.build, o.getChildren()))
187 for el in i:
188 d[el] = i.next()
189 return d
191 def build_Tuple(self, o):
192 return tuple(self.build_List(o))
194 def build_Name(self, o):
195 if o.name == 'None':
196 return None
197 if o.name == 'True':
198 return True
199 if o.name == 'False':
200 return False
202 # An undefinted Name
203 raise UnknownType('Undefined Name')
205 def build_Add(self, o):
206 real, imag = map(self.build_Const, o.getChildren())
207 try:
208 real = float(real)
209 except TypeError:
210 raise UnknownType('Add')
211 if not isinstance(imag, complex) or imag.real != 0.0:
212 raise UnknownType('Add')
213 return real+imag
215 def build_Getattr(self, o):
216 parent = self.build(o.expr)
217 return getattr(parent, o.attrname)
219 def build_UnarySub(self, o):
220 return -self.build_Const(o.getChildren()[0])
222 def build_UnaryAdd(self, o):
223 return self.build_Const(o.getChildren()[0])
225 def unrepr(s):
226 if not s:
227 return s
228 return Builder().build(getObj(s))
230 def _splitlines(instring):
231 """Split a string on lines, without losing line endings or truncating."""
234 class ConfigObjError(SyntaxError):
235 """
236 This is the base class for all errors that ConfigObj raises.
237 It is a subclass of SyntaxError.
238 """
239 def __init__(self, message='', line_number=None, line=''):
240 self.line = line
241 self.line_number = line_number
242 self.message = message
243 SyntaxError.__init__(self, message)
245 class NestingError(ConfigObjError):
246 """
247 This error indicates a level of nesting that doesn't match.
248 """
250 class ParseError(ConfigObjError):
251 """
252 This error indicates that a line is badly written.
253 It is neither a valid ``key = value`` line,
254 nor a valid section marker line.
255 """
257 class DuplicateError(ConfigObjError):
258 """
259 The keyword or section specified already exists.
260 """
262 class ConfigspecError(ConfigObjError):
263 """
264 An error occurred whilst parsing a configspec.
265 """
267 class InterpolationError(ConfigObjError):
268 """Base class for the two interpolation errors."""
270 class InterpolationLoopError(InterpolationError):
271 """Maximum interpolation depth exceeded in string interpolation."""
273 def __init__(self, option):
274 InterpolationError.__init__(
275 self,
276 'interpolation loop detected in value "%s".' % option)
278 class RepeatSectionError(ConfigObjError):
279 """
280 This error indicates additional sections in a section with a
281 ``__many__`` (repeated) section.
282 """
284 class MissingInterpolationOption(InterpolationError):
285 """A value specified for interpolation was missing."""
287 def __init__(self, option):
288 InterpolationError.__init__(
289 self,
290 'missing option "%s" in interpolation.' % option)
292 class UnreprError(ConfigObjError):
293 """An error parsing in unrepr mode."""
296 class InterpolationEngine(object):
297 """
298 A helper class to help perform string interpolation.
300 This class is an abstract base class; its descendants perform
301 the actual work.
302 """
304 # compiled regexp to use in self.interpolate()
305 _KEYCRE = re.compile(r"%\(([^)]*)\)s")
307 def __init__(self, section):
308 # the Section instance that "owns" this engine
309 self.section = section
311 def interpolate(self, key, value):
312 def recursive_interpolate(key, value, section, backtrail):
313 """The function that does the actual work.
315 ``value``: the string we're trying to interpolate.
316 ``section``: the section in which that string was found
317 ``backtrail``: a dict to keep track of where we've been,
318 to detect and prevent infinite recursion loops
320 This is similar to a depth-first-search algorithm.
321 """
322 # Have we been here already?
323 if backtrail.has_key((key, section.name)):
324 # Yes - infinite loop detected
325 raise InterpolationLoopError(key)
326 # Place a marker on our backtrail so we won't come back here again
327 backtrail[(key, section.name)] = 1
329 # Now start the actual work
330 match = self._KEYCRE.search(value)
331 while match:
332 # The actual parsing of the match is implementation-dependent,
333 # so delegate to our helper function
334 k, v, s = self._parse_match(match)
335 if k is None:
336 # That's the signal that no further interpolation is needed
337 replacement = v
338 else:
339 # Further interpolation may be needed to obtain final value
340 replacement = recursive_interpolate(k, v, s, backtrail)
341 # Replace the matched string with its final value
342 start, end = match.span()
343 value = ''.join((value[:start], replacement, value[end:]))
344 new_search_start = start + len(replacement)
345 # Pick up the next interpolation key, if any, for next time
346 # through the while loop
347 match = self._KEYCRE.search(value, new_search_start)
349 # Now safe to come back here again; remove marker from backtrail
350 del backtrail[(key, section.name)]
352 return value
354 # Back in interpolate(), all we have to do is kick off the recursive
355 # function with appropriate starting values
356 value = recursive_interpolate(key, value, self.section, {})
357 return value
359 def _fetch(self, key):
360 """Helper function to fetch values from owning section.
362 Returns a 2-tuple: the value, and the section where it was found.
363 """
364 # switch off interpolation before we try and fetch anything !
365 save_interp = self.section.main.interpolation
366 self.section.main.interpolation = False
368 # Start at section that "owns" this InterpolationEngine
369 current_section = self.section
370 while True:
371 # try the current section first
372 val = current_section.get(key)
373 if val is not None:
374 break
375 # try "DEFAULT" next
376 val = current_section.get('DEFAULT', {}).get(key)
377 if val is not None:
378 break
379 # move up to parent and try again
380 # top-level's parent is itself
381 if current_section.parent is current_section:
382 # reached top level, time to give up
383 break
384 current_section = current_section.parent
386 # restore interpolation to previous value before returning
387 self.section.main.interpolation = save_interp
388 if val is None:
389 raise MissingInterpolationOption(key)
390 return val, current_section
392 def _parse_match(self, match):
393 """Implementation-dependent helper function.
395 Will be passed a match object corresponding to the interpolation
396 key we just found (e.g., "%(foo)s" or "$foo"). Should look up that
397 key in the appropriate config file section (using the ``_fetch()``
398 helper function) and return a 3-tuple: (key, value, section)
400 ``key`` is the name of the key we're looking for
401 ``value`` is the value found for that key
402 ``section`` is a reference to the section where it was found
404 ``key`` and ``section`` should be None if no further
405 interpolation should be performed on the resulting value
406 (e.g., if we interpolated "$$" and returned "$").
407 """
408 raise NotImplementedError
411 class ConfigParserInterpolation(InterpolationEngine):
412 """Behaves like ConfigParser."""
413 _KEYCRE = re.compile(r"%\(([^)]*)\)s")
415 def _parse_match(self, match):
416 key = match.group(1)
417 value, section = self._fetch(key)
418 return key, value, section
421 class TemplateInterpolation(InterpolationEngine):
422 """Behaves like string.Template."""
423 _delimiter = '$'
424 _KEYCRE = re.compile(r"""
425 \$(?:
426 (?P<escaped>\$) | # Two $ signs
427 (?P<named>[_a-z][_a-z0-9]*) | # $name format
428 {(?P<braced>[^}]*)} # ${name} format
429 )
430 """, re.IGNORECASE | re.VERBOSE)
432 def _parse_match(self, match):
433 # Valid name (in or out of braces): fetch value from section
434 key = match.group('named') or match.group('braced')
435 if key is not None:
436 value, section = self._fetch(key)
437 return key, value, section
438 # Escaped delimiter (e.g., $$): return single delimiter
439 if match.group('escaped') is not None:
440 # Return None for key and section to indicate it's time to stop
441 return None, self._delimiter, None
442 # Anything else: ignore completely, just return it unchanged
443 return None, match.group(), None
445 interpolation_engines = {
446 'configparser': ConfigParserInterpolation,
447 'template': TemplateInterpolation,
448 }
450 class Section(dict):
451 """
452 A dictionary-like object that represents a section in a config file.
454 It does string interpolation if the 'interpolation' attribute
455 of the 'main' object is set to True.
457 Interpolation is tried first from this object, then from the 'DEFAULT'
458 section of this object, next from the parent and its 'DEFAULT' section,
459 and so on until the main object is reached.
461 A Section will behave like an ordered dictionary - following the
462 order of the ``scalars`` and ``sections`` attributes.
463 You can use this to change the order of members.
465 Iteration follows the order: scalars, then sections.
466 """
468 def __init__(self, parent, depth, main, indict=None, name=None):
469 """
470 * parent is the section above
471 * depth is the depth level of this section
472 * main is the main ConfigObj
473 * indict is a dictionary to initialise the section with
474 """
475 if indict is None:
476 indict = {}
477 dict.__init__(self)
478 # used for nesting level *and* interpolation
479 self.parent = parent
480 # used for the interpolation attribute
481 self.main = main
482 # level of nesting depth of this Section
483 self.depth = depth
484 # the sequence of scalar values in this Section
485 self.scalars = []
486 # the sequence of sections in this Section
487 self.sections = []
488 # purely for information
489 self.name = name
490 # for comments :-)
491 self.comments = {}
492 self.inline_comments = {}
493 # for the configspec
494 self.configspec = {}
495 self._order = []
496 self._configspec_comments = {}
497 self._configspec_inline_comments = {}
498 self._cs_section_comments = {}
499 self._cs_section_inline_comments = {}
500 # for defaults
501 self.defaults = []
502 #
503 # we do this explicitly so that __setitem__ is used properly
504 # (rather than just passing to ``dict.__init__``)
505 for entry in indict:
506 self[entry] = indict[entry]
508 def _interpolate(self, key, value):
509 try:
510 # do we already have an interpolation engine?
511 engine = self._interpolation_engine
512 except AttributeError:
513 # not yet: first time running _interpolate(), so pick the engine
514 name = self.main.interpolation
515 if name == True: # note that "if name:" would be incorrect here
516 # backwards-compatibility: interpolation=True means use default
517 name = DEFAULT_INTERPOLATION
518 name = name.lower() # so that "Template", "template", etc. all work
519 class_ = interpolation_engines.get(name, None)
520 if class_ is None:
521 # invalid value for self.main.interpolation
522 self.main.interpolation = False
523 return value
524 else:
525 # save reference to engine so we don't have to do this again
526 engine = self._interpolation_engine = class_(self)
527 # let the engine do the actual work
528 return engine.interpolate(key, value)
530 def __getitem__(self, key):
531 """Fetch the item and do string interpolation."""
532 val = dict.__getitem__(self, key)
533 if self.main.interpolation and isinstance(val, StringTypes):
534 return self._interpolate(key, val)
535 return val
537 def __setitem__(self, key, value, unrepr=False):
538 """
539 Correctly set a value.
541 Making dictionary values Section instances.
542 (We have to special case 'Section' instances - which are also dicts)
544 Keys must be strings.
545 Values need only be strings (or lists of strings) if
546 ``main.stringify`` is set.
548 `unrepr`` must be set when setting a value to a dictionary, without
549 creating a new sub-section.
550 """
551 if not isinstance(key, StringTypes):
552 raise ValueError, 'The key "%s" is not a string.' % key
553 # add the comment
554 if not self.comments.has_key(key):
555 self.comments[key] = []
556 self.inline_comments[key] = ''
557 # remove the entry from defaults
558 if key in self.defaults:
559 self.defaults.remove(key)
560 #
561 if isinstance(value, Section):
562 if not self.has_key(key):
563 self.sections.append(key)
564 dict.__setitem__(self, key, value)
565 elif isinstance(value, dict) and not unrepr:
566 # First create the new depth level,
567 # then create the section
568 if not self.has_key(key):
569 self.sections.append(key)
570 new_depth = self.depth + 1
571 dict.__setitem__(
572 self,
573 key,
574 Section(
575 self,
576 new_depth,
577 self.main,
578 indict=value,
579 name=key))
580 else:
581 if not self.has_key(key):
582 self.scalars.append(key)
583 if not self.main.stringify:
584 if isinstance(value, StringTypes):
585 pass
586 elif isinstance(value, (list, tuple)):
587 for entry in value:
588 if not isinstance(entry, StringTypes):
589 raise TypeError, (
590 'Value is not a string "%s".' % entry)
591 else:
592 raise TypeError, 'Value is not a string "%s".' % value
593 dict.__setitem__(self, key, value)
595 def __delitem__(self, key):
596 """Remove items from the sequence when deleting."""
597 dict. __delitem__(self, key)
598 if key in self.scalars:
599 self.scalars.remove(key)
600 else:
601 self.sections.remove(key)
602 del self.comments[key]
603 del self.inline_comments[key]
605 def get(self, key, default=None):
606 """A version of ``get`` that doesn't bypass string interpolation."""
607 try:
608 return self[key]
609 except KeyError:
610 return default
612 def update(self, indict):
613 """
614 A version of update that uses our ``__setitem__``.
615 """
616 for entry in indict:
617 self[entry] = indict[entry]
619 def pop(self, key, *args):
620 """ """
621 val = dict.pop(self, key, *args)
622 if key in self.scalars:
623 del self.comments[key]
624 del self.inline_comments[key]
625 self.scalars.remove(key)
626 elif key in self.sections:
627 del self.comments[key]
628 del self.inline_comments[key]
629 self.sections.remove(key)
630 if self.main.interpolation and isinstance(val, StringTypes):
631 return self._interpolate(key, val)
632 return val
634 def popitem(self):
635 """Pops the first (key,val)"""
636 sequence = (self.scalars + self.sections)
637 if not sequence:
638 raise KeyError, ": 'popitem(): dictionary is empty'"
639 key = sequence[0]
640 val = self[key]
641 del self[key]
642 return key, val
644 def clear(self):
645 """
646 A version of clear that also affects scalars/sections
647 Also clears comments and configspec.
649 Leaves other attributes alone :
650 depth/main/parent are not affected
651 """
652 dict.clear(self)
653 self.scalars = []
654 self.sections = []
655 self.comments = {}
656 self.inline_comments = {}
657 self.configspec = {}
659 def setdefault(self, key, default=None):
660 """A version of setdefault that sets sequence if appropriate."""
661 try:
662 return self[key]
663 except KeyError:
664 self[key] = default
665 return self[key]
667 def items(self):
668 """ """
669 return zip((self.scalars + self.sections), self.values())
671 def keys(self):
672 """ """
673 return (self.scalars + self.sections)
675 def values(self):
676 """ """
677 return [self[key] for key in (self.scalars + self.sections)]
679 def iteritems(self):
680 """ """
681 return iter(self.items())
683 def iterkeys(self):
684 """ """
685 return iter((self.scalars + self.sections))
687 __iter__ = iterkeys
689 def itervalues(self):
690 """ """
691 return iter(self.values())
693 def __repr__(self):
694 return '{%s}' % ', '.join([('%s: %s' % (repr(key), repr(self[key])))
695 for key in (self.scalars + self.sections)])
697 __str__ = __repr__
699 # Extra methods - not in a normal dictionary
701 def dict(self):
702 """
703 Return a deepcopy of self as a dictionary.
705 All members that are ``Section`` instances are recursively turned to
706 ordinary dictionaries - by calling their ``dict`` method.
708 >>> n = a.dict()
709 >>> n == a
710 1
711 >>> n is a
712 0
713 """
714 newdict = {}
715 for entry in self:
716 this_entry = self[entry]
717 if isinstance(this_entry, Section):
718 this_entry = this_entry.dict()
719 elif isinstance(this_entry, list):
720 # create a copy rather than a reference
721 this_entry = list(this_entry)
722 elif isinstance(this_entry, tuple):
723 # create a copy rather than a reference
724 this_entry = tuple(this_entry)
725 newdict[entry] = this_entry
726 return newdict
728 def merge(self, indict):
729 """
730 A recursive update - useful for merging config files.
732 >>> a = '''[section1]
733 ... option1 = True
734 ... [[subsection]]
735 ... more_options = False
736 ... # end of file'''.splitlines()
737 >>> b = '''# File is user.ini
738 ... [section1]
739 ... option1 = False
740 ... # end of file'''.splitlines()
741 >>> c1 = ConfigObj(b)
742 >>> c2 = ConfigObj(a)
743 >>> c2.merge(c1)
744 >>> c2
745 {'section1': {'option1': 'False', 'subsection': {'more_options': 'False'}}}
746 """
747 for key, val in indict.items():
748 if (key in self and isinstance(self[key], dict) and
749 isinstance(val, dict)):
750 self[key].merge(val)
751 else:
752 self[key] = val
754 def rename(self, oldkey, newkey):
755 """
756 Change a keyname to another, without changing position in sequence.
758 Implemented so that transformations can be made on keys,
759 as well as on values. (used by encode and decode)
761 Also renames comments.
762 """
763 if oldkey in self.scalars:
764 the_list = self.scalars
765 elif oldkey in self.sections:
766 the_list = self.sections
767 else:
768 raise KeyError, 'Key "%s" not found.' % oldkey
769 pos = the_list.index(oldkey)
770 #
771 val = self[oldkey]
772 dict.__delitem__(self, oldkey)
773 dict.__setitem__(self, newkey, val)
774 the_list.remove(oldkey)
775 the_list.insert(pos, newkey)
776 comm = self.comments[oldkey]
777 inline_comment = self.inline_comments[oldkey]
778 del self.comments[oldkey]
779 del self.inline_comments[oldkey]
780 self.comments[newkey] = comm
781 self.inline_comments[newkey] = inline_comment
783 def walk(self, function, raise_errors=True,
784 call_on_sections=False, **keywargs):
785 """
786 Walk every member and call a function on the keyword and value.
788 Return a dictionary of the return values
790 If the function raises an exception, raise the errror
791 unless ``raise_errors=False``, in which case set the return value to
792 ``False``.
794 Any unrecognised keyword arguments you pass to walk, will be pased on
795 to the function you pass in.
797 Note: if ``call_on_sections`` is ``True`` then - on encountering a
798 subsection, *first* the function is called for the *whole* subsection,
799 and then recurses into its members. This means your function must be
800 able to handle strings, dictionaries and lists. This allows you
801 to change the key of subsections as well as for ordinary members. The
802 return value when called on the whole subsection has to be discarded.
804 See the encode and decode methods for examples, including functions.
806 .. caution::
808 You can use ``walk`` to transform the names of members of a section
809 but you mustn't add or delete members.
811 >>> config = '''[XXXXsection]
812 ... XXXXkey = XXXXvalue'''.splitlines()
813 >>> cfg = ConfigObj(config)
814 >>> cfg
815 {'XXXXsection': {'XXXXkey': 'XXXXvalue'}}
816 >>> def transform(section, key):
817 ... val = section[key]
818 ... newkey = key.replace('XXXX', 'CLIENT1')
819 ... section.rename(key, newkey)
820 ... if isinstance(val, (tuple, list, dict)):
821 ... pass
822 ... else:
823 ... val = val.replace('XXXX', 'CLIENT1')
824 ... section[newkey] = val
825 >>> cfg.walk(transform, call_on_sections=True)
826 {'CLIENT1section': {'CLIENT1key': None}}
827 >>> cfg
828 {'CLIENT1section': {'CLIENT1key': 'CLIENT1value'}}
829 """
830 out = {}
831 # scalars first
832 for i in range(len(self.scalars)):
833 entry = self.scalars[i]
834 try:
835 val = function(self, entry, **keywargs)
836 # bound again in case name has changed
837 entry = self.scalars[i]
838 out[entry] = val
839 except Exception:
840 if raise_errors:
841 raise
842 else:
843 entry = self.scalars[i]
844 out[entry] = False
845 # then sections
846 for i in range(len(self.sections)):
847 entry = self.sections[i]
848 if call_on_sections:
849 try:
850 function(self, entry, **keywargs)
851 except Exception:
852 if raise_errors:
853 raise
854 else:
855 entry = self.sections[i]
856 out[entry] = False
857 # bound again in case name has changed
858 entry = self.sections[i]
859 # previous result is discarded
860 out[entry] = self[entry].walk(
861 function,
862 raise_errors=raise_errors,
863 call_on_sections=call_on_sections,
864 **keywargs)
865 return out
867 def decode(self, encoding):
868 """
869 Decode all strings and values to unicode, using the specified encoding.
871 Works with subsections and list values.
873 Uses the ``walk`` method.
875 Testing ``encode`` and ``decode``.
876 >>> m = ConfigObj(a)
877 >>> m.decode('ascii')
878 >>> def testuni(val):
879 ... for entry in val:
880 ... if not isinstance(entry, unicode):
881 ... print >> sys.stderr, type(entry)
882 ... raise AssertionError, 'decode failed.'
883 ... if isinstance(val[entry], dict):
884 ... testuni(val[entry])
885 ... elif not isinstance(val[entry], unicode):
886 ... raise AssertionError, 'decode failed.'
887 >>> testuni(m)
888 >>> m.encode('ascii')
889 >>> a == m
890 1
891 """
892 warn('use of ``decode`` is deprecated.', DeprecationWarning)
893 def decode(section, key, encoding=encoding, warn=True):
894 """ """
895 val = section[key]
896 if isinstance(val, (list, tuple)):
897 newval = []
898 for entry in val:
899 newval.append(entry.decode(encoding))
900 elif isinstance(val, dict):
901 newval = val
902 else:
903 newval = val.decode(encoding)
904 newkey = key.decode(encoding)
905 section.rename(key, newkey)
906 section[newkey] = newval
907 # using ``call_on_sections`` allows us to modify section names
908 self.walk(decode, call_on_sections=True)
910 def encode(self, encoding):
911 """
912 Encode all strings and values from unicode,
913 using the specified encoding.
915 Works with subsections and list values.
916 Uses the ``walk`` method.
917 """
918 warn('use of ``encode`` is deprecated.', DeprecationWarning)
919 def encode(section, key, encoding=encoding):
920 """ """
921 val = section[key]
922 if isinstance(val, (list, tuple)):
923 newval = []
924 for entry in val:
925 newval.append(entry.encode(encoding))
926 elif isinstance(val, dict):
927 newval = val
928 else:
929 newval = val.encode(encoding)
930 newkey = key.encode(encoding)
931 section.rename(key, newkey)
932 section[newkey] = newval
933 self.walk(encode, call_on_sections=True)
935 def istrue(self, key):
936 """A deprecated version of ``as_bool``."""
937 warn('use of ``istrue`` is deprecated. Use ``as_bool`` method '
938 'instead.', DeprecationWarning)
939 return self.as_bool(key)
941 def as_bool(self, key):
942 """
943 Accepts a key as input. The corresponding value must be a string or
944 the objects (``True`` or 1) or (``False`` or 0). We allow 0 and 1 to
945 retain compatibility with Python 2.2.
947 If the string is one of ``True``, ``On``, ``Yes``, or ``1`` it returns
948 ``True``.
950 If the string is one of ``False``, ``Off``, ``No``, or ``0`` it returns
951 ``False``.
953 ``as_bool`` is not case sensitive.
955 Any other input will raise a ``ValueError``.
957 >>> a = ConfigObj()
958 >>> a['a'] = 'fish'
959 >>> a.as_bool('a')
960 Traceback (most recent call last):
961 ValueError: Value "fish" is neither True nor False
962 >>> a['b'] = 'True'
963 >>> a.as_bool('b')
964 1
965 >>> a['b'] = 'off'
966 >>> a.as_bool('b')
967 0
968 """
969 val = self[key]
970 if val == True:
971 return True
972 elif val == False:
973 return False
974 else:
975 try:
976 if not isinstance(val, StringTypes):
977 raise KeyError
978 else:
979 return self.main._bools[val.lower()]
980 except KeyError:
981 raise ValueError('Value "%s" is neither True nor False' % val)
983 def as_int(self, key):
984 """
985 A convenience method which coerces the specified value to an integer.
987 If the value is an invalid literal for ``int``, a ``ValueError`` will
988 be raised.
990 >>> a = ConfigObj()
991 >>> a['a'] = 'fish'
992 >>> a.as_int('a')
993 Traceback (most recent call last):
994 ValueError: invalid literal for int(): fish
995 >>> a['b'] = '1'
996 >>> a.as_int('b')
997 1
998 >>> a['b'] = '3.2'
999 >>> a.as_int('b')
1000 Traceback (most recent call last):
1001 ValueError: invalid literal for int(): 3.2
1002 """
1003 return int(self[key])
1005 def as_float(self, key):
1006 """
1007 A convenience method which coerces the specified value to a float.
1009 If the value is an invalid literal for ``float``, a ``ValueError`` will
1010 be raised.
1012 >>> a = ConfigObj()
1013 >>> a['a'] = 'fish'
1014 >>> a.as_float('a')
1015 Traceback (most recent call last):
1016 ValueError: invalid literal for float(): fish
1017 >>> a['b'] = '1'
1018 >>> a.as_float('b')
1019 1.0
1020 >>> a['b'] = '3.2'
1021 >>> a.as_float('b')
1022 3.2000000000000002
1023 """
1024 return float(self[key])
1027 class ConfigObj(Section):
1028 """An object to read, create, and write config files."""
1030 _keyword = re.compile(r'''^ # line start
1031 (\s*) # indentation
1032 ( # keyword
1033 (?:".*?")| # double quotes
1034 (?:'.*?')| # single quotes
1035 (?:[^'"=].*?) # no quotes
1036 )
1037 \s*=\s* # divider
1038 (.*) # value (including list values and comments)
1039 $ # line end
1040 ''',
1041 re.VERBOSE)
1043 _sectionmarker = re.compile(r'''^
1044 (\s*) # 1: indentation
1045 ((?:\[\s*)+) # 2: section marker open
1046 ( # 3: section name open
1047 (?:"\s*\S.*?\s*")| # at least one non-space with double quotes
1048 (?:'\s*\S.*?\s*')| # at least one non-space with single quotes
1049 (?:[^'"\s].*?) # at least one non-space unquoted
1050 ) # section name close
1051 ((?:\s*\])+) # 4: section marker close
1052 \s*(\#.*)? # 5: optional comment
1053 $''',
1054 re.VERBOSE)
1056 # this regexp pulls list values out as a single string
1057 # or single values and comments
1058 # FIXME: this regex adds a '' to the end of comma terminated lists
1059 # workaround in ``_handle_value``
1060 _valueexp = re.compile(r'''^
1061 (?:
1062 (?:
1063 (
1064 (?:
1065 (?:
1066 (?:".*?")| # double quotes
1067 (?:'.*?')| # single quotes
1068 (?:[^'",\#][^,\#]*?) # unquoted
1069 )
1070 \s*,\s* # comma
1071 )* # match all list items ending in a comma (if any)
1072 )
1073 (
1074 (?:".*?")| # double quotes
1075 (?:'.*?')| # single quotes
1076 (?:[^'",\#\s][^,]*?)| # unquoted
1077 (?:(?<!,)) # Empty value
1078 )? # last item in a list - or string value
1079 )|
1080 (,) # alternatively a single comma - empty list
1081 )
1082 \s*(\#.*)? # optional comment
1083 $''',
1084 re.VERBOSE)
1086 # use findall to get the members of a list value
1087 _listvalueexp = re.compile(r'''
1088 (
1089 (?:".*?")| # double quotes
1090 (?:'.*?')| # single quotes
1091 (?:[^'",\#].*?) # unquoted
1092 )
1093 \s*,\s* # comma
1094 ''',
1095 re.VERBOSE)
1097 # this regexp is used for the value
1098 # when lists are switched off
1099 _nolistvalue = re.compile(r'''^
1100 (
1101 (?:".*?")| # double quotes
1102 (?:'.*?')| # single quotes
1103 (?:[^'"\#].*?)| # unquoted
1104 (?:) # Empty value
1105 )
1106 \s*(\#.*)? # optional comment
1107 $''',
1108 re.VERBOSE)
1110 # regexes for finding triple quoted values on one line
1111 _single_line_single = re.compile(r"^'''(.*?)'''\s*(#.*)?$")
1112 _single_line_double = re.compile(r'^"""(.*?)"""\s*(#.*)?$')
1113 _multi_line_single = re.compile(r"^(.*?)'''\s*(#.*)?$")
1114 _multi_line_double = re.compile(r'^(.*?)"""\s*(#.*)?$')
1116 _triple_quote = {
1117 "'''": (_single_line_single, _multi_line_single),
1118 '"""': (_single_line_double, _multi_line_double),
1119 }
1121 # Used by the ``istrue`` Section method
1122 _bools = {
1123 'yes': True, 'no': False,
1124 'on': True, 'off': False,
1125 '1': True, '0': False,
1126 'true': True, 'false': False,
1127 }
1129 def __init__(self, infile=None, options=None, **kwargs):
1130 """
1131 Parse or create a config file object.
1133 ``ConfigObj(infile=None, options=None, **kwargs)``
1134 """
1135 if infile is None:
1136 infile = []
1137 if options is None:
1138 options = {}
1139 else:
1140 options = dict(options)
1141 # keyword arguments take precedence over an options dictionary
1142 options.update(kwargs)
1143 # init the superclass
1144 Section.__init__(self, self, 0, self)
1145 #
1146 defaults = OPTION_DEFAULTS.copy()
1147 for entry in options.keys():
1148 if entry not in defaults.keys():
1149 raise TypeError, 'Unrecognised option "%s".' % entry
1150 # TODO: check the values too.
1151 #
1152 # Add any explicit options to the defaults
1153 defaults.update(options)
1154 #
1155 # initialise a few variables
1156 self.filename = None
1157 self._errors = []
1158 self.raise_errors = defaults['raise_errors']
1159 self.interpolation = defaults['interpolation']
1160 self.list_values = defaults['list_values']
1161 self.create_empty = defaults['create_empty']
1162 self.file_error = defaults['file_error']
1163 self.stringify = defaults['stringify']
1164 self.indent_type = defaults['indent_type']
1165 self.encoding = defaults['encoding']
1166 self.default_encoding = defaults['default_encoding']
1167 self.BOM = False
1168 self.newlines = None
1169 self.write_empty_values = defaults['write_empty_values']
1170 self.unrepr = defaults['unrepr']
1171 #
1172 self.initial_comment = []
1173 self.final_comment = []
1174 #
1175 self._terminated = False
1176 #
1177 if isinstance(infile, StringTypes):
1178 self.filename = infile
1179 if os.path.isfile(infile):
1180 infile = open(infile).read() or []
1181 elif self.file_error:
1182 # raise an error if the file doesn't exist
1183 raise IOError, 'Config file not found: "%s".' % self.filename
1184 else:
1185 # file doesn't already exist
1186 if self.create_empty:
1187 # this is a good test that the filename specified
1188 # isn't impossible - like on a non existent device
1189 h = open(infile, 'w')
1190 h.write('')
1191 h.close()
1192 infile = []
1193 elif isinstance(infile, (list, tuple)):
1194 infile = list(infile)
1195 elif isinstance(infile, dict):
1196 # initialise self
1197 # the Section class handles creating subsections
1198 if isinstance(infile, ConfigObj):
1199 # get a copy of our ConfigObj
1200 infile = infile.dict()
1201 for entry in infile:
1202 self[entry] = infile[entry]
1203 del self._errors
1204 if defaults['configspec'] is not None:
1205 self._handle_configspec(defaults['configspec'])
1206 else:
1207 self.configspec = None
1208 return
1209 elif hasattr(infile, 'read'):
1210 # This supports file like objects
1211 infile = infile.read() or []
1212 # needs splitting into lines - but needs doing *after* decoding
1213 # in case it's not an 8 bit encoding
1214 else:
1215 raise TypeError, ('infile must be a filename,'
1216 ' file like object, or list of lines.')
1217 #
1218 if infile:
1219 # don't do it for the empty ConfigObj
1220 infile = self._handle_bom(infile)
1221 # infile is now *always* a list
1222 #
1223 # Set the newlines attribute (first line ending it finds)
1224 # and strip trailing '\n' or '\r' from lines
1225 for line in infile:
1226 if (not line) or (line[-1] not in ('\r', '\n', '\r\n')):
1227 continue
1228 for end in ('\r\n', '\n', '\r'):
1229 if line.endswith(end):
1230 self.newlines = end
1231 break
1232 break
1233 if infile[-1] and infile[-1] in ('\r', '\n', '\r\n'):
1234 self._terminated = True
1235 infile = [line.rstrip('\r\n') for line in infile]
1236 #
1237 self._parse(infile)
1238 # if we had any errors, now is the time to raise them
1239 if self._errors:
1240 info = "at line %s." % self._errors[0].line_number
1241 if len(self._errors) > 1:
1242 msg = ("Parsing failed with several errors.\nFirst error %s" %
1243 info)
1244 error = ConfigObjError(msg)
1245 else:
1246 error = self._errors[0]
1247 # set the errors attribute; it's a list of tuples:
1248 # (error_type, message, line_number)
1249 error.errors = self._errors
1250 # set the config attribute
1251 error.config = self
1252 raise error
1253 # delete private attributes
1254 del self._errors
1255 #
1256 if defaults['configspec'] is None:
1257 self.configspec = None
1258 else:
1259 self._handle_configspec(defaults['configspec'])
1261 def __repr__(self):
1262 return 'ConfigObj({%s})' % ', '.join(
1263 [('%s: %s' % (repr(key), repr(self[key]))) for key in
1264 (self.scalars + self.sections)])
1266 def _handle_bom(self, infile):
1267 """
1268 Handle any BOM, and decode if necessary.
1270 If an encoding is specified, that *must* be used - but the BOM should
1271 still be removed (and the BOM attribute set).
1273 (If the encoding is wrongly specified, then a BOM for an alternative
1274 encoding won't be discovered or removed.)
1276 If an encoding is not specified, UTF8 or UTF16 BOM will be detected and
1277 removed. The BOM attribute will be set. UTF16 will be decoded to
1278 unicode.
1280 NOTE: This method must not be called with an empty ``infile``.
1282 Specifying the *wrong* encoding is likely to cause a
1283 ``UnicodeDecodeError``.
1285 ``infile`` must always be returned as a list of lines, but may be
1286 passed in as a single string.
1287 """
1288 if ((self.encoding is not None) and
1289 (self.encoding.lower() not in BOM_LIST)):
1290 # No need to check for a BOM
1291 # the encoding specified doesn't have one
1292 # just decode
1293 return self._decode(infile, self.encoding)
1294 #
1295 if isinstance(infile, (list, tuple)):
1296 line = infile[0]
1297 else:
1298 line = infile
1299 if self.encoding is not None:
1300 # encoding explicitly supplied
1301 # And it could have an associated BOM
1302 # TODO: if encoding is just UTF16 - we ought to check for both
1303 # TODO: big endian and little endian versions.
1304 enc = BOM_LIST[self.encoding.lower()]
1305 if enc == 'utf_16':
1306 # For UTF16 we try big endian and little endian
1307 for BOM, (encoding, final_encoding) in BOMS.items():
1308 if not final_encoding:
1309 # skip UTF8
1310 continue
1311 if infile.startswith(BOM):
1312 ### BOM discovered
1313 ##self.BOM = True
1314 # Don't need to remove BOM
1315 return self._decode(infile, encoding)
1316 #
1317 # If we get this far, will *probably* raise a DecodeError
1318 # As it doesn't appear to start with a BOM
1319 return self._decode(infile, self.encoding)
1320 #
1321 # Must be UTF8
1322 BOM = BOM_SET[enc]
1323 if not line.startswith(BOM):
1324 return self._decode(infile, self.encoding)
1325 #
1326 newline = line[len(BOM):]
1327 #
1328 # BOM removed
1329 if isinstance(infile, (list, tuple)):
1330 infile[0] = newline
1331 else:
1332 infile = newline
1333 self.BOM = True
1334 return self._decode(infile, self.encoding)
1335 #
1336 # No encoding specified - so we need to check for UTF8/UTF16
1337 for BOM, (encoding, final_encoding) in BOMS.items():
1338 if not line.startswith(BOM):
1339 continue
1340 else:
1341 # BOM discovered
1342 self.encoding = final_encoding
1343 if not final_encoding:
1344 self.BOM = True
1345 # UTF8
1346 # remove BOM
1347 newline = line[len(BOM):]
1348 if isinstance(infile, (list, tuple)):
1349 infile[0] = newline
1350 else:
1351 infile = newline
1352 # UTF8 - don't decode
1353 if isinstance(infile, StringTypes):
1354 return infile.splitlines(True)
1355 else:
1356 return infile
1357 # UTF16 - have to decode
1358 return self._decode(infile, encoding)
1359 #
1360 # No BOM discovered and no encoding specified, just return
1361 if isinstance(infile, StringTypes):
1362 # infile read from a file will be a single string
1363 return infile.splitlines(True)
1364 else:
1365 return infile
1367 def _a_to_u(self, aString):
1368 """Decode ASCII strings to unicode if a self.encoding is specified."""
1369 if self.encoding:
1370 return aString.decode('ascii')
1371 else:
1372 return aString
1374 def _decode(self, infile, encoding):
1375 """
1376 Decode infile to unicode. Using the specified encoding.
1378 if is a string, it also needs converting to a list.
1379 """
1380 if isinstance(infile, StringTypes):
1381 # can't be unicode
1382 # NOTE: Could raise a ``UnicodeDecodeError``
1383 return infile.decode(encoding).splitlines(True)
1384 for i, line in enumerate(infile):
1385 if not isinstance(line, unicode):
1386 # NOTE: The isinstance test here handles mixed lists of unicode/string
1387 # NOTE: But the decode will break on any non-string values
1388 # NOTE: Or could raise a ``UnicodeDecodeError``
1389 infile[i] = line.decode(encoding)
1390 return infile
1392 def _decode_element(self, line):
1393 """Decode element to unicode if necessary."""
1394 if not self.encoding:
1395 return line
1396 if isinstance(line, str) and self.default_encoding:
1397 return line.decode(self.default_encoding)
1398 return line
1400 def _str(self, value):
1401 """
1402 Used by ``stringify`` within validate, to turn non-string values
1403 into strings.
1404 """
1405 if not isinstance(value, StringTypes):
1406 return str(value)
1407 else:
1408 return value
1410 def _parse(self, infile):
1411 """Actually parse the config file."""
1412 temp_list_values = self.list_values
1413 if self.unrepr:
1414 self.list_values = False
1415 comment_list = []
1416 done_start = False
1417 this_section = self
1418 maxline = len(infile) - 1
1419 cur_index = -1
1420 reset_comment = False
1421 while cur_index < maxline:
1422 if reset_comment:
1423 comment_list = []
1424 cur_index += 1
1425 line = infile[cur_index]
1426 sline = line.strip()
1427 # do we have anything on the line ?
1428 if not sline or sline.startswith('#') or sline.startswith(';'):
1429 reset_comment = False
1430 comment_list.append(line)
1431 continue
1432 if not done_start:
1433 # preserve initial comment
1434 self.initial_comment = comment_list
1435 comment_list = []
1436 done_start = True
1437 reset_comment = True
1438 # first we check if it's a section marker
1439 mat = self._sectionmarker.match(line)
1440 if mat is not None:
1441 # is a section line
1442 (indent, sect_open, sect_name, sect_close, comment) = (
1443 mat.groups())
1444 if indent and (self.indent_type is None):
1445 self.indent_type = indent
1446 cur_depth = sect_open.count('[')
1447 if cur_depth != sect_close.count(']'):
1448 self._handle_error(
1449 "Cannot compute the section depth at line %s.",
1450 NestingError, infile, cur_index)
1451 continue
1452 #
1453 if cur_depth < this_section.depth:
1454 # the new section is dropping back to a previous level
1455 try:
1456 parent = self._match_depth(
1457 this_section,
1458 cur_depth).parent
1459 except SyntaxError:
1460 self._handle_error(
1461 "Cannot compute nesting level at line %s.",
1462 NestingError, infile, cur_index)
1463 continue
1464 elif cur_depth == this_section.depth:
1465 # the new section is a sibling of the current section
1466 parent = this_section.parent
1467 elif cur_depth == this_section.depth + 1:
1468 # the new section is a child the current section
1469 parent = this_section
1470 else:
1471 self._handle_error(
1472 "Section too nested at line %s.",
1473 NestingError, infile, cur_index)
1474 #
1475 sect_name = self._unquote(sect_name)
1476 if parent.has_key(sect_name):
1477 self._handle_error(
1478 'Duplicate section name at line %s.',
1479 DuplicateError, infile, cur_index)
1480 continue
1481 # create the new section
1482 this_section = Section(
1483 parent,
1484 cur_depth,
1485 self,
1486 name=sect_name)
1487 parent[sect_name] = this_section
1488 parent.inline_comments[sect_name] = comment
1489 parent.comments[sect_name] = comment_list
1490 continue
1491 #
1492 # it's not a section marker,
1493 # so it should be a valid ``key = value`` line
1494 mat = self._keyword.match(line)
1495 if mat is None:
1496 # it neither matched as a keyword
1497 # or a section marker
1498 self._handle_error(
1499 'Invalid line at line "%s".',
1500 ParseError, infile, cur_index)
1501 else:
1502 # is a keyword value
1503 # value will include any inline comment
1504 (indent, key, value) = mat.groups()
1505 if indent and (self.indent_type is None):
1506 self.indent_type = indent
1507 # check for a multiline value
1508 if value[:3] in ['"""', "'''"]:
1509 try:
1510 (value, comment, cur_index) = self._multiline(
1511 value, infile, cur_index, maxline)
1512 except SyntaxError:
1513 self._handle_error(
1514 'Parse error in value at line %s.',
1515 ParseError, infile, cur_index)
1516 continue
1517 else:
1518 if self.unrepr:
1519 comment = ''
1520 try:
1521 value = unrepr(value)
1522 except Exception, e:
1523 if type(e) == UnknownType:
1524 msg = 'Unknown name or type in value at line %s.'
1525 else:
1526 msg = 'Parse error in value at line %s.'
1527 self._handle_error(msg, UnreprError, infile,
1528 cur_index)
1529 continue
1530 else:
1531 if self.unrepr:
1532 comment = ''
1533 try:
1534 value = unrepr(value)
1535 except Exception, e:
1536 if isinstance(e, UnknownType):
1537 msg = 'Unknown name or type in value at line %s.'
1538 else:
1539 msg = 'Parse error in value at line %s.'
1540 self._handle_error(msg, UnreprError, infile,
1541 cur_index)
1542 continue
1543 else:
1544 # extract comment and lists
1545 try:
1546 (value, comment) = self._handle_value(value)
1547 except SyntaxError:
1548 self._handle_error(
1549 'Parse error in value at line %s.',
1550 ParseError, infile, cur_index)
1551 continue
1552 #
1553 key = self._unquote(key)
1554 if this_section.has_key(key):
1555 self._handle_error(
1556 'Duplicate keyword name at line %s.',
1557 DuplicateError, infile, cur_index)
1558 continue
1559 # add the key.
1560 # we set unrepr because if we have got this far we will never
1561 # be creating a new section
1562 this_section.__setitem__(key, value, unrepr=True)
1563 this_section.inline_comments[key] = comment
1564 this_section.comments[key] = comment_list
1565 continue
1566 #
1567 if self.indent_type is None:
1568 # no indentation used, set the type accordingly
1569 self.indent_type = ''
1570 #
1571 if self._terminated:
1572 comment_list.append('')
1573 # preserve the final comment
1574 if not self and not self.initial_comment:
1575 self.initial_comment = comment_list
1576 elif not reset_comment:
1577 self.final_comment = comment_list
1578 self.list_values = temp_list_values
1580 def _match_depth(self, sect, depth):
1581 """
1582 Given a section and a depth level, walk back through the sections
1583 parents to see if the depth level matches a previous section.
1585 Return a reference to the right section,
1586 or raise a SyntaxError.
1587 """
1588 while depth < sect.depth:
1589 if sect is sect.parent:
1590 # we've reached the top level already
1591 raise SyntaxError
1592 sect = sect.parent
1593 if sect.depth == depth:
1594 return sect
1595 # shouldn't get here
1596 raise SyntaxError
1598 def _handle_error(self, text, ErrorClass, infile, cur_index):
1599 """
1600 Handle an error according to the error settings.
1602 Either raise the error or store it.
1603 The error will have occurred at ``cur_index``
1604 """
1605 line = infile[cur_index]
1606 cur_index += 1
1607 message = text % cur_index
1608 error = ErrorClass(message, cur_index, line)
1609 if self.raise_errors:
1610 # raise the error - parsing stops here
1611 raise error
1612 # store the error
1613 # reraise when parsing has finished
1614 self._errors.append(error)
1616 def _unquote(self, value):
1617 """Return an unquoted version of a value"""
1618 if (value[0] == value[-1]) and (value[0] in ('"', "'")):
1619 value = value[1:-1]
1620 return value
1622 def _quote(self, value, multiline=True):
1623 """
1624 Return a safely quoted version of a value.
1626 Raise a ConfigObjError if the value cannot be safely quoted.
1627 If multiline is ``True`` (default) then use triple quotes
1628 if necessary.
1630 Don't quote values that don't need it.
1631 Recursively quote members of a list and return a comma joined list.
1632 Multiline is ``False`` for lists.
1633 Obey list syntax for empty and single member lists.
1635 If ``list_values=False`` then the value is only quoted if it contains
1636 a ``\n`` (is multiline).
1638 If ``write_empty_values`` is set, and the value is an empty string, it
1639 won't be quoted.
1640 """
1641 if multiline and self.write_empty_values and value == '':
1642 # Only if multiline is set, so that it is used for values not
1643 # keys, and not values that are part of a list
1644 return ''
1645 if multiline and isinstance(value, (list, tuple)):
1646 if not value:
1647 return ','
1648 elif len(value) == 1:
1649 return self._quote(value[0], multiline=False) + ','
1650 return ', '.join([self._quote(val, multiline=False)
1651 for val in value])
1652 if not isinstance(value, StringTypes):
1653 if self.stringify:
1654 value = str(value)
1655 else:
1656 raise TypeError, 'Value "%s" is not a string.' % value
1657 squot = "'%s'"
1658 dquot = '"%s"'
1659 noquot = "%s"
1660 wspace_plus = ' \r\t\n\v\t\'"'
1661 tsquot = '"""%s"""'
1662 tdquot = "'''%s'''"
1663 if not value:
1664 return '""'
1665 if (not self.list_values and '\n' not in value) or not (multiline and
1666 ((("'" in value) and ('"' in value)) or ('\n' in value))):
1667 if not self.list_values:
1668 # we don't quote if ``list_values=False``
1669 quot = noquot
1670 # for normal values either single or double quotes will do
1671 elif '\n' in value:
1672 # will only happen if multiline is off - e.g. '\n' in key
1673 raise ConfigObjError, ('Value "%s" cannot be safely quoted.' %
1674 value)
1675 elif ((value[0] not in wspace_plus) and
1676 (value[-1] not in wspace_plus) and
1677 (',' not in value)):
1678 quot = noquot
1679 else:
1680 if ("'" in value) and ('"' in value):
1681 raise ConfigObjError, (
1682 'Value "%s" cannot be safely quoted.' % value)
1683 elif '"' in value:
1684 quot = squot
1685 else:
1686 quot = dquot
1687 else:
1688 # if value has '\n' or "'" *and* '"', it will need triple quotes
1689 if (value.find('"""') != -1) and (value.find("'''") != -1):
1690 raise ConfigObjError, (
1691 'Value "%s" cannot be safely quoted.' % value)
1692 if value.find('"""') == -1:
1693 quot = tdquot
1694 else:
1695 quot = tsquot
1696 return quot % value
1698 def _handle_value(self, value):
1699 """
1700 Given a value string, unquote, remove comment,
1701 handle lists. (including empty and single member lists)
1702 """
1703 # do we look for lists in values ?
1704 if not self.list_values:
1705 mat = self._nolistvalue.match(value)
1706 if mat is None:
1707 raise SyntaxError
1708 # NOTE: we don't unquote here
1709 return mat.groups()
1710 #
1711 mat = self._valueexp.match(value)
1712 if mat is None:
1713 # the value is badly constructed, probably badly quoted,
1714 # or an invalid list
1715 raise SyntaxError
1716 (list_values, single, empty_list, comment) = mat.groups()
1717 if (list_values == '') and (single is None):
1718 # change this if you want to accept empty values
1719 raise SyntaxError
1720 # NOTE: note there is no error handling from here if the regex
1721 # is wrong: then incorrect values will slip through
1722 if empty_list is not None:
1723 # the single comma - meaning an empty list
1724 return ([], comment)
1725 if single is not None:
1726 # handle empty values
1727 if list_values and not single:
1728 # FIXME: the '' is a workaround because our regex now matches
1729 # '' at the end of a list if it has a trailing comma
1730 single = None
1731 else:
1732 single = single or '""'
1733 single = self._unquote(single)
1734 if list_values == '':
1735 # not a list value
1736 return (single, comment)
1737 the_list = self._listvalueexp.findall(list_values)
1738 the_list = [self._unquote(val) for val in the_list]
1739 if single is not None:
1740 the_list += [single]
1741 return (the_list, comment)
1743 def _multiline(self, value, infile, cur_index, maxline):
1744 """Extract the value, where we are in a multiline situation."""
1745 quot = value[:3]
1746 newvalue = value[3:]
1747 single_line = self._triple_quote[quot][0]
1748 multi_line = self._triple_quote[quot][1]
1749 mat = single_line.match(value)
1750 if mat is not None:
1751 retval = list(mat.groups())
1752 retval.append(cur_index)
1753 return retval
1754 elif newvalue.find(quot) != -1:
1755 # somehow the triple quote is missing
1756 raise SyntaxError
1757 #
1758 while cur_index < maxline:
1759 cur_index += 1
1760 newvalue += '\n'
1761 line = infile[cur_index]
1762 if line.find(quot) == -1:
1763 newvalue += line
1764 else:
1765 # end of multiline, process it
1766 break
1767 else:
1768 # we've got to the end of the config, oops...
1769 raise SyntaxError
1770 mat = multi_line.match(line)
1771 if mat is None:
1772 # a badly formed line
1773 raise SyntaxError
1774 (value, comment) = mat.groups()
1775 return (newvalue + value, comment, cur_index)
1777 def _handle_configspec(self, configspec):
1778 """Parse the configspec."""
1779 # FIXME: Should we check that the configspec was created with the
1780 # correct settings ? (i.e. ``list_values=False``)
1781 if not isinstance(configspec, ConfigObj):
1782 try:
1783 configspec = ConfigObj(
1784 configspec,
1785 raise_errors=True,
1786 file_error=True,
1787 list_values=False)
1788 except ConfigObjError, e:
1789 # FIXME: Should these errors have a reference
1790 # to the already parsed ConfigObj ?
1791 raise ConfigspecError('Parsing configspec failed: %s' % e)
1792 except IOError, e:
1793 raise IOError('Reading configspec failed: %s' % e)
1794 self._set_configspec_value(configspec, self)
1796 def _set_configspec_value(self, configspec, section):
1797 """Used to recursively set configspec values."""
1798 if '__many__' in configspec.sections:
1799 section.configspec['__many__'] = configspec['__many__']
1800 if len(configspec.sections) > 1:
1801 # FIXME: can we supply any useful information here ?
1802 raise RepeatSectionError
1803 if hasattr(configspec, 'initial_comment'):
1804 section._configspec_initial_comment = configspec.initial_comment
1805 section._configspec_final_comment = configspec.final_comment
1806 section._configspec_encoding = configspec.encoding
1807 section._configspec_BOM = configspec.BOM
1808 section._configspec_newlines = configspec.newlines
1809 section._configspec_indent_type = configspec.indent_type
1810 for entry in configspec.scalars:
1811 section._configspec_comments[entry] = configspec.comments[entry]
1812 section._configspec_inline_comments[entry] = (
1813 configspec.inline_comments[entry])
1814 section.configspec[entry] = configspec[entry]
1815 section._order.append(entry)
1816 for entry in configspec.sections:
1817 if entry == '__many__':
1818 continue
1819 section._cs_section_comments[entry] = configspec.comments[entry]
1820 section._cs_section_inline_comments[entry] = (
1821 configspec.inline_comments[entry])
1822 if not section.has_key(entry):
1823 section[entry] = {}
1824 self._set_configspec_value(configspec[entry], section[entry])
1826 def _handle_repeat(self, section, configspec):
1827 """Dynamically assign configspec for repeated section."""
1828 try:
1829 section_keys = configspec.sections
1830 scalar_keys = configspec.scalars
1831 except AttributeError:
1832 section_keys = [entry for entry in configspec
1833 if isinstance(configspec[entry], dict)]
1834 scalar_keys = [entry for entry in configspec
1835 if not isinstance(configspec[entry], dict)]
1836 if '__many__' in section_keys and len(section_keys) > 1:
1837 # FIXME: can we supply any useful information here ?
1838 raise RepeatSectionError
1839 scalars = {}
1840 sections = {}
1841 for entry in scalar_keys:
1842 val = configspec[entry]
1843 scalars[entry] = val
1844 for entry in section_keys:
1845 val = configspec[entry]
1846 if entry == '__many__':
1847 scalars[entry] = val
1848 continue
1849 sections[entry] = val
1850 #
1851 section.configspec = scalars
1852 for entry in sections:
1853 if not section.has_key(entry):
1854 section[entry] = {}
1855 self._handle_repeat(section[entry], sections[entry])
1857 def _write_line(self, indent_string, entry, this_entry, comment):
1858 """Write an individual line, for the write method"""
1859 # NOTE: the calls to self._quote here handles non-StringType values.
1860 if not self.unrepr:
1861 val = self._decode_element(self._quote(this_entry))
1862 else:
1863 val = repr(this_entry)
1864 return '%s%s%s%s%s' % (
1865 indent_string,
1866 self._decode_element(self._quote(entry, multiline=False)),
1867 self._a_to_u(' = '),
1868 val,
1869 self._decode_element(comment))
1871 def _write_marker(self, indent_string, depth, entry, comment):
1872 """Write a section marker line"""
1873 return '%s%s%s%s%s' % (
1874 indent_string,
1875 self._a_to_u('[' * depth),
1876 self._quote(self._decode_element(entry), multiline=False),
1877 self._a_to_u(']' * depth),
1878 self._decode_element(comment))
1880 def _handle_comment(self, comment):
1881 """Deal with a comment."""
1882 if not comment:
1883 return ''
1884 start = self.indent_type
1885 if not comment.startswith('#'):
1886 start += self._a_to_u(' # ')
1887 return (start + comment)
1889 # Public methods
1891 def write(self, outfile=None, section=None):
1892 """
1893 Write the current ConfigObj as a file
1895 tekNico: FIXME: use StringIO instead of real files
1897 >>> filename = a.filename
1898 >>> a.filename = 'test.ini'
1899 >>> a.write()
1900 >>> a.filename = filename
1901 >>> a == ConfigObj('test.ini', raise_errors=True)
1902 1
1903 """
1904 if self.indent_type is None:
1905 # this can be true if initialised from a dictionary
1906 self.indent_type = DEFAULT_INDENT_TYPE
1907 #
1908 out = []
1909 cs = self._a_to_u('#')
1910 csp = self._a_to_u('# ')
1911 if section is None:
1912 int_val = self.interpolation
1913 self.interpolation = False
1914 section = self
1915 for line in self.initial_comment:
1916 line = self._decode_element(line)
1917 stripped_line = line.strip()
1918 if stripped_line and not stripped_line.startswith(cs):
1919 line = csp + line
1920 out.append(line)
1921 #
1922 indent_string = self.indent_type * section.depth
1923 for entry in (section.scalars + section.sections):
1924 if entry in section.defaults:
1925 # don't write out default values
1926 continue
1927 for comment_line in section.comments[entry]:
1928 comment_line = self._decode_element(comment_line.lstrip())
1929 if comment_line and not comment_line.startswith(cs):
1930 comment_line = csp + comment_line
1931 out.append(indent_string + comment_line)
1932 this_entry = section[entry]
1933 comment = self._handle_comment(section.inline_comments[entry])
1934 #
1935 if isinstance(this_entry, dict):
1936 # a section
1937 out.append(self._write_marker(
1938 indent_string,
1939 this_entry.depth,
1940 entry,
1941 comment))
1942 out.extend(self.write(section=this_entry))
1943 else:
1944 out.append(self._write_line(
1945 indent_string,
1946 entry,
1947 this_entry,
1948 comment))
1949 #
1950 if section is self:
1951 for line in self.final_comment:
1952 line = self._decode_element(line)
1953 stripped_line = line.strip()
1954 if stripped_line and not stripped_line.startswith(cs):
1955 line = csp + line
1956 out.append(line)
1957 self.interpolation = int_val
1958 #
1959 if section is not self:
1960 return out
1961 #
1962 if (self.filename is None) and (outfile is None):
1963 # output a list of lines
1964 # might need to encode
1965 # NOTE: This will *screw* UTF16, each line will start with the BOM
1966 if self.encoding:
1967 out = [l.encode(self.encoding) for l in out]
1968 if (self.BOM and ((self.encoding is None) or
1969 (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
1970 # Add the UTF8 BOM
1971 if not out:
1972 out.append('')
1973 out[0] = BOM_UTF8 + out[0]
1974 return out
1975 #
1976 # Turn the list to a string, joined with correct newlines
1977 output = (self._a_to_u(self.newlines or os.linesep)
1978 ).join(out)
1979 if self.encoding:
1980 output = output.encode(self.encoding)
1981 if (self.BOM and ((self.encoding is None) or
1982 (BOM_LIST.get(self.encoding.lower()) == 'utf_8'))):
1983 # Add the UTF8 BOM
1984 output = BOM_UTF8 + output
1985 if outfile is not None:
1986 outfile.write(output)
1987 else:
1988 h = open(self.filename, 'wb')
1989 h.write(output)
1990 h.close()
1992 def validate(self, validator, preserve_errors=False, copy=False,
1993 section=None):
1994 """
1995 Test the ConfigObj against a configspec.
1997 It uses the ``validator`` object from *validate.py*.
1999 To run ``validate`` on the current ConfigObj, call: ::
2001 test = config.validate(validator)
2003 (Normally having previously passed in the configspec when the ConfigObj
2004 was created - you can dynamically assign a dictionary of checks to the
2005 ``configspec`` attribute of a section though).
2007 It returns ``True`` if everything passes, or a dictionary of
2008 pass/fails (True/False). If every member of a subsection passes, it
2009 will just have the value ``True``. (It also returns ``False`` if all
2010 members fail).
2012 In addition, it converts the values from strings to their native
2013 types if their checks pass (and ``stringify`` is set).
2015 If ``preserve_errors`` is ``True`` (``False`` is default) then instead
2016 of a marking a fail with a ``False``, it will preserve the actual
2017 exception object. This can contain info about the reason for failure.
2018 For example the ``VdtValueTooSmallError`` indeicates that the value
2019 supplied was too small. If a value (or section) is missing it will
2020 still be marked as ``False``.
2022 You must have the validate module to use ``preserve_errors=True``.
2024 You can then use the ``flatten_errors`` function to turn your nested
2025 results dictionary into a flattened list of failures - useful for
2026 displaying meaningful error messages.
2027 """
2028 if section is None:
2029 if self.configspec is None:
2030 raise ValueError, 'No configspec supplied.'
2031 if preserve_errors:
2032 if VdtMissingValue is None:
2033 raise ImportError('Missing validate module.')
2034 section = self
2035 #
2036 spec_section = section.configspec
2037 if copy and hasattr(section, '_configspec_initial_comment'):
2038 section.initial_comment = section._configspec_initial_comment
2039 section.final_comment = section._configspec_final_comment
2040 section.encoding = section._configspec_encoding
2041 section.BOM = section._configspec_BOM
2042 section.newlines = section._configspec_newlines
2043 section.indent_type = section._configspec_indent_type
2044 if '__many__' in section.configspec:
2045 many = spec_section['__many__']
2046 # dynamically assign the configspecs
2047 # for the sections below
2048 for entry in section.sections:
2049 self._handle_repeat(section[entry], many)
2050 #
2051 out = {}
2052 ret_true = True
2053 ret_false = True
2054 order = [k for k in section._order if k in spec_section]
2055 order += [k for k in spec_section if k not in order]
2056 for entry in order:
2057 if entry == '__many__':
2058 continue
2059 if (not entry in section.scalars) or (entry in section.defaults):
2060 # missing entries
2061 # or entries from defaults
2062 missing = True
2063 val = None
2064 if copy and not entry in section.scalars:
2065 # copy comments
2066 section.comments[entry] = (
2067 section._configspec_comments.get(entry, []))
2068 section.inline_comments[entry] = (
2069 section._configspec_inline_comments.get(entry, ''))
2070 #
2071 else:
2072 missing = False
2073 val = section[entry]
2074 try:
2075 check = validator.check(spec_section[entry],
2076 val,
2077 missing=missing
2078 )
2079 except validator.baseErrorClass, e:
2080 if not preserve_errors or isinstance(e, VdtMissingValue):
2081 out[entry] = False
2082 else:
2083 # preserve the error
2084 out[entry] = e
2085 ret_false = False
2086 ret_true = False
2087 else:
2088 ret_false = False
2089 out[entry] = True
2090 if self.stringify or missing:
2091 # if we are doing type conversion
2092 # or the value is a supplied default
2093 if not self.stringify:
2094 if isinstance(check, (list, tuple)):
2095 # preserve lists
2096 check = [self._str(item) for item in check]
2097 elif missing and check is None:
2098 # convert the None from a default to a ''
2099 check = ''
2100 else:
2101 check = self._str(check)
2102 if (check != val) or missing:
2103 section[entry] = check
2104 if not copy and missing and entry not in section.defaults:
2105 section.defaults.append(entry)
2106 #
2107 # Missing sections will have been created as empty ones when the
2108 # configspec was read.
2109 for entry in section.sections:
2110 # FIXME: this means DEFAULT is not copied in copy mode
2111 if section is self and entry == 'DEFAULT':
2112 continue
2113 if copy:
2114 section.comments[entry] = section._cs_section_comments[entry]
2115 section.inline_comments[entry] = (
2116 section._cs_section_inline_comments[entry])
2117 check = self.validate(validator, preserve_errors=preserve_errors,
2118 copy=copy, section=section[entry])
2119 out[entry] = check
2120 if check == False:
2121 ret_true = False
2122 elif check == True:
2123 ret_false = False
2124 else:
2125 ret_true = False
2126 ret_false = False
2127 #
2128 if ret_true:
2129 return True
2130 elif ret_false:
2131 return False
2132 else:
2133 return out
2135 class SimpleVal(object):
2136 """
2137 A simple validator.
2138 Can be used to check that all members expected are present.
2140 To use it, provide a configspec with all your members in (the value given
2141 will be ignored). Pass an instance of ``SimpleVal`` to the ``validate``
2142 method of your ``ConfigObj``. ``validate`` will return ``True`` if all
2143 members are present, or a dictionary with True/False meaning
2144 present/missing. (Whole missing sections will be replaced with ``False``)
2145 """
2147 def __init__(self):
2148 self.baseErrorClass = ConfigObjError
2150 def check(self, check, member, missing=False):
2151 """A dummy check method, always returns the value unchanged."""
2152 if missing:
2153 raise self.baseErrorClass
2154 return member
2156 # Check / processing functions for options
2157 def flatten_errors(cfg, res, levels=None, results=None):
2158 """
2159 An example function that will turn a nested dictionary of results
2160 (as returned by ``ConfigObj.validate``) into a flat list.
2162 ``cfg`` is the ConfigObj instance being checked, ``res`` is the results
2163 dictionary returned by ``validate``.
2165 (This is a recursive function, so you shouldn't use the ``levels`` or
2166 ``results`` arguments - they are used by the function.
2168 Returns a list of keys that failed. Each member of the list is a tuple :
2169 ::
2171 ([list of sections...], key, result)
2173 If ``validate`` was called with ``preserve_errors=False`` (the default)
2174 then ``result`` will always be ``False``.
2176 *list of sections* is a flattened list of sections that the key was found
2177 in.
2179 If the section was missing then key will be ``None``.
2181 If the value (or section) was missing then ``result`` will be ``False``.
2183 If ``validate`` was called with ``preserve_errors=True`` and a value
2184 was present, but failed the check, then ``result`` will be the exception
2185 object returned. You can use this as a string that describes the failure.
2187 For example *The value "3" is of the wrong type*.
2189 >>> import validate
2190 >>> vtor = validate.Validator()
2191 >>> my_ini = '''
2192 ... option1 = True
2193 ... [section1]
2194 ... option1 = True
2195 ... [section2]
2196 ... another_option = Probably
2197 ... [section3]
2198 ... another_option = True
2199 ... [[section3b]]
2200 ... value = 3
2201 ... value2 = a
2202 ... value3 = 11
2203 ... '''
2204 >>> my_cfg = '''
2205 ... option1 = boolean()
2206 ... option2 = boolean()
2207 ... option3 = boolean(default=Bad_value)
2208 ... [section1]
2209 ... option1 = boolean()
2210 ... option2 = boolean()
2211 ... option3 = boolean(default=Bad_value)
2212 ... [section2]
2213 ... another_option = boolean()
2214 ... [section3]
2215 ... another_option = boolean()
2216 ... [[section3b]]
2217 ... value = integer
2218 ... value2 = integer
2219 ... value3 = integer(0, 10)
2220 ... [[[section3b-sub]]]
2221 ... value = string
2222 ... [section4]
2223 ... another_option = boolean()
2224 ... '''
2225 >>> cs = my_cfg.split('\\n')
2226 >>> ini = my_ini.split('\\n')
2227 >>> cfg = ConfigObj(ini, configspec=cs)
2228 >>> res = cfg.validate(vtor, preserve_errors=True)
2229 >>> errors = []
2230 >>> for entry in flatten_errors(cfg, res):
2231 ... section_list, key, error = entry
2232 ... section_list.insert(0, '[root]')
2233 ... if key is not None:
2234 ... section_list.append(key)
2235 ... else:
2236 ... section_list.append('[missing]')
2237 ... section_string = ', '.join(section_list)
2238 ... errors.append((section_string, ' = ', error))
2239 >>> errors.sort()
2240 >>> for entry in errors:
2241 ... print entry[0], entry[1], (entry[2] or 0)
2242 [root], option2 = 0
2243 [root], option3 = the value "Bad_value" is of the wrong type.
2244 [root], section1, option2 = 0
2245 [root], section1, option3 = the value "Bad_value" is of the wrong type.
2246 [root], section2, another_option = the value "Probably" is of the wrong type.
2247 [root], section3, section3b, section3b-sub, [missing] = 0
2248 [root], section3, section3b, value2 = the value "a" is of the wrong type.
2249 [root], section3, section3b, value3 = the value "11" is too big.
2250 [root], section4, [missing] = 0
2251 """
2252 if levels is None:
2253 # first time called
2254 levels = []
2255 results = []
2256 if res is True:
2257 return results
2258 if res is False:
2259 results.append((levels[:], None, False))
2260 if levels:
2261 levels.pop()
2262 return results
2263 for (key, val) in res.items():
2264 if val == True:
2265 continue
2266 if isinstance(cfg.get(key), dict):
2267 # Go down one level
2268 levels.append(key)
2269 flatten_errors(cfg[key], val, levels, results)
2270 continue
2271 results.append((levels[:], key, val))
2272 #
2273 # Go up one level
2274 if levels:
2275 levels.pop()
2276 #
2277 return results
2279 """*A programming language is a medium of expression.* - Paul Graham"""
