]> arthur.barton.de Git - netdata.git/blob - python.d/python_modules/pyyaml2/constructor.py
bundle pyyaml
[netdata.git] / python.d / python_modules / pyyaml2 / constructor.py
1
2 __all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor',
3     'ConstructorError']
4
5 from error import *
6 from nodes import *
7
8 import datetime
9
10 import binascii, re, sys, types
11
12 class ConstructorError(MarkedYAMLError):
13     pass
14
15 class BaseConstructor(object):
16
17     yaml_constructors = {}
18     yaml_multi_constructors = {}
19
20     def __init__(self):
21         self.constructed_objects = {}
22         self.recursive_objects = {}
23         self.state_generators = []
24         self.deep_construct = False
25
26     def check_data(self):
27         # If there are more documents available?
28         return self.check_node()
29
30     def get_data(self):
31         # Construct and return the next document.
32         if self.check_node():
33             return self.construct_document(self.get_node())
34
35     def get_single_data(self):
36         # Ensure that the stream contains a single document and construct it.
37         node = self.get_single_node()
38         if node is not None:
39             return self.construct_document(node)
40         return None
41
42     def construct_document(self, node):
43         data = self.construct_object(node)
44         while self.state_generators:
45             state_generators = self.state_generators
46             self.state_generators = []
47             for generator in state_generators:
48                 for dummy in generator:
49                     pass
50         self.constructed_objects = {}
51         self.recursive_objects = {}
52         self.deep_construct = False
53         return data
54
55     def construct_object(self, node, deep=False):
56         if node in self.constructed_objects:
57             return self.constructed_objects[node]
58         if deep:
59             old_deep = self.deep_construct
60             self.deep_construct = True
61         if node in self.recursive_objects:
62             raise ConstructorError(None, None,
63                     "found unconstructable recursive node", node.start_mark)
64         self.recursive_objects[node] = None
65         constructor = None
66         tag_suffix = None
67         if node.tag in self.yaml_constructors:
68             constructor = self.yaml_constructors[node.tag]
69         else:
70             for tag_prefix in self.yaml_multi_constructors:
71                 if node.tag.startswith(tag_prefix):
72                     tag_suffix = node.tag[len(tag_prefix):]
73                     constructor = self.yaml_multi_constructors[tag_prefix]
74                     break
75             else:
76                 if None in self.yaml_multi_constructors:
77                     tag_suffix = node.tag
78                     constructor = self.yaml_multi_constructors[None]
79                 elif None in self.yaml_constructors:
80                     constructor = self.yaml_constructors[None]
81                 elif isinstance(node, ScalarNode):
82                     constructor = self.__class__.construct_scalar
83                 elif isinstance(node, SequenceNode):
84                     constructor = self.__class__.construct_sequence
85                 elif isinstance(node, MappingNode):
86                     constructor = self.__class__.construct_mapping
87         if tag_suffix is None:
88             data = constructor(self, node)
89         else:
90             data = constructor(self, tag_suffix, node)
91         if isinstance(data, types.GeneratorType):
92             generator = data
93             data = generator.next()
94             if self.deep_construct:
95                 for dummy in generator:
96                     pass
97             else:
98                 self.state_generators.append(generator)
99         self.constructed_objects[node] = data
100         del self.recursive_objects[node]
101         if deep:
102             self.deep_construct = old_deep
103         return data
104
105     def construct_scalar(self, node):
106         if not isinstance(node, ScalarNode):
107             raise ConstructorError(None, None,
108                     "expected a scalar node, but found %s" % node.id,
109                     node.start_mark)
110         return node.value
111
112     def construct_sequence(self, node, deep=False):
113         if not isinstance(node, SequenceNode):
114             raise ConstructorError(None, None,
115                     "expected a sequence node, but found %s" % node.id,
116                     node.start_mark)
117         return [self.construct_object(child, deep=deep)
118                 for child in node.value]
119
120     def construct_mapping(self, node, deep=False):
121         if not isinstance(node, MappingNode):
122             raise ConstructorError(None, None,
123                     "expected a mapping node, but found %s" % node.id,
124                     node.start_mark)
125         mapping = {}
126         for key_node, value_node in node.value:
127             key = self.construct_object(key_node, deep=deep)
128             try:
129                 hash(key)
130             except TypeError, exc:
131                 raise ConstructorError("while constructing a mapping", node.start_mark,
132                         "found unacceptable key (%s)" % exc, key_node.start_mark)
133             value = self.construct_object(value_node, deep=deep)
134             mapping[key] = value
135         return mapping
136
137     def construct_pairs(self, node, deep=False):
138         if not isinstance(node, MappingNode):
139             raise ConstructorError(None, None,
140                     "expected a mapping node, but found %s" % node.id,
141                     node.start_mark)
142         pairs = []
143         for key_node, value_node in node.value:
144             key = self.construct_object(key_node, deep=deep)
145             value = self.construct_object(value_node, deep=deep)
146             pairs.append((key, value))
147         return pairs
148
149     def add_constructor(cls, tag, constructor):
150         if not 'yaml_constructors' in cls.__dict__:
151             cls.yaml_constructors = cls.yaml_constructors.copy()
152         cls.yaml_constructors[tag] = constructor
153     add_constructor = classmethod(add_constructor)
154
155     def add_multi_constructor(cls, tag_prefix, multi_constructor):
156         if not 'yaml_multi_constructors' in cls.__dict__:
157             cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy()
158         cls.yaml_multi_constructors[tag_prefix] = multi_constructor
159     add_multi_constructor = classmethod(add_multi_constructor)
160
161 class SafeConstructor(BaseConstructor):
162
163     def construct_scalar(self, node):
164         if isinstance(node, MappingNode):
165             for key_node, value_node in node.value:
166                 if key_node.tag == u'tag:yaml.org,2002:value':
167                     return self.construct_scalar(value_node)
168         return BaseConstructor.construct_scalar(self, node)
169
170     def flatten_mapping(self, node):
171         merge = []
172         index = 0
173         while index < len(node.value):
174             key_node, value_node = node.value[index]
175             if key_node.tag == u'tag:yaml.org,2002:merge':
176                 del node.value[index]
177                 if isinstance(value_node, MappingNode):
178                     self.flatten_mapping(value_node)
179                     merge.extend(value_node.value)
180                 elif isinstance(value_node, SequenceNode):
181                     submerge = []
182                     for subnode in value_node.value:
183                         if not isinstance(subnode, MappingNode):
184                             raise ConstructorError("while constructing a mapping",
185                                     node.start_mark,
186                                     "expected a mapping for merging, but found %s"
187                                     % subnode.id, subnode.start_mark)
188                         self.flatten_mapping(subnode)
189                         submerge.append(subnode.value)
190                     submerge.reverse()
191                     for value in submerge:
192                         merge.extend(value)
193                 else:
194                     raise ConstructorError("while constructing a mapping", node.start_mark,
195                             "expected a mapping or list of mappings for merging, but found %s"
196                             % value_node.id, value_node.start_mark)
197             elif key_node.tag == u'tag:yaml.org,2002:value':
198                 key_node.tag = u'tag:yaml.org,2002:str'
199                 index += 1
200             else:
201                 index += 1
202         if merge:
203             node.value = merge + node.value
204
205     def construct_mapping(self, node, deep=False):
206         if isinstance(node, MappingNode):
207             self.flatten_mapping(node)
208         return BaseConstructor.construct_mapping(self, node, deep=deep)
209
210     def construct_yaml_null(self, node):
211         self.construct_scalar(node)
212         return None
213
214     bool_values = {
215         u'yes':     True,
216         u'no':      False,
217         u'true':    True,
218         u'false':   False,
219         u'on':      True,
220         u'off':     False,
221     }
222
223     def construct_yaml_bool(self, node):
224         value = self.construct_scalar(node)
225         return self.bool_values[value.lower()]
226
227     def construct_yaml_int(self, node):
228         value = str(self.construct_scalar(node))
229         value = value.replace('_', '')
230         sign = +1
231         if value[0] == '-':
232             sign = -1
233         if value[0] in '+-':
234             value = value[1:]
235         if value == '0':
236             return 0
237         elif value.startswith('0b'):
238             return sign*int(value[2:], 2)
239         elif value.startswith('0x'):
240             return sign*int(value[2:], 16)
241         elif value[0] == '0':
242             return sign*int(value, 8)
243         elif ':' in value:
244             digits = [int(part) for part in value.split(':')]
245             digits.reverse()
246             base = 1
247             value = 0
248             for digit in digits:
249                 value += digit*base
250                 base *= 60
251             return sign*value
252         else:
253             return sign*int(value)
254
255     inf_value = 1e300
256     while inf_value != inf_value*inf_value:
257         inf_value *= inf_value
258     nan_value = -inf_value/inf_value   # Trying to make a quiet NaN (like C99).
259
260     def construct_yaml_float(self, node):
261         value = str(self.construct_scalar(node))
262         value = value.replace('_', '').lower()
263         sign = +1
264         if value[0] == '-':
265             sign = -1
266         if value[0] in '+-':
267             value = value[1:]
268         if value == '.inf':
269             return sign*self.inf_value
270         elif value == '.nan':
271             return self.nan_value
272         elif ':' in value:
273             digits = [float(part) for part in value.split(':')]
274             digits.reverse()
275             base = 1
276             value = 0.0
277             for digit in digits:
278                 value += digit*base
279                 base *= 60
280             return sign*value
281         else:
282             return sign*float(value)
283
284     def construct_yaml_binary(self, node):
285         value = self.construct_scalar(node)
286         try:
287             return str(value).decode('base64')
288         except (binascii.Error, UnicodeEncodeError), exc:
289             raise ConstructorError(None, None,
290                     "failed to decode base64 data: %s" % exc, node.start_mark) 
291
292     timestamp_regexp = re.compile(
293             ur'''^(?P<year>[0-9][0-9][0-9][0-9])
294                 -(?P<month>[0-9][0-9]?)
295                 -(?P<day>[0-9][0-9]?)
296                 (?:(?:[Tt]|[ \t]+)
297                 (?P<hour>[0-9][0-9]?)
298                 :(?P<minute>[0-9][0-9])
299                 :(?P<second>[0-9][0-9])
300                 (?:\.(?P<fraction>[0-9]*))?
301                 (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
302                 (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X)
303
304     def construct_yaml_timestamp(self, node):
305         value = self.construct_scalar(node)
306         match = self.timestamp_regexp.match(node.value)
307         values = match.groupdict()
308         year = int(values['year'])
309         month = int(values['month'])
310         day = int(values['day'])
311         if not values['hour']:
312             return datetime.date(year, month, day)
313         hour = int(values['hour'])
314         minute = int(values['minute'])
315         second = int(values['second'])
316         fraction = 0
317         if values['fraction']:
318             fraction = values['fraction'][:6]
319             while len(fraction) < 6:
320                 fraction += '0'
321             fraction = int(fraction)
322         delta = None
323         if values['tz_sign']:
324             tz_hour = int(values['tz_hour'])
325             tz_minute = int(values['tz_minute'] or 0)
326             delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute)
327             if values['tz_sign'] == '-':
328                 delta = -delta
329         data = datetime.datetime(year, month, day, hour, minute, second, fraction)
330         if delta:
331             data -= delta
332         return data
333
334     def construct_yaml_omap(self, node):
335         # Note: we do not check for duplicate keys, because it's too
336         # CPU-expensive.
337         omap = []
338         yield omap
339         if not isinstance(node, SequenceNode):
340             raise ConstructorError("while constructing an ordered map", node.start_mark,
341                     "expected a sequence, but found %s" % node.id, node.start_mark)
342         for subnode in node.value:
343             if not isinstance(subnode, MappingNode):
344                 raise ConstructorError("while constructing an ordered map", node.start_mark,
345                         "expected a mapping of length 1, but found %s" % subnode.id,
346                         subnode.start_mark)
347             if len(subnode.value) != 1:
348                 raise ConstructorError("while constructing an ordered map", node.start_mark,
349                         "expected a single mapping item, but found %d items" % len(subnode.value),
350                         subnode.start_mark)
351             key_node, value_node = subnode.value[0]
352             key = self.construct_object(key_node)
353             value = self.construct_object(value_node)
354             omap.append((key, value))
355
356     def construct_yaml_pairs(self, node):
357         # Note: the same code as `construct_yaml_omap`.
358         pairs = []
359         yield pairs
360         if not isinstance(node, SequenceNode):
361             raise ConstructorError("while constructing pairs", node.start_mark,
362                     "expected a sequence, but found %s" % node.id, node.start_mark)
363         for subnode in node.value:
364             if not isinstance(subnode, MappingNode):
365                 raise ConstructorError("while constructing pairs", node.start_mark,
366                         "expected a mapping of length 1, but found %s" % subnode.id,
367                         subnode.start_mark)
368             if len(subnode.value) != 1:
369                 raise ConstructorError("while constructing pairs", node.start_mark,
370                         "expected a single mapping item, but found %d items" % len(subnode.value),
371                         subnode.start_mark)
372             key_node, value_node = subnode.value[0]
373             key = self.construct_object(key_node)
374             value = self.construct_object(value_node)
375             pairs.append((key, value))
376
377     def construct_yaml_set(self, node):
378         data = set()
379         yield data
380         value = self.construct_mapping(node)
381         data.update(value)
382
383     def construct_yaml_str(self, node):
384         value = self.construct_scalar(node)
385         try:
386             return value.encode('ascii')
387         except UnicodeEncodeError:
388             return value
389
390     def construct_yaml_seq(self, node):
391         data = []
392         yield data
393         data.extend(self.construct_sequence(node))
394
395     def construct_yaml_map(self, node):
396         data = {}
397         yield data
398         value = self.construct_mapping(node)
399         data.update(value)
400
401     def construct_yaml_object(self, node, cls):
402         data = cls.__new__(cls)
403         yield data
404         if hasattr(data, '__setstate__'):
405             state = self.construct_mapping(node, deep=True)
406             data.__setstate__(state)
407         else:
408             state = self.construct_mapping(node)
409             data.__dict__.update(state)
410
411     def construct_undefined(self, node):
412         raise ConstructorError(None, None,
413                 "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'),
414                 node.start_mark)
415
416 SafeConstructor.add_constructor(
417         u'tag:yaml.org,2002:null',
418         SafeConstructor.construct_yaml_null)
419
420 SafeConstructor.add_constructor(
421         u'tag:yaml.org,2002:bool',
422         SafeConstructor.construct_yaml_bool)
423
424 SafeConstructor.add_constructor(
425         u'tag:yaml.org,2002:int',
426         SafeConstructor.construct_yaml_int)
427
428 SafeConstructor.add_constructor(
429         u'tag:yaml.org,2002:float',
430         SafeConstructor.construct_yaml_float)
431
432 SafeConstructor.add_constructor(
433         u'tag:yaml.org,2002:binary',
434         SafeConstructor.construct_yaml_binary)
435
436 SafeConstructor.add_constructor(
437         u'tag:yaml.org,2002:timestamp',
438         SafeConstructor.construct_yaml_timestamp)
439
440 SafeConstructor.add_constructor(
441         u'tag:yaml.org,2002:omap',
442         SafeConstructor.construct_yaml_omap)
443
444 SafeConstructor.add_constructor(
445         u'tag:yaml.org,2002:pairs',
446         SafeConstructor.construct_yaml_pairs)
447
448 SafeConstructor.add_constructor(
449         u'tag:yaml.org,2002:set',
450         SafeConstructor.construct_yaml_set)
451
452 SafeConstructor.add_constructor(
453         u'tag:yaml.org,2002:str',
454         SafeConstructor.construct_yaml_str)
455
456 SafeConstructor.add_constructor(
457         u'tag:yaml.org,2002:seq',
458         SafeConstructor.construct_yaml_seq)
459
460 SafeConstructor.add_constructor(
461         u'tag:yaml.org,2002:map',
462         SafeConstructor.construct_yaml_map)
463
464 SafeConstructor.add_constructor(None,
465         SafeConstructor.construct_undefined)
466
467 class Constructor(SafeConstructor):
468
469     def construct_python_str(self, node):
470         return self.construct_scalar(node).encode('utf-8')
471
472     def construct_python_unicode(self, node):
473         return self.construct_scalar(node)
474
475     def construct_python_long(self, node):
476         return long(self.construct_yaml_int(node))
477
478     def construct_python_complex(self, node):
479        return complex(self.construct_scalar(node))
480
481     def construct_python_tuple(self, node):
482         return tuple(self.construct_sequence(node))
483
484     def find_python_module(self, name, mark):
485         if not name:
486             raise ConstructorError("while constructing a Python module", mark,
487                     "expected non-empty name appended to the tag", mark)
488         try:
489             __import__(name)
490         except ImportError, exc:
491             raise ConstructorError("while constructing a Python module", mark,
492                     "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark)
493         return sys.modules[name]
494
495     def find_python_name(self, name, mark):
496         if not name:
497             raise ConstructorError("while constructing a Python object", mark,
498                     "expected non-empty name appended to the tag", mark)
499         if u'.' in name:
500             module_name, object_name = name.rsplit('.', 1)
501         else:
502             module_name = '__builtin__'
503             object_name = name
504         try:
505             __import__(module_name)
506         except ImportError, exc:
507             raise ConstructorError("while constructing a Python object", mark,
508                     "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark)
509         module = sys.modules[module_name]
510         if not hasattr(module, object_name):
511             raise ConstructorError("while constructing a Python object", mark,
512                     "cannot find %r in the module %r" % (object_name.encode('utf-8'),
513                         module.__name__), mark)
514         return getattr(module, object_name)
515
516     def construct_python_name(self, suffix, node):
517         value = self.construct_scalar(node)
518         if value:
519             raise ConstructorError("while constructing a Python name", node.start_mark,
520                     "expected the empty value, but found %r" % value.encode('utf-8'),
521                     node.start_mark)
522         return self.find_python_name(suffix, node.start_mark)
523
524     def construct_python_module(self, suffix, node):
525         value = self.construct_scalar(node)
526         if value:
527             raise ConstructorError("while constructing a Python module", node.start_mark,
528                     "expected the empty value, but found %r" % value.encode('utf-8'),
529                     node.start_mark)
530         return self.find_python_module(suffix, node.start_mark)
531
532     class classobj: pass
533
534     def make_python_instance(self, suffix, node,
535             args=None, kwds=None, newobj=False):
536         if not args:
537             args = []
538         if not kwds:
539             kwds = {}
540         cls = self.find_python_name(suffix, node.start_mark)
541         if newobj and isinstance(cls, type(self.classobj))  \
542                 and not args and not kwds:
543             instance = self.classobj()
544             instance.__class__ = cls
545             return instance
546         elif newobj and isinstance(cls, type):
547             return cls.__new__(cls, *args, **kwds)
548         else:
549             return cls(*args, **kwds)
550
551     def set_python_instance_state(self, instance, state):
552         if hasattr(instance, '__setstate__'):
553             instance.__setstate__(state)
554         else:
555             slotstate = {}
556             if isinstance(state, tuple) and len(state) == 2:
557                 state, slotstate = state
558             if hasattr(instance, '__dict__'):
559                 instance.__dict__.update(state)
560             elif state:
561                 slotstate.update(state)
562             for key, value in slotstate.items():
563                 setattr(object, key, value)
564
565     def construct_python_object(self, suffix, node):
566         # Format:
567         #   !!python/object:module.name { ... state ... }
568         instance = self.make_python_instance(suffix, node, newobj=True)
569         yield instance
570         deep = hasattr(instance, '__setstate__')
571         state = self.construct_mapping(node, deep=deep)
572         self.set_python_instance_state(instance, state)
573
574     def construct_python_object_apply(self, suffix, node, newobj=False):
575         # Format:
576         #   !!python/object/apply       # (or !!python/object/new)
577         #   args: [ ... arguments ... ]
578         #   kwds: { ... keywords ... }
579         #   state: ... state ...
580         #   listitems: [ ... listitems ... ]
581         #   dictitems: { ... dictitems ... }
582         # or short format:
583         #   !!python/object/apply [ ... arguments ... ]
584         # The difference between !!python/object/apply and !!python/object/new
585         # is how an object is created, check make_python_instance for details.
586         if isinstance(node, SequenceNode):
587             args = self.construct_sequence(node, deep=True)
588             kwds = {}
589             state = {}
590             listitems = []
591             dictitems = {}
592         else:
593             value = self.construct_mapping(node, deep=True)
594             args = value.get('args', [])
595             kwds = value.get('kwds', {})
596             state = value.get('state', {})
597             listitems = value.get('listitems', [])
598             dictitems = value.get('dictitems', {})
599         instance = self.make_python_instance(suffix, node, args, kwds, newobj)
600         if state:
601             self.set_python_instance_state(instance, state)
602         if listitems:
603             instance.extend(listitems)
604         if dictitems:
605             for key in dictitems:
606                 instance[key] = dictitems[key]
607         return instance
608
609     def construct_python_object_new(self, suffix, node):
610         return self.construct_python_object_apply(suffix, node, newobj=True)
611
612 Constructor.add_constructor(
613     u'tag:yaml.org,2002:python/none',
614     Constructor.construct_yaml_null)
615
616 Constructor.add_constructor(
617     u'tag:yaml.org,2002:python/bool',
618     Constructor.construct_yaml_bool)
619
620 Constructor.add_constructor(
621     u'tag:yaml.org,2002:python/str',
622     Constructor.construct_python_str)
623
624 Constructor.add_constructor(
625     u'tag:yaml.org,2002:python/unicode',
626     Constructor.construct_python_unicode)
627
628 Constructor.add_constructor(
629     u'tag:yaml.org,2002:python/int',
630     Constructor.construct_yaml_int)
631
632 Constructor.add_constructor(
633     u'tag:yaml.org,2002:python/long',
634     Constructor.construct_python_long)
635
636 Constructor.add_constructor(
637     u'tag:yaml.org,2002:python/float',
638     Constructor.construct_yaml_float)
639
640 Constructor.add_constructor(
641     u'tag:yaml.org,2002:python/complex',
642     Constructor.construct_python_complex)
643
644 Constructor.add_constructor(
645     u'tag:yaml.org,2002:python/list',
646     Constructor.construct_yaml_seq)
647
648 Constructor.add_constructor(
649     u'tag:yaml.org,2002:python/tuple',
650     Constructor.construct_python_tuple)
651
652 Constructor.add_constructor(
653     u'tag:yaml.org,2002:python/dict',
654     Constructor.construct_yaml_map)
655
656 Constructor.add_multi_constructor(
657     u'tag:yaml.org,2002:python/name:',
658     Constructor.construct_python_name)
659
660 Constructor.add_multi_constructor(
661     u'tag:yaml.org,2002:python/module:',
662     Constructor.construct_python_module)
663
664 Constructor.add_multi_constructor(
665     u'tag:yaml.org,2002:python/object:',
666     Constructor.construct_python_object)
667
668 Constructor.add_multi_constructor(
669     u'tag:yaml.org,2002:python/object/apply:',
670     Constructor.construct_python_object_apply)
671
672 Constructor.add_multi_constructor(
673     u'tag:yaml.org,2002:python/object/new:',
674     Constructor.construct_python_object_new)
675