Port v2 to the PyYAML C extension

v2/ansible/parsing/__init__.py

@@ -29,7 +29,7 @@ from ansible.errors.yaml_strings import YAML_SYNTAX_ERROR
 from ansible.parsing.vault import VaultLib
 from ansible.parsing.splitter import unquote
 from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject
+from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleUnicode
 from ansible.utils.path import unfrackpath
 
 class DataLoader():
@@ -70,13 +70,27 @@ class DataLoader():
             # we first try to load this data as JSON
             return json.loads(data)
         except:
+            # if loading JSON failed for any reason, we go ahead
+            # and try to parse it as YAML instead
+
+            if isinstance(data, AnsibleUnicode):
+                # The PyYAML's libyaml bindings use PyUnicode_CheckExact so
+                # they are unable to cope with our subclass.
+                # Unwrap and re-wrap the unicode so we can keep track of line
+                # numbers
+                new_data = unicode(data)
+            else:
+                new_data = data
             try:
-                # if loading JSON failed for any reason, we go ahead
+                new_data = self._safe_load(new_data, file_name=file_name)
-                # and try to parse it as YAML instead
-                return self._safe_load(data, file_name=file_name)
             except YAMLError as yaml_exc:
                 self._handle_error(yaml_exc, file_name, show_content)
 
+            if isinstance(data, AnsibleUnicode):
+                new_data = AnsibleUnicode(new_data)
+                new_data.ansible_pos = data.ansible_pos
+            return new_data
+
     def load_from_file(self, file_name):
         ''' Loads data from a file, which can contain either JSON or YAML.  '''
 

v2/ansible/parsing/yaml/constructor.py

@@ -20,7 +20,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from yaml.constructor import Constructor
-from ansible.utils.unicode import to_unicode
 from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleUnicode
 
 class AnsibleConstructor(Constructor):
@@ -33,20 +32,11 @@ class AnsibleConstructor(Constructor):
         yield data
         value = self.construct_mapping(node)
         data.update(value)
-        data.ansible_pos = value.ansible_pos
+        data.ansible_pos = self._node_position_info(node)
 
     def construct_mapping(self, node, deep=False):
         ret = AnsibleMapping(super(Constructor, self).construct_mapping(node, deep))
-
+        ret.ansible_pos = self._node_position_info(node)
-        # in some cases, we may have pre-read the data and then
-        # passed it to the load() call for YAML, in which case we
-        # want to override the default datasource (which would be
-        # '<string>') to the actual filename we read in
-        if self._ansible_file_name:
-            data_source = self._ansible_file_name
-        else:
-            data_source = node.__datasource__
-        ret.ansible_pos = (data_source, node.__line__, node.__column__)
 
         return ret
 
@@ -54,17 +44,25 @@ class AnsibleConstructor(Constructor):
         # Override the default string handling function
         # to always return unicode objects
         value = self.construct_scalar(node)
-        value = to_unicode(value)
+        ret = AnsibleUnicode(value)
-        ret = AnsibleUnicode(self.construct_scalar(node))
 
-        if self._ansible_file_name:
+        ret.ansible_pos = self._node_position_info(node)
-            data_source = self._ansible_file_name
-        else:
-            data_source = node.__datasource__
-        ret.ansible_pos = (data_source, node.__line__, node.__column__)
 
         return ret
 
+    def _node_position_info(self, node):
+        # the line number where the previous token has ended (plus empty lines)
+        column = node.start_mark.column + 1
+        line = node.start_mark.line + 1
+
+        # in some cases, we may have pre-read the data and then
+        # passed it to the load() call for YAML, in which case we
+        # want to override the default datasource (which would be
+        # '<string>') to the actual filename we read in
+        datasource = self._ansible_file_name or node.start_mark.name
+
+        return (datasource, line, column)
+
 AnsibleConstructor.add_constructor(
     u'tag:yaml.org,2002:map',
     AnsibleConstructor.construct_yaml_map)

v2/ansible/parsing/yaml/loader.py

@@ -19,20 +19,34 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-from yaml.reader import Reader
+try:
-from yaml.scanner import Scanner
+    from _yaml import CParser, CEmitter
-from yaml.parser import Parser
+    HAVE_PYYAML_C = True
+except ImportError:
+    HAVE_PYYAML_C = False
+
 from yaml.resolver import Resolver
 
-from ansible.parsing.yaml.composer import AnsibleComposer
 from ansible.parsing.yaml.constructor import AnsibleConstructor
 
-class AnsibleLoader(Reader, Scanner, Parser, AnsibleComposer, AnsibleConstructor, Resolver):
+if HAVE_PYYAML_C:
-    def __init__(self, stream, file_name=None):
+    class AnsibleLoader(CParser, AnsibleConstructor, Resolver):
-        Reader.__init__(self, stream)
+        def __init__(self, stream, file_name=None):
-        Scanner.__init__(self)
+            CParser.__init__(self, stream)
-        Parser.__init__(self)
+            AnsibleConstructor.__init__(self, file_name=file_name)
-        AnsibleComposer.__init__(self)
+            Resolver.__init__(self)
-        AnsibleConstructor.__init__(self, file_name=file_name)
+else:
-        Resolver.__init__(self)
+    from yaml.reader import Reader
+    from yaml.scanner import Scanner
+    from yaml.parser import Parser
 
+    from ansible.parsing.yaml.composer import AnsibleComposer
+
+    class AnsibleLoader(Reader, Scanner, Parser, AnsibleComposer, AnsibleConstructor, Resolver):
+        def __init__(self, stream, file_name=None):
+            Reader.__init__(self, stream)
+            Scanner.__init__(self)
+            Parser.__init__(self)
+            AnsibleComposer.__init__(self)
+            AnsibleConstructor.__init__(self, file_name=file_name)
+            Resolver.__init__(self)