[buildstream] 01/05: element.py: Extract __expand_splits to a cython module

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch bschubert/optimize-element-init
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit efd9d179916540c1aa10c5458126d90caf882d5a
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Thu Oct 10 15:19:37 2019 +0100

    element.py: Extract __expand_splits to a cython module
    
    __expand_splits takes roughly 4% of our runtime in our benchmarks.
    
    Moving it to cython allows us to regain part of this time and speeds
    up all buildstream's invocations.
    
    - export _variables symbols for cython
    - introduce _element.pyx for cython extension of element.py
---
 setup.py                       |  1 +
 src/buildstream/_element.pyx   | 37 +++++++++++++++++++++++++++++++++++++
 src/buildstream/_variables.pxd | 26 ++++++++++++++++++++++++++
 src/buildstream/_variables.pyx |  6 +-----
 src/buildstream/element.py     | 18 ++----------------
 5 files changed, 67 insertions(+), 21 deletions(-)

diff --git a/setup.py b/setup.py
index 19779fb..e0d3dcd 100755
--- a/setup.py
+++ b/setup.py
@@ -403,6 +403,7 @@ def register_cython_module(module_name, dependencies=None):
 BUILD_EXTENSIONS = []
 
 register_cython_module("buildstream.node")
+register_cython_module("buildstream._element", dependencies=["buildstream.node", "buildstream._variables"])
 register_cython_module("buildstream._loader._loader")
 register_cython_module("buildstream._loader.loadelement")
 register_cython_module("buildstream._loader.types", dependencies=["buildstream.node"])
diff --git a/src/buildstream/_element.pyx b/src/buildstream/_element.pyx
new file mode 100644
index 0000000..f4e2011
--- /dev/null
+++ b/src/buildstream/_element.pyx
@@ -0,0 +1,37 @@
+#
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+from .node cimport MappingNode, SequenceNode
+from ._variables cimport Variables
+
+
+# Expand the splits in the public data using the Variables in the element
+def expand_splits(MappingNode element_public not None, Variables variables not None):
+    cdef MappingNode element_bst = element_public.get_mapping('bst', default={})
+    cdef MappingNode element_splits = element_bst.get_mapping('split-rules', default={})
+    cdef str domain
+    cdef list new_splits
+    cdef SequenceNode splits
+
+    # Resolve any variables in the public split rules directly
+    for domain, splits in element_splits.items():
+        new_splits = [
+            variables.subst(split.strip())
+            for split in splits.as_str_list()
+        ]
+        element_splits[domain] = new_splits
+
+    return element_public
diff --git a/src/buildstream/_variables.pxd b/src/buildstream/_variables.pxd
new file mode 100644
index 0000000..28f9507
--- /dev/null
+++ b/src/buildstream/_variables.pxd
@@ -0,0 +1,26 @@
+#
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+from .node cimport MappingNode
+
+cdef class Variables:
+    cdef MappingNode original
+    cdef dict _expstr_map
+    cdef public dict flat
+
+    cpdef subst(self, str string)
+    cdef dict _resolve(self, MappingNode node)
+    cdef dict _flatten(self)
diff --git a/src/buildstream/_variables.pyx b/src/buildstream/_variables.pyx
index f8662ac..d5db8cd 100644
--- a/src/buildstream/_variables.pyx
+++ b/src/buildstream/_variables.pyx
@@ -65,10 +65,6 @@ PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
 #
 cdef class Variables:
 
-    cdef MappingNode original
-    cdef dict _expstr_map
-    cdef public dict flat
-
     def __init__(self, MappingNode node):
         self.original = node
         self._expstr_map = self._resolve(node)
@@ -87,7 +83,7 @@ cdef class Variables:
     # Raises:
     #    LoadError, if the string contains unresolved variable references.
     #
-    def subst(self, str string):
+    cpdef subst(self, str string):
         expstr = _parse_expstr(string)
 
         try:
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 966f0f7..e22495f 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -95,6 +95,7 @@ from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
 from .utils import FileListResult
 from . import utils
 from . import _cachekey
+from . import _element  # type: any
 from . import _signals
 from . import _site
 from ._platform import Platform
@@ -318,7 +319,7 @@ class Element(Plugin):
 
         # Grab public domain data declared for this instance
         unexpanded_public = self.__extract_public(meta)
-        self.__public = self.__expand_splits(unexpanded_public)
+        self.__public = _element.expand_splits(unexpanded_public, self.__variables)
         self.__dynamic_public = None
 
         # Collect the composited element configuration and
@@ -2846,21 +2847,6 @@ class Element(Plugin):
 
         return element_public
 
-    # Expand the splits in the public data using the Variables in the element
-    def __expand_splits(self, element_public):
-        element_bst = element_public.get_mapping('bst', default={})
-        element_splits = element_bst.get_mapping('split-rules', default={})
-
-        # Resolve any variables in the public split rules directly
-        for domain, splits in element_splits.items():
-            splits = [
-                self.__variables.subst(split.strip())
-                for split in splits.as_str_list()
-            ]
-            element_splits[domain] = splits
-
-        return element_public
-
     def __init_splits(self):
         bstdata = self.get_public_data('bst')
         splits = bstdata.get_mapping('split-rules')