[buildstream] 20/27: WIP: pickle first_pass_config too

[no...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

not-in-ldap pushed a commit to branch aevri/check_spawn_ci_working
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit ac39de3ca9e4a8acce4c1ceedf04a7fcb380e0f5
Author: Angelos Evripiotis <je...@bloomberg.net>
AuthorDate: Thu Oct 17 10:13:41 2019 +0100

    WIP: pickle first_pass_config too
---
 src/buildstream/_scheduler/jobs/jobpickler.py | 39 ++++++++++++++++++++++++++-
 src/buildstream/element.py                    |  4 +--
 src/buildstream/source.py                     |  3 ++-
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/src/buildstream/_scheduler/jobs/jobpickler.py b/src/buildstream/_scheduler/jobs/jobpickler.py
index c67353e..0b15feb 100644
--- a/src/buildstream/_scheduler/jobs/jobpickler.py
+++ b/src/buildstream/_scheduler/jobs/jobpickler.py
@@ -66,17 +66,29 @@ from ..._messenger import Messenger
 #
 def pickle_child_job(child_job, projects):
 
+    p = child_job._element._Plugin__project
+
     element_classes = [
         cls
         for p in projects
         if p.config.element_factory is not None
         for cls, _ in p.config.element_factory.all_loaded_plugins()
+    ] + [
+        cls
+        for p in projects
+        if p.first_pass_config.element_factory is not None
+        for cls, _ in p.first_pass_config.element_factory.all_loaded_plugins()
     ]
     source_classes = [
         cls
         for p in projects
         if p.config.source_factory is not None
         for cls, _ in p.config.source_factory.all_loaded_plugins()
+    ] + [
+        cls
+        for p in projects
+        if p.first_pass_config.source_factory is not None
+        for cls, _ in p.first_pass_config.source_factory.all_loaded_plugins()
     ]
 
     data = io.BytesIO()
@@ -121,7 +133,32 @@ def _new_artifact_proto_from_reduction_args(data):
 
 
 def _reduce_plugin(plugin):
-    factory, meta_kind, state = plugin._get_args_for_child_job_pickling()
+    project, meta_kind, state = plugin._get_args_for_child_job_pickling()
+    assert project
+    assert meta_kind
+
+    factories = [
+        project.config.element_factory,
+        project.first_pass_config.element_factory,
+        project.config.source_factory,
+        project.first_pass_config.source_factory,
+    ]
+
+    print("plugin:", plugin)
+    print("factories:", factories)
+
+    factory = None
+    for f in factories:
+        if f is None:
+            continue
+        for cls, _ in f.all_loaded_plugins():
+            print(f, "comparing", type(plugin), cls)
+            if type(plugin) == cls:
+                factory = f
+
+    if factory is None:
+        raise Exception("Couldn't find plugin in factories")
+
     args = (factory, meta_kind)
     return (_new_plugin_from_reduction_args, args, state)
 
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 966f0f7..3be6b80 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -2373,8 +2373,8 @@ class Element(Plugin):
         # let us know, and we will need to update accordingly.
         del state["_Element__required_callback"]
 
-        factory = self._get_project().config.element_factory
-        return factory, self.__meta_kind, state
+        # factory = self._get_project().config.element_factory
+        return self._get_project(), self.__meta_kind, state
 
     def _walk_artifact_files(self):
         yield from self.__artifact.get_files().walk()
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index 1fb318b..76c5d57 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -1131,7 +1131,8 @@ class Source(Plugin):
         #   o The code sketch of how pickling works also returns `self.__dict__`:
         #     https://docs.python.org/3/library/pickle.html#pickling-class-instances
         #
-        return factory, self.__meta_kind, self.__dict__
+        # return factory, self.__meta_kind, self.__dict__
+        return self._get_project(), self.__meta_kind, self.__dict__
 
     #############################################################
     #                   Local Private Methods                   #