You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ju...@apache.org on 2021/03/02 16:02:23 UTC
[buildstream] 08/08: Call _initialize_state() in
Element._new_from_load_element()
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/cache-query
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2787576dac8216440ca6d78cbfe848eb28c96b89
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu Oct 8 17:09:00 2020 +0200
Call _initialize_state() in Element._new_from_load_element()
With the cache queries moved out, `_initialize_state()` is fairly
lightweight and can be called earlier.
---
src/buildstream/_loader/loadelement.pyx | 1 -
src/buildstream/_loader/loader.py | 1 -
src/buildstream/_stream.py | 35 ---------------------------------
src/buildstream/element.py | 3 +++
tests/artifactcache/push.py | 7 -------
tests/sourcecache/fetch.py | 6 ------
tests/sourcecache/push.py | 2 --
tests/sourcecache/staging.py | 3 ---
8 files changed, 3 insertions(+), 55 deletions(-)
diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx
index 210869e..f69e138 100644
--- a/src/buildstream/_loader/loadelement.pyx
+++ b/src/buildstream/_loader/loadelement.pyx
@@ -286,7 +286,6 @@ cdef class LoadElement:
from ..element import Element
element = Element._new_from_load_element(self)
- element._initialize_state()
# Custom error for link dependencies, since we don't completely
# parse their dependencies we cannot rely on the built-in ElementError.
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 6db2017..df4df84 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -816,7 +816,6 @@ class Loader:
)
element = Element._new_from_load_element(load_element)
- element._initialize_state()
# Handle the case where a subproject has no ref
#
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 87bfeed..cef605e 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -1628,7 +1628,6 @@ class Stream:
# Now move on to loading primary selection.
#
- self._resolve_elements(self.targets)
selected = _pipeline.get_selection(self._context, self.targets, selection, silent=False)
selected = _pipeline.except_elements(self.targets, selected, except_elements)
@@ -1644,40 +1643,6 @@ class Stream:
return selected
- # _resolve_elements()
- #
- # Resolve element state and cache keys.
- #
- # Args:
- # targets (list of Element): The list of toplevel element targets
- #
- def _resolve_elements(self, targets):
- with self._context.messenger.simple_task("Resolving cached state", silent_nested=True) as task:
- # We need to go through the project to access the loader
- #
- # FIXME: We need to calculate the total elements to resolve differently so that
- # it can include artifact elements
- #
- if task and self._project:
- task.set_maximum_progress(self._project.loader.loaded)
-
- # XXX: Now that Element._update_state() can trigger recursive update_state calls
- # it is possible that we could get a RecursionError. However, this is unlikely
- # to happen, even for large projects (tested with the Debian stack). Although,
- # if it does become a problem we may have to set the recursion limit to a
- # greater value.
- for element in _pipeline.dependencies(targets, _Scope.ALL):
- # Determine initial element state.
- element._initialize_state()
-
- # We may already have Elements which are cached and have their runtimes
- # cached, if this is the case, we should immediately notify their reverse
- # dependencies.
- element._update_ready_for_runtime_and_cached()
-
- if task:
- task.add_current_progress()
-
# _reset()
#
# Resets the internal state related to a given scheduler run.
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 943852f..3695a42 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1135,6 +1135,8 @@ class Element(Plugin):
element.__preflight()
+ element._initialize_state()
+
if task:
task.add_current_progress()
@@ -2884,6 +2886,7 @@ class Element(Plugin):
self._mimic_artifact()
if not artifact.cached():
self.__pull_pending = True
+ self._initialize_state()
@classmethod
def __compose_default_splits(cls, project, defaults, first_pass):
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 6b31542..24a1ee1 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -6,7 +6,6 @@ import os
import pytest
from buildstream import _yaml
-from buildstream.types import _Scope
from buildstream._project import Project
from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from buildstream.testing import cli # pylint: disable=unused-import
@@ -33,12 +32,6 @@ def _push(cli, cache_dir, project_dir, config_file, target):
# Create a local artifact cache handle
artifactcache = context.artifactcache
- # Ensure the element's artifact memeber is initialised
- # This is duplicated from Pipeline.resolve_elements()
- # as this test does not use the cli frontend.
- for e in element._dependencies(_Scope.ALL):
- e._initialize_state()
-
# Initialize remotes
context.initialize_remotes(True, True, None, None)
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index 46f39b0..f37242f 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -74,7 +74,6 @@ def test_source_fetch(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
element = project.load_elements([element_name])[0]
- element._initialize_state()
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
@@ -115,7 +114,6 @@ def test_source_fetch(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
element = project.load_elements([element_name])[0]
- element._initialize_state()
# check that we have the source in the cas now and it's not fetched
element._query_source_cache()
@@ -136,7 +134,6 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
element = project.load_elements([element_name])[0]
- element._initialize_state()
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
@@ -155,7 +152,6 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
# Check that the source in both in the source dir and the local CAS
element = project.load_elements([element_name])[0]
- element._initialize_state()
element._query_source_cache()
assert element._cached_sources()
@@ -172,7 +168,6 @@ def test_pull_fail(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
element = project.load_elements([element_name])[0]
- element._initialize_state()
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
@@ -205,7 +200,6 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
element = project.load_elements([element_name])[0]
- element._initialize_state()
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 76333ca..47e845b 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -86,7 +86,6 @@ def test_source_push_split(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
element = project.load_elements(["push.bst"])[0]
- element._initialize_state()
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
@@ -137,7 +136,6 @@ def test_source_push(cli, tmpdir, datafiles):
project.ensure_fully_loaded()
element = project.load_elements(["push.bst"])[0]
- element._initialize_state()
element._query_source_cache()
assert not element._cached_sources()
source = list(element.sources())[0]
diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py
index e0e7002..6671c79 100644
--- a/tests/sourcecache/staging.py
+++ b/tests/sourcecache/staging.py
@@ -64,7 +64,6 @@ def test_source_staged(tmpdir, cli, datafiles):
# now check that the source is in the refs file, this is pretty messy but
# seems to be the only way to get the sources?
element = project.load_elements(["import-bin.bst"])[0]
- element._initialize_state()
element._query_source_cache()
source = list(element.sources())[0]
assert element._cached_sources()
@@ -100,7 +99,6 @@ def test_source_fetch(tmpdir, cli, datafiles):
sourcecache = context.sourcecache
element = project.load_elements(["import-dev.bst"])[0]
- element._initialize_state()
element._query_source_cache()
source = list(element.sources())[0]
assert element._cached_sources()
@@ -135,7 +133,6 @@ def test_staged_source_build(tmpdir, datafiles, cli):
project.ensure_fully_loaded()
element = project.load_elements(["import-dev.bst"])[0]
- element._initialize_state()
# check consistency of the source
element._query_source_cache()