You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by st...@apache.org on 2026/03/23 00:12:51 UTC
(buildstream) 04/15: speculative actions: Add unit tests
This is an automated email from the ASF dual-hosted git repository.
striker pushed a commit to branch striker/speculative-actions
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1dd1e11a675edb4cae02718b9d060cfc66434bca
Author: Sander Striker <st...@apache.org>
AuthorDate: Mon Mar 16 18:28:12 2026 +0100
speculative actions: Add unit tests
32 tests covering the full speculative actions pipeline without sandbox:
Weak key (test_weak_key.py, 13 tests):
- Stability: same inputs, dep version changes, dependency ordering
- Invalidation: source, command, env, sandbox, plugin, dep changes
Generator (test_generator_unit.py, 6 tests):
- SOURCE/ARTIFACT overlay production, priority, unknown digests
Instantiator (test_instantiator_unit.py, 6 tests):
- Digest replacement, nested dirs, multiple overlays, artifact overlays
Pipeline integration (test_pipeline_integration.py, 7 tests):
- Generate -> store -> retrieve -> instantiate roundtrips
- Priming scenario: dep changes, adapted actions have updated digests
Co-Authored-By: Claude Opus 4.6 (1M context) <no...@anthropic.com>
---
tests/speculative_actions/__init__.py | 0
tests/speculative_actions/test_generator_unit.py | 402 +++++++++++
.../speculative_actions/test_instantiator_unit.py | 434 ++++++++++++
.../test_pipeline_integration.py | 771 +++++++++++++++++++++
tests/speculative_actions/test_weak_key.py | 211 ++++++
5 files changed, 1818 insertions(+)
diff --git a/tests/speculative_actions/__init__.py b/tests/speculative_actions/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/speculative_actions/test_generator_unit.py b/tests/speculative_actions/test_generator_unit.py
new file mode 100644
index 000000000..db5aa3b24
--- /dev/null
+++ b/tests/speculative_actions/test_generator_unit.py
@@ -0,0 +1,402 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Unit tests for SpeculativeActionsGenerator.
+
+These tests construct Action + Directory protos in-memory and verify
+that the Generator correctly produces overlays. No sandbox needed.
+"""
+
+import hashlib
+import pytest
+
+from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildstream._protos.buildstream.v2 import speculative_actions_pb2
+
+
+def _make_digest(content):
+ """Create a Digest proto from content bytes."""
+ digest = remote_execution_pb2.Digest()
+ digest.hash = hashlib.sha256(content).hexdigest()
+ digest.size_bytes = len(content)
+ return digest
+
+
+class FakeCAS:
+ """In-memory CAS for testing without a real CAS daemon."""
+
+ def __init__(self):
+ self._blobs = {} # hash -> bytes
+ self._directories = {} # hash -> Directory proto
+ self._actions = {} # hash -> Action proto
+
+ def store_directory_proto(self, directory):
+ data = directory.SerializeToString()
+ digest = _make_digest(data)
+ self._directories[digest.hash] = directory
+ self._blobs[digest.hash] = data
+ return digest
+
+ def fetch_directory_proto(self, digest):
+ return self._directories.get(digest.hash)
+
+ def store_action(self, action):
+ data = action.SerializeToString()
+ digest = _make_digest(data)
+ self._actions[digest.hash] = action
+ self._blobs[digest.hash] = data
+ return digest
+
+ def fetch_action(self, digest):
+ return self._actions.get(digest.hash)
+
+ def store_proto(self, proto):
+ data = proto.SerializeToString()
+ return _make_digest(data)
+
+ def fetch_proto(self, digest, proto_class):
+ data = self._blobs.get(digest.hash)
+ if data is None:
+ return None
+ proto = proto_class()
+ proto.ParseFromString(data)
+ return proto
+
+
+class FakeSourceDir:
+ """Fake source directory with a digest."""
+
+ def __init__(self, digest):
+ self._digest = digest
+
+ def _get_digest(self):
+ return self._digest
+
+
+class FakeSources:
+ """Fake ElementSources."""
+
+ def __init__(self, files_dir):
+ self._files_dir = files_dir
+ self._cached = True
+
+ def cached(self):
+ return self._cached
+
+ def get_files(self):
+ return self._files_dir
+
+
+class FakeArtifact:
+ """Fake Artifact."""
+
+ def __init__(self, files_dir, is_cached=True):
+ self._files_dir = files_dir
+ self._cached = is_cached
+
+ def cached(self):
+ return self._cached
+
+ def get_files(self):
+ return self._files_dir
+
+
+class FakeElement:
+ """Fake Element for testing Generator without a real Element."""
+
+ def __init__(self, name, sources=None, artifact=None):
+ self.name = name
+ self._Element__sources = sources
+ self._artifact = artifact
+
+ def sources(self):
+ if self._Element__sources:
+ yield True # Just needs to be non-empty
+
+ def _cached(self):
+ return self._artifact is not None and self._artifact.cached()
+
+ def _get_artifact(self):
+ return self._artifact
+
+
+def _build_source_tree(cas, files):
+ """Build a CAS directory tree from a dict of {path: content_bytes}.
+
+ Args:
+ cas: FakeCAS instance
+ files: Dict mapping relative paths to content bytes
+
+ Returns:
+ Digest of root directory
+ """
+ # Group files by directory
+ dirs = {}
+ for path, content in files.items():
+ parts = path.rsplit("/", 1)
+ if len(parts) == 1:
+ dirname, filename = "", parts[0]
+ else:
+ dirname, filename = parts
+ dirs.setdefault(dirname, []).append((filename, content))
+
+ # Build leaf directories first, then work up
+ dir_digests = {}
+
+ # Sort paths by depth (deepest first)
+ all_dirs = set()
+ for path in files:
+ parts = path.split("/")
+ for i in range(len(parts) - 1):
+ all_dirs.add("/".join(parts[: i + 1]))
+ all_dirs.add("") # root
+
+ # Process deepest directories first, root ("") always last
+ non_root = sorted((d for d in all_dirs if d), key=lambda d: -d.count("/"))
+ non_root.append("")
+
+ for dirpath in non_root:
+ directory = remote_execution_pb2.Directory()
+
+ # Add files in this directory
+ for filename, content in dirs.get(dirpath, []):
+ file_node = directory.files.add()
+ file_node.name = filename
+ file_node.digest.CopyFrom(_make_digest(content))
+
+ # Add subdirectories
+ for child_dir, child_digest in sorted(dir_digests.items()):
+ # Check if child_dir is a direct subdirectory of dirpath
+ if dirpath == "":
+ if "/" not in child_dir:
+ dir_node = directory.directories.add()
+ dir_node.name = child_dir
+ dir_node.digest.CopyFrom(child_digest)
+ else:
+ prefix = dirpath + "/"
+ if child_dir.startswith(prefix) and "/" not in child_dir[len(prefix) :]:
+ dir_node = directory.directories.add()
+ dir_node.name = child_dir[len(prefix) :]
+ dir_node.digest.CopyFrom(child_digest)
+
+ digest = cas.store_directory_proto(directory)
+ dir_digests[dirpath] = digest
+
+ return dir_digests[""]
+
+
+def _build_action(cas, input_root_digest):
+ """Build an Action proto with the given input root."""
+ action = remote_execution_pb2.Action()
+ action.input_root_digest.CopyFrom(input_root_digest)
+ return cas.store_action(action)
+
+
+class TestGeneratorOverlayProduction:
+ """Test that Generator correctly produces overlays from subactions."""
+
+ def test_generates_source_overlays(self):
+ """Files found in element sources should produce SOURCE overlays."""
+ from buildstream._speculative_actions.generator import SpeculativeActionsGenerator
+
+ cas = FakeCAS()
+
+ # Create source files
+ source_files = {
+ "main.c": b'int main() { return 0; }',
+ "util.h": b'#pragma once\nvoid util();',
+ }
+ source_root = _build_source_tree(cas, source_files)
+ sources = FakeSources(FakeSourceDir(source_root))
+
+ # Create an action that uses these source files in its input tree
+ action_input = _build_source_tree(cas, {
+ "src/main.c": b'int main() { return 0; }',
+ "src/util.h": b'#pragma once\nvoid util();',
+ })
+ action_digest = _build_action(cas, action_input)
+
+ element = FakeElement("test-element.bst", sources=sources)
+ generator = SpeculativeActionsGenerator(cas)
+
+ spec_actions = generator.generate_speculative_actions(element, [action_digest], [])
+
+ assert spec_actions is not None
+ assert len(spec_actions.actions) == 1
+
+ action = spec_actions.actions[0]
+ # Should have overlays for the source files found in the action input
+ assert len(action.overlays) > 0
+ # All overlays should be SOURCE type
+ for overlay in action.overlays:
+ assert overlay.type == speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+
+ def test_generates_artifact_overlays_for_dependencies(self):
+ """Files from dependency artifacts should produce ARTIFACT overlays."""
+ from buildstream._speculative_actions.generator import SpeculativeActionsGenerator
+
+ cas = FakeCAS()
+
+ # Create a dependency artifact with library files
+ dep_files = {
+ "lib/libfoo.so": b'fake-shared-object-content',
+ }
+ dep_root = _build_source_tree(cas, dep_files)
+ dep_artifact = FakeArtifact(FakeSourceDir(dep_root))
+ dep_element = FakeElement("dep.bst", artifact=dep_artifact)
+
+ # Create element sources (no overlap with dep)
+ source_files = {
+ "main.c": b'int main() { return 0; }',
+ }
+ source_root = _build_source_tree(cas, source_files)
+ sources = FakeSources(FakeSourceDir(source_root))
+
+ # Create an action that uses both source files and dep artifacts
+ action_input = _build_source_tree(cas, {
+ "src/main.c": b'int main() { return 0; }',
+ "lib/libfoo.so": b'fake-shared-object-content',
+ })
+ action_digest = _build_action(cas, action_input)
+
+ element = FakeElement("test-element.bst", sources=sources)
+ generator = SpeculativeActionsGenerator(cas)
+
+ spec_actions = generator.generate_speculative_actions(element, [action_digest], [dep_element])
+
+ assert spec_actions is not None
+ assert len(spec_actions.actions) == 1
+
+ action = spec_actions.actions[0]
+ overlay_types = {o.type for o in action.overlays}
+ # Should have both SOURCE and ARTIFACT overlays
+ assert speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE in overlay_types
+ assert speculative_actions_pb2.SpeculativeActions.Overlay.ARTIFACT in overlay_types
+
+ def test_source_priority_over_artifact(self):
+ """When same digest exists in both source and artifact, SOURCE wins."""
+ from buildstream._speculative_actions.generator import SpeculativeActionsGenerator
+
+ cas = FakeCAS()
+
+ shared_content = b'shared-file-content'
+
+ # Create element sources with the shared file
+ source_root = _build_source_tree(cas, {
+ "shared.h": shared_content,
+ })
+ sources = FakeSources(FakeSourceDir(source_root))
+
+ # Create dependency artifact with the same file
+ dep_root = _build_source_tree(cas, {
+ "include/shared.h": shared_content,
+ })
+ dep_artifact = FakeArtifact(FakeSourceDir(dep_root))
+ dep_element = FakeElement("dep.bst", artifact=dep_artifact)
+
+ # Action uses the shared file
+ action_input = _build_source_tree(cas, {
+ "shared.h": shared_content,
+ })
+ action_digest = _build_action(cas, action_input)
+
+ element = FakeElement("test-element.bst", sources=sources)
+ generator = SpeculativeActionsGenerator(cas)
+
+ spec_actions = generator.generate_speculative_actions(element, [action_digest], [dep_element])
+
+ assert len(spec_actions.actions) == 1
+ action = spec_actions.actions[0]
+ # The overlay should be SOURCE (higher priority)
+ for overlay in action.overlays:
+ if overlay.target_digest.hash == _make_digest(shared_content).hash:
+ assert overlay.type == speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+
+ def test_no_overlays_for_unknown_digests(self):
+ """Digests not found in sources or artifacts should produce no overlays."""
+ from buildstream._speculative_actions.generator import SpeculativeActionsGenerator
+
+ cas = FakeCAS()
+
+ # Empty sources
+ source_root = _build_source_tree(cas, {})
+ sources = FakeSources(FakeSourceDir(source_root))
+
+ # Action with files not in any source
+ action_input = _build_source_tree(cas, {
+ "unknown.bin": b'mystery-content',
+ })
+ action_digest = _build_action(cas, action_input)
+
+ element = FakeElement("test-element.bst", sources=sources)
+ generator = SpeculativeActionsGenerator(cas)
+
+ spec_actions = generator.generate_speculative_actions(element, [action_digest], [])
+
+ # No overlays should be generated (action with no overlays is excluded)
+ assert len(spec_actions.actions) == 0
+
+ def test_multiple_subactions(self):
+ """Multiple subaction digests should each produce a SpeculativeAction."""
+ from buildstream._speculative_actions.generator import SpeculativeActionsGenerator
+
+ cas = FakeCAS()
+
+ source_files = {
+ "a.c": b'void a() {}',
+ "b.c": b'void b() {}',
+ }
+ source_root = _build_source_tree(cas, source_files)
+ sources = FakeSources(FakeSourceDir(source_root))
+
+ # Two separate actions
+ action1_input = _build_source_tree(cas, {"src/a.c": b'void a() {}'})
+ action1_digest = _build_action(cas, action1_input)
+
+ action2_input = _build_source_tree(cas, {"src/b.c": b'void b() {}'})
+ action2_digest = _build_action(cas, action2_input)
+
+ element = FakeElement("test-element.bst", sources=sources)
+ generator = SpeculativeActionsGenerator(cas)
+
+ spec_actions = generator.generate_speculative_actions(
+ element, [action1_digest, action2_digest], []
+ )
+
+ assert len(spec_actions.actions) == 2
+
+ def test_element_artifact_overlays_generated(self):
+ """artifact_overlays should be generated for cached element output."""
+ from buildstream._speculative_actions.generator import SpeculativeActionsGenerator
+
+ cas = FakeCAS()
+
+ source_files = {"main.c": b'int main() { return 0; }'}
+ source_root = _build_source_tree(cas, source_files)
+ sources = FakeSources(FakeSourceDir(source_root))
+
+ # Element also has a cached artifact
+ artifact_files = {"bin/main": b'compiled-binary'}
+ artifact_root = _build_source_tree(cas, artifact_files)
+ artifact = FakeArtifact(FakeSourceDir(artifact_root))
+
+ element = FakeElement("test-element.bst", sources=sources, artifact=artifact)
+
+ # No subactions, just check artifact_overlays
+ generator = SpeculativeActionsGenerator(cas)
+ spec_actions = generator.generate_speculative_actions(element, [], [])
+
+ # No subaction overlays but artifact_overlays may be present
+ # (bin/main is not in source, so it won't be resolved)
+ assert spec_actions is not None
diff --git a/tests/speculative_actions/test_instantiator_unit.py b/tests/speculative_actions/test_instantiator_unit.py
new file mode 100644
index 000000000..9a60a2df8
--- /dev/null
+++ b/tests/speculative_actions/test_instantiator_unit.py
@@ -0,0 +1,434 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Unit tests for SpeculativeActionInstantiator.
+
+Given overlays and new file digests, verify correct digest replacements
+in action input trees. No sandbox needed.
+"""
+
+import hashlib
+import pytest
+
+from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildstream._protos.buildstream.v2 import speculative_actions_pb2
+
+
+def _make_digest(content):
+ """Create a Digest proto from content bytes."""
+ digest = remote_execution_pb2.Digest()
+ digest.hash = hashlib.sha256(content).hexdigest()
+ digest.size_bytes = len(content)
+ return digest
+
+
+class FakeCAS:
+ """In-memory CAS for testing."""
+
+ def __init__(self):
+ self._blobs = {}
+ self._directories = {}
+ self._actions = {}
+
+ def store_directory_proto(self, directory):
+ data = directory.SerializeToString()
+ digest = _make_digest(data)
+ self._directories[digest.hash] = directory
+ self._blobs[digest.hash] = data
+ return digest
+
+ def fetch_directory_proto(self, digest):
+ return self._directories.get(digest.hash)
+
+ def store_action(self, action):
+ data = action.SerializeToString()
+ digest = _make_digest(data)
+ self._actions[digest.hash] = action
+ self._blobs[digest.hash] = data
+ return digest
+
+ def fetch_action(self, digest):
+ return self._actions.get(digest.hash)
+
+ def store_proto(self, proto):
+ data = proto.SerializeToString()
+ return _make_digest(data)
+
+ def fetch_proto(self, digest, proto_class):
+ data = self._blobs.get(digest.hash)
+ if data is None:
+ return None
+ proto = proto_class()
+ proto.ParseFromString(data)
+ return proto
+
+
+class FakeSourceDir:
+ def __init__(self, digest):
+ self._digest = digest
+
+ def _get_digest(self):
+ return self._digest
+
+
+class FakeSources:
+ def __init__(self, files_dir):
+ self._files_dir = files_dir
+ self._cached = True
+
+ def cached(self):
+ return self._cached
+
+ def get_files(self):
+ return self._files_dir
+
+
+class FakeArtifact:
+ def __init__(self, files_dir=None, is_cached=True, proto=None):
+ self._files_dir = files_dir
+ self._cached = is_cached
+ self._proto = proto
+
+ def cached(self):
+ return self._cached
+
+ def get_files(self):
+ return self._files_dir
+
+ def _get_proto(self):
+ return self._proto
+
+
+class FakeArtifactCache:
+ def __init__(self):
+ self._spec_actions = {}
+
+ def get_speculative_actions(self, artifact, structural_key=None):
+ return self._spec_actions.get(id(artifact))
+
+ def store_speculative_actions(self, artifact, spec_actions, structural_key=None):
+ self._spec_actions[id(artifact)] = spec_actions
+
+
+class FakeElement:
+ def __init__(self, name, sources=None, artifact=None, project_name="project"):
+ self.name = name
+ self.project_name = project_name
+ self._Element__sources = sources
+ self._artifact = artifact
+
+ def sources(self):
+ if self._Element__sources:
+ yield True
+
+ def _cached(self):
+ return self._artifact is not None and self._artifact.cached()
+
+ def _get_artifact(self):
+ return self._artifact
+
+ def _dependencies(self, scope, recurse=False):
+ return []
+
+ def _get_cache_key(self):
+ return "fake-cache-key"
+
+ def info(self, msg):
+ pass
+
+ def warn(self, msg):
+ pass
+
+
+def _build_source_tree(cas, files):
+ """Build a CAS directory tree from a dict of {path: content_bytes}."""
+ dirs = {}
+ for path, content in files.items():
+ parts = path.rsplit("/", 1)
+ if len(parts) == 1:
+ dirname, filename = "", parts[0]
+ else:
+ dirname, filename = parts
+ dirs.setdefault(dirname, []).append((filename, content))
+
+ dir_digests = {}
+ all_dirs = set()
+ for path in files:
+ parts = path.split("/")
+ for i in range(len(parts) - 1):
+ all_dirs.add("/".join(parts[: i + 1]))
+ all_dirs.add("")
+
+ # Process deepest directories first, root ("") always last
+ non_root = sorted((d for d in all_dirs if d), key=lambda d: -d.count("/"))
+ non_root.append("")
+
+ for dirpath in non_root:
+ directory = remote_execution_pb2.Directory()
+ for filename, content in dirs.get(dirpath, []):
+ file_node = directory.files.add()
+ file_node.name = filename
+ file_node.digest.CopyFrom(_make_digest(content))
+
+ for child_dir, child_digest in sorted(dir_digests.items()):
+ if dirpath == "":
+ if "/" not in child_dir:
+ dir_node = directory.directories.add()
+ dir_node.name = child_dir
+ dir_node.digest.CopyFrom(child_digest)
+ else:
+ prefix = dirpath + "/"
+ if child_dir.startswith(prefix) and "/" not in child_dir[len(prefix) :]:
+ dir_node = directory.directories.add()
+ dir_node.name = child_dir[len(prefix) :]
+ dir_node.digest.CopyFrom(child_digest)
+
+ digest = cas.store_directory_proto(directory)
+ dir_digests[dirpath] = digest
+
+ return dir_digests[""]
+
+
+class TestInstantiatorDigestReplacement:
+ """Test that Instantiator correctly replaces digests in action trees."""
+
+ def test_replaces_source_digest(self):
+ """SOURCE overlay should replace old digest with current source digest."""
+ from buildstream._speculative_actions.instantiator import SpeculativeActionInstantiator
+
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache()
+
+ old_content = b'old source content'
+ new_content = b'new source content'
+ old_digest = _make_digest(old_content)
+ new_digest = _make_digest(new_content)
+
+ # Build the original action input tree with old content
+ input_root = _build_source_tree(cas, {"main.c": old_content})
+ action = remote_execution_pb2.Action()
+ action.input_root_digest.CopyFrom(input_root)
+ action_digest = cas.store_action(action)
+
+ # Build current source tree with new content
+ new_source_root = _build_source_tree(cas, {"main.c": new_content})
+ sources = FakeSources(FakeSourceDir(new_source_root))
+ element = FakeElement("test.bst", sources=sources)
+
+ # Create a SpeculativeAction with SOURCE overlay
+ spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction()
+ spec_action.base_action_digest.CopyFrom(action_digest)
+ overlay = spec_action.overlays.add()
+ overlay.type = speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+ overlay.source_element = "" # self
+ overlay.source_path = "main.c"
+ overlay.target_digest.CopyFrom(old_digest)
+
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(spec_action, element, {})
+
+ assert result_digest is not None
+ # The result should be a new action (different digest since content changed)
+ assert result_digest.hash != action_digest.hash
+
+ # Verify the new action has the updated input tree
+ new_action = cas.fetch_action(result_digest)
+ assert new_action is not None
+ new_root = cas.fetch_directory_proto(new_action.input_root_digest)
+ assert new_root is not None
+ assert len(new_root.files) == 1
+ assert new_root.files[0].digest.hash == new_digest.hash
+
+ def test_unchanged_digest_returns_base(self):
+ """When no digests actually change, return the base action digest."""
+ from buildstream._speculative_actions.instantiator import SpeculativeActionInstantiator
+
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache()
+
+ content = b'same content'
+ digest = _make_digest(content)
+
+ input_root = _build_source_tree(cas, {"main.c": content})
+ action = remote_execution_pb2.Action()
+ action.input_root_digest.CopyFrom(input_root)
+ action_digest = cas.store_action(action)
+
+ # Sources have the same content
+ source_root = _build_source_tree(cas, {"main.c": content})
+ sources = FakeSources(FakeSourceDir(source_root))
+ element = FakeElement("test.bst", sources=sources)
+
+ spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction()
+ spec_action.base_action_digest.CopyFrom(action_digest)
+ overlay = spec_action.overlays.add()
+ overlay.type = speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+ overlay.source_element = ""
+ overlay.source_path = "main.c"
+ overlay.target_digest.CopyFrom(digest)
+
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(spec_action, element, {})
+
+ # Should return the base action digest (no modifications)
+ assert result_digest.hash == action_digest.hash
+
+ def test_missing_base_action_returns_none(self):
+ """If the base action can't be fetched, return None."""
+ from buildstream._speculative_actions.instantiator import SpeculativeActionInstantiator
+
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache()
+
+ # Create a digest for a non-existent action
+ fake_digest = _make_digest(b'does-not-exist')
+
+ spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction()
+ spec_action.base_action_digest.CopyFrom(fake_digest)
+
+ element = FakeElement("test.bst")
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result = instantiator.instantiate_action(spec_action, element, {})
+
+ assert result is None
+
+ def test_replaces_in_nested_directories(self):
+ """Digests in nested directory trees should be replaced."""
+ from buildstream._speculative_actions.instantiator import SpeculativeActionInstantiator
+
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache()
+
+ old_content = b'old nested file'
+ new_content = b'new nested file'
+ old_digest = _make_digest(old_content)
+ new_digest = _make_digest(new_content)
+
+ # Build nested input tree
+ input_root = _build_source_tree(cas, {"src/lib/util.c": old_content})
+ action = remote_execution_pb2.Action()
+ action.input_root_digest.CopyFrom(input_root)
+ action_digest = cas.store_action(action)
+
+ # New sources with updated content
+ source_root = _build_source_tree(cas, {"lib/util.c": new_content})
+ sources = FakeSources(FakeSourceDir(source_root))
+ element = FakeElement("test.bst", sources=sources)
+
+ spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction()
+ spec_action.base_action_digest.CopyFrom(action_digest)
+ overlay = spec_action.overlays.add()
+ overlay.type = speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+ overlay.source_element = ""
+ overlay.source_path = "lib/util.c"
+ overlay.target_digest.CopyFrom(old_digest)
+
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(spec_action, element, {})
+
+ assert result_digest is not None
+ assert result_digest.hash != action_digest.hash
+
+ def test_multiple_overlays_applied(self):
+ """Multiple overlays should all be applied to the same action."""
+ from buildstream._speculative_actions.instantiator import SpeculativeActionInstantiator
+
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache()
+
+ old_a = b'old a.c'
+ old_b = b'old b.c'
+ new_a = b'new a.c'
+ new_b = b'new b.c'
+
+ input_root = _build_source_tree(cas, {"a.c": old_a, "b.c": old_b})
+ action = remote_execution_pb2.Action()
+ action.input_root_digest.CopyFrom(input_root)
+ action_digest = cas.store_action(action)
+
+ source_root = _build_source_tree(cas, {"a.c": new_a, "b.c": new_b})
+ sources = FakeSources(FakeSourceDir(source_root))
+ element = FakeElement("test.bst", sources=sources)
+
+ spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction()
+ spec_action.base_action_digest.CopyFrom(action_digest)
+
+ overlay_a = spec_action.overlays.add()
+ overlay_a.type = speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+ overlay_a.source_element = ""
+ overlay_a.source_path = "a.c"
+ overlay_a.target_digest.CopyFrom(_make_digest(old_a))
+
+ overlay_b = spec_action.overlays.add()
+ overlay_b.type = speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+ overlay_b.source_element = ""
+ overlay_b.source_path = "b.c"
+ overlay_b.target_digest.CopyFrom(_make_digest(old_b))
+
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(spec_action, element, {})
+
+ assert result_digest is not None
+ assert result_digest.hash != action_digest.hash
+
+ # Verify both files were replaced
+ new_action = cas.fetch_action(result_digest)
+ new_root = cas.fetch_directory_proto(new_action.input_root_digest)
+ file_hashes = {f.name: f.digest.hash for f in new_root.files}
+ assert file_hashes["a.c"] == _make_digest(new_a).hash
+ assert file_hashes["b.c"] == _make_digest(new_b).hash
+
+
+class TestInstantiatorArtifactOverlay:
+ """Test ARTIFACT overlay resolution."""
+
+ def test_resolves_artifact_overlay_from_dep(self):
+ """ARTIFACT overlay should resolve file digest from dependency artifact."""
+ from buildstream._speculative_actions.instantiator import SpeculativeActionInstantiator
+
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache()
+
+ old_lib = b'old-lib-content'
+ new_lib = b'new-lib-content'
+ old_digest = _make_digest(old_lib)
+ new_digest = _make_digest(new_lib)
+
+ # Build original action
+ input_root = _build_source_tree(cas, {"lib/libfoo.so": old_lib})
+ action = remote_execution_pb2.Action()
+ action.input_root_digest.CopyFrom(input_root)
+ action_digest = cas.store_action(action)
+
+ # Dependency element with updated artifact
+ dep_artifact_root = _build_source_tree(cas, {"lib/libfoo.so": new_lib})
+ dep_artifact = FakeArtifact(FakeSourceDir(dep_artifact_root))
+ dep_element = FakeElement("dep.bst", artifact=dep_artifact)
+
+ element = FakeElement("test.bst")
+ element_lookup = {"dep.bst": dep_element}
+
+ spec_action = speculative_actions_pb2.SpeculativeActions.SpeculativeAction()
+ spec_action.base_action_digest.CopyFrom(action_digest)
+ overlay = spec_action.overlays.add()
+ overlay.type = speculative_actions_pb2.SpeculativeActions.Overlay.ARTIFACT
+ overlay.source_element = "dep.bst"
+ overlay.source_path = "lib/libfoo.so"
+ overlay.target_digest.CopyFrom(old_digest)
+
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(spec_action, element, element_lookup)
+
+ assert result_digest is not None
+ assert result_digest.hash != action_digest.hash
diff --git a/tests/speculative_actions/test_pipeline_integration.py b/tests/speculative_actions/test_pipeline_integration.py
new file mode 100644
index 000000000..a957b68d6
--- /dev/null
+++ b/tests/speculative_actions/test_pipeline_integration.py
@@ -0,0 +1,771 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Pipeline integration tests for speculative actions.
+
+These tests exercise the full generate → store → retrieve → instantiate
+pipeline using in-memory fakes for CAS and artifact cache. No sandbox
+or real trexe binary needed — subaction digests are constructed directly
+from proto objects.
+
+The scenario modeled:
+ 1. "Build" an element by constructing Action protos with known input trees
+ 2. Run the Generator to produce SpeculativeActions from those subactions
+ 3. Store SpeculativeActions via the artifact cache (weak key path)
+ 4. Simulate a source change (new file content)
+ 5. Retrieve SpeculativeActions and run the Instantiator
+ 6. Verify the instantiated action has the updated file digests
+"""
+
+import hashlib
+import os
+import tempfile
+import pytest
+
+from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildstream._protos.buildstream.v2 import speculative_actions_pb2
+from buildstream._speculative_actions.generator import SpeculativeActionsGenerator
+from buildstream._speculative_actions.instantiator import SpeculativeActionInstantiator
+
+
+# ---------------------------------------------------------------------------
+# Shared test helpers
+# ---------------------------------------------------------------------------
+
+def _make_digest(content):
+ """Create a Digest proto from content bytes."""
+ digest = remote_execution_pb2.Digest()
+ digest.hash = hashlib.sha256(content).hexdigest()
+ digest.size_bytes = len(content)
+ return digest
+
+
+class FakeCAS:
+ """In-memory CAS that supports the operations used by generator and instantiator."""
+
+ def __init__(self):
+ self._blobs = {} # hash -> bytes
+ self._directories = {} # hash -> Directory proto
+ self._actions = {} # hash -> Action proto
+
+ def store_directory_proto(self, directory):
+ data = directory.SerializeToString()
+ digest = _make_digest(data)
+ self._directories[digest.hash] = directory
+ self._blobs[digest.hash] = data
+ return digest
+
+ def fetch_directory_proto(self, digest):
+ return self._directories.get(digest.hash)
+
+ def store_action(self, action):
+ data = action.SerializeToString()
+ digest = _make_digest(data)
+ self._actions[digest.hash] = action
+ self._blobs[digest.hash] = data
+ return digest
+
+ def fetch_action(self, digest):
+ return self._actions.get(digest.hash)
+
+ def store_proto(self, proto):
+ data = proto.SerializeToString()
+ digest = _make_digest(data)
+ self._blobs[digest.hash] = data
+ return digest
+
+ def fetch_proto(self, digest, proto_class):
+ data = self._blobs.get(digest.hash)
+ if data is None:
+ return None
+ proto = proto_class()
+ proto.ParseFromString(data)
+ return proto
+
+
+class FakeSourceDir:
+ def __init__(self, digest):
+ self._digest = digest
+
+ def _get_digest(self):
+ return self._digest
+
+
+class FakeSources:
+ def __init__(self, files_dir):
+ self._files_dir = files_dir
+ self._cached = True
+
+ def cached(self):
+ return self._cached
+
+ def get_files(self):
+ return self._files_dir
+
+
+class FakeArtifactProto:
+ """Minimal artifact proto supporting HasField and speculative_actions."""
+
+ def __init__(self):
+ self._speculative_actions = None
+ self.build_deps = []
+
+ def HasField(self, name):
+ if name == "speculative_actions":
+ return self._speculative_actions is not None
+ return False
+
+ @property
+ def speculative_actions(self):
+ return self._speculative_actions
+
+ @speculative_actions.setter
+ def speculative_actions(self, value):
+ self._speculative_actions = value
+
+
+class FakeArtifact:
+ def __init__(self, files_dir=None, is_cached=True, element=None):
+ self._files_dir = files_dir
+ self._cached = is_cached
+ self._element = element
+
+ def cached(self):
+ return self._cached
+
+ def get_files(self):
+ return self._files_dir
+
+ def _get_proto(self):
+ return None
+
+ def get_extract_key(self):
+ return "extract-key"
+
+
+class FakeProject:
+ def __init__(self, name="test-project"):
+ self.name = name
+
+
+class FakeElement:
+ def __init__(self, name, sources=None, artifact=None, project_name="project"):
+ self.name = name
+ self.project_name = project_name
+ self._Element__sources = sources
+ self._artifact = artifact
+ self._project = FakeProject()
+
+ def sources(self):
+ if self._Element__sources:
+ yield True
+
+ def _cached(self):
+ return self._artifact is not None and self._artifact.cached()
+
+ def _get_artifact(self):
+ return self._artifact
+
+ def _get_project(self):
+ return self._project
+
+ def _dependencies(self, scope, recurse=False):
+ return []
+
+ def _get_cache_key(self):
+ return "fake-cache-key"
+
+ def get_artifact_name(self, key):
+ return "{}/{}/{}".format(self._project.name, self.name, key)
+
+ def info(self, msg):
+ pass
+
+ def warn(self, msg):
+ pass
+
+
+def _build_source_tree(cas, files):
+ """Build a CAS directory tree from {path: content_bytes}, return root Digest."""
+ dirs = {}
+ for path, content in files.items():
+ parts = path.rsplit("/", 1)
+ if len(parts) == 1:
+ dirname, filename = "", parts[0]
+ else:
+ dirname, filename = parts
+ dirs.setdefault(dirname, []).append((filename, content))
+
+ dir_digests = {}
+ all_dirs = set()
+ for path in files:
+ parts = path.split("/")
+ for i in range(len(parts) - 1):
+ all_dirs.add("/".join(parts[: i + 1]))
+ all_dirs.add("")
+
+ non_root = sorted((d for d in all_dirs if d), key=lambda d: -d.count("/"))
+ non_root.append("")
+
+ for dirpath in non_root:
+ directory = remote_execution_pb2.Directory()
+ for filename, content in dirs.get(dirpath, []):
+ file_node = directory.files.add()
+ file_node.name = filename
+ file_node.digest.CopyFrom(_make_digest(content))
+
+ for child_dir, child_digest in sorted(dir_digests.items()):
+ if dirpath == "":
+ if "/" not in child_dir:
+ dir_node = directory.directories.add()
+ dir_node.name = child_dir
+ dir_node.digest.CopyFrom(child_digest)
+ else:
+ prefix = dirpath + "/"
+ if child_dir.startswith(prefix) and "/" not in child_dir[len(prefix):]:
+ dir_node = directory.directories.add()
+ dir_node.name = child_dir[len(prefix):]
+ dir_node.digest.CopyFrom(child_digest)
+
+ digest = cas.store_directory_proto(directory)
+ dir_digests[dirpath] = digest
+
+ return dir_digests[""]
+
+
+def _build_action(cas, input_root_digest):
+ """Build an Action proto with the given input root, store it, return Digest."""
+ action = remote_execution_pb2.Action()
+ action.input_root_digest.CopyFrom(input_root_digest)
+ return cas.store_action(action)
+
+
+class FakeArtifactCache:
+ """Artifact cache backed by a temp directory, using real file paths like the production code."""
+
+ def __init__(self, cas, basedir):
+ self.cas = cas
+ self._basedir = basedir
+
+ def store_speculative_actions(self, artifact, spec_actions, weak_key=None):
+ # Store proto in CAS
+ spec_actions_digest = self.cas.store_proto(spec_actions)
+
+ # Store weak key reference
+ if weak_key:
+ element = artifact._element
+ project = element._get_project()
+ sa_ref = "{}/{}/speculative-{}".format(project.name, element.name, weak_key)
+ sa_ref_path = os.path.join(self._basedir, sa_ref)
+ os.makedirs(os.path.dirname(sa_ref_path), exist_ok=True)
+ with open(sa_ref_path, mode="w+b") as f:
+ f.write(spec_actions.SerializeToString())
+
+ def get_speculative_actions(self, artifact, weak_key=None):
+ if weak_key:
+ element = artifact._element
+ project = element._get_project()
+ sa_ref = "{}/{}/speculative-{}".format(project.name, element.name, weak_key)
+ sa_ref_path = os.path.join(self._basedir, sa_ref)
+ if os.path.exists(sa_ref_path):
+ spec_actions = speculative_actions_pb2.SpeculativeActions()
+ with open(sa_ref_path, mode="r+b") as f:
+ spec_actions.ParseFromString(f.read())
+ return spec_actions
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+
+class TestGenerateStoreRetrieveInstantiate:
+ """Full pipeline: generate overlays, store, retrieve, instantiate with changed sources."""
+
+ def test_source_change_roundtrip(self, tmp_path):
+ """
+ Scenario: element has source file main.c. A build records a subaction
+ whose input tree contains main.c. After the build, we generate and
+ store SpeculativeActions. Later, main.c changes. We retrieve the
+ stored SA and instantiate — the action's input tree should now
+ reference the new main.c digest.
+ """
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache(cas, str(tmp_path))
+
+ # --- Build phase (v1) ---
+ v1_content = b'int main() { return 0; }'
+ v1_digest = _make_digest(v1_content)
+
+ # Element sources contain main.c v1
+ source_root_v1 = _build_source_tree(cas, {"main.c": v1_content})
+ sources_v1 = FakeSources(FakeSourceDir(source_root_v1))
+
+ element = FakeElement("app.bst", sources=sources_v1)
+
+ # The build produced a subaction whose input tree includes main.c
+ subaction_input = _build_source_tree(cas, {"main.c": v1_content})
+ subaction_digest = _build_action(cas, subaction_input)
+
+ # --- Generate phase ---
+ generator = SpeculativeActionsGenerator(cas)
+ spec_actions = generator.generate_speculative_actions(element, [subaction_digest], [])
+
+ assert len(spec_actions.actions) == 1
+ assert len(spec_actions.actions[0].overlays) == 1
+ overlay = spec_actions.actions[0].overlays[0]
+ assert overlay.type == speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE
+ assert overlay.source_path == "main.c"
+ assert overlay.target_digest.hash == v1_digest.hash
+
+ # --- Store phase ---
+ weak_key = "fake-weak-key-v1"
+ artifact = FakeArtifact(element=element)
+ artifactcache.store_speculative_actions(artifact, spec_actions, weak_key=weak_key)
+
+ # --- Source change (v2) ---
+ v2_content = b'int main() { return 42; }'
+ v2_digest = _make_digest(v2_content)
+ source_root_v2 = _build_source_tree(cas, {"main.c": v2_content})
+ sources_v2 = FakeSources(FakeSourceDir(source_root_v2))
+
+ # New element state with updated sources (same weak key because
+ # in real life, the weak key for downstream elements is stable
+ # across dependency version changes — here we're the leaf element
+ # whose source changed, so in practice this SA would be stored
+ # under a *different* weak key. But the retrieve+instantiate
+ # logic is the same.)
+ element_v2 = FakeElement("app.bst", sources=sources_v2)
+ artifact_v2 = FakeArtifact(element=element_v2)
+
+ # --- Retrieve phase ---
+ retrieved = artifactcache.get_speculative_actions(artifact_v2, weak_key=weak_key)
+ assert retrieved is not None
+ assert len(retrieved.actions) == 1
+
+ # --- Instantiate phase ---
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(retrieved.actions[0], element_v2, {})
+
+ assert result_digest is not None
+ # The action should be different (new input digest)
+ assert result_digest.hash != subaction_digest.hash
+
+ # Verify the new action's input tree has main.c with v2 content digest
+ new_action = cas.fetch_action(result_digest)
+ assert new_action is not None
+ new_root = cas.fetch_directory_proto(new_action.input_root_digest)
+ assert new_root is not None
+ assert len(new_root.files) == 1
+ assert new_root.files[0].name == "main.c"
+ assert new_root.files[0].digest.hash == v2_digest.hash
+
+ def test_dependency_artifact_change_roundtrip(self, tmp_path):
+ """
+ Scenario: element depends on dep.bst whose artifact provides libfoo.so.
+ A build records a subaction using both main.c (source) and libfoo.so
+ (from dep). After storing SA, dep.bst is rebuilt with new libfoo.so.
+ Instantiation should produce an action with the new libfoo.so digest.
+ """
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache(cas, str(tmp_path))
+
+ # --- Build phase (v1) ---
+ src_content = b'#include "foo.h"\nint main() { foo(); }'
+ lib_v1 = b'libfoo-v1-content'
+ lib_v1_digest = _make_digest(lib_v1)
+
+ # Element sources
+ source_root = _build_source_tree(cas, {"main.c": src_content})
+ sources = FakeSources(FakeSourceDir(source_root))
+
+ # Dependency artifact with libfoo.so v1
+ dep_artifact_root_v1 = _build_source_tree(cas, {"lib/libfoo.so": lib_v1})
+ dep_artifact_v1 = FakeArtifact(FakeSourceDir(dep_artifact_root_v1))
+ dep_element_v1 = FakeElement("dep.bst", artifact=dep_artifact_v1)
+
+ element = FakeElement("app.bst", sources=sources)
+
+ # Subaction input tree has both source file and dep library
+ subaction_input = _build_source_tree(cas, {
+ "main.c": src_content,
+ "lib/libfoo.so": lib_v1,
+ })
+ subaction_digest = _build_action(cas, subaction_input)
+
+ # --- Generate ---
+ generator = SpeculativeActionsGenerator(cas)
+ spec_actions = generator.generate_speculative_actions(
+ element, [subaction_digest], [dep_element_v1]
+ )
+
+ assert len(spec_actions.actions) == 1
+ overlays = spec_actions.actions[0].overlays
+ overlay_types = {o.type for o in overlays}
+ assert speculative_actions_pb2.SpeculativeActions.Overlay.SOURCE in overlay_types
+ assert speculative_actions_pb2.SpeculativeActions.Overlay.ARTIFACT in overlay_types
+
+ # --- Store ---
+ weak_key = "fake-weak-key-app"
+ artifact = FakeArtifact(element=element)
+ artifactcache.store_speculative_actions(artifact, spec_actions, weak_key=weak_key)
+
+ # --- Dependency change (v2) ---
+ lib_v2 = b'libfoo-v2-content'
+ lib_v2_digest = _make_digest(lib_v2)
+ dep_artifact_root_v2 = _build_source_tree(cas, {"lib/libfoo.so": lib_v2})
+ dep_artifact_v2 = FakeArtifact(FakeSourceDir(dep_artifact_root_v2))
+ dep_element_v2 = FakeElement("dep.bst", artifact=dep_artifact_v2)
+
+ # Element sources unchanged
+ element_v2 = FakeElement("app.bst", sources=sources)
+ artifact_v2 = FakeArtifact(element=element_v2)
+
+ # --- Retrieve ---
+ retrieved = artifactcache.get_speculative_actions(artifact_v2, weak_key=weak_key)
+ assert retrieved is not None
+
+ # --- Instantiate ---
+ element_lookup = {"dep.bst": dep_element_v2}
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(
+ retrieved.actions[0], element_v2, element_lookup
+ )
+
+ assert result_digest is not None
+ assert result_digest.hash != subaction_digest.hash
+
+ # Verify: main.c unchanged, libfoo.so updated to v2
+ new_action = cas.fetch_action(result_digest)
+ new_root = cas.fetch_directory_proto(new_action.input_root_digest)
+
+ # Collect all files recursively
+ all_files = {}
+ self._collect_files(cas, new_root, "", all_files)
+
+ assert all_files["main.c"] == _make_digest(src_content).hash
+ assert all_files["lib/libfoo.so"] == lib_v2_digest.hash
+
+ def test_no_change_returns_base_action(self, tmp_path):
+ """
+ When sources haven't changed between generate and instantiate,
+ the instantiator should return the base action digest unchanged.
+ """
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache(cas, str(tmp_path))
+
+ content = b'unchanged source'
+ source_root = _build_source_tree(cas, {"file.c": content})
+ sources = FakeSources(FakeSourceDir(source_root))
+ element = FakeElement("app.bst", sources=sources)
+
+ subaction_input = _build_source_tree(cas, {"file.c": content})
+ subaction_digest = _build_action(cas, subaction_input)
+
+ # Generate and store
+ generator = SpeculativeActionsGenerator(cas)
+ spec_actions = generator.generate_speculative_actions(element, [subaction_digest], [])
+
+ weak_key = "unchanged-key"
+ artifact = FakeArtifact(element=element)
+ artifactcache.store_speculative_actions(artifact, spec_actions, weak_key=weak_key)
+
+ # Retrieve and instantiate with same sources
+ retrieved = artifactcache.get_speculative_actions(artifact, weak_key=weak_key)
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result_digest = instantiator.instantiate_action(retrieved.actions[0], element, {})
+
+ # Should return the original action digest (no modifications needed)
+ assert result_digest.hash == subaction_digest.hash
+
+ def test_multiple_subactions_roundtrip(self, tmp_path):
+ """
+ Multiple subactions from a single build should each be independently
+ instantiatable after a source change.
+ """
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache(cas, str(tmp_path))
+
+ v1_a = b'void a_v1() {}'
+ v1_b = b'void b_v1() {}'
+
+ source_root = _build_source_tree(cas, {"a.c": v1_a, "b.c": v1_b})
+ sources = FakeSources(FakeSourceDir(source_root))
+ element = FakeElement("app.bst", sources=sources)
+
+ # Two subactions, each using a different source file
+ sub1_input = _build_source_tree(cas, {"a.c": v1_a})
+ sub1_digest = _build_action(cas, sub1_input)
+ sub2_input = _build_source_tree(cas, {"b.c": v1_b})
+ sub2_digest = _build_action(cas, sub2_input)
+
+ generator = SpeculativeActionsGenerator(cas)
+ spec_actions = generator.generate_speculative_actions(
+ element, [sub1_digest, sub2_digest], []
+ )
+ assert len(spec_actions.actions) == 2
+
+ weak_key = "multi-sub"
+ artifact = FakeArtifact(element=element)
+ artifactcache.store_speculative_actions(artifact, spec_actions, weak_key=weak_key)
+
+ # Change both source files
+ v2_a = b'void a_v2() {}'
+ v2_b = b'void b_v2() {}'
+ source_root_v2 = _build_source_tree(cas, {"a.c": v2_a, "b.c": v2_b})
+ sources_v2 = FakeSources(FakeSourceDir(source_root_v2))
+ element_v2 = FakeElement("app.bst", sources=sources_v2)
+ artifact_v2 = FakeArtifact(element=element_v2)
+
+ retrieved = artifactcache.get_speculative_actions(artifact_v2, weak_key=weak_key)
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+
+ # Both actions should be instantiatable
+ for i, spec_action in enumerate(retrieved.actions):
+ result = instantiator.instantiate_action(spec_action, element_v2, {})
+ assert result is not None
+ assert result.hash != [sub1_digest, sub2_digest][i].hash
+
+ new_action = cas.fetch_action(result)
+ new_root = cas.fetch_directory_proto(new_action.input_root_digest)
+ # Each action should have exactly one file with the v2 digest
+ assert len(new_root.files) == 1
+ expected_hash = _make_digest([v2_a, v2_b][i]).hash
+ assert new_root.files[0].digest.hash == expected_hash
+
+ def test_nested_source_tree_roundtrip(self, tmp_path):
+ """
+ Source files in nested directories should be correctly tracked
+ through generate and instantiate.
+ """
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache(cas, str(tmp_path))
+
+ v1 = b'nested file v1'
+ source_root = _build_source_tree(cas, {"src/lib/util.c": v1})
+ sources = FakeSources(FakeSourceDir(source_root))
+ element = FakeElement("app.bst", sources=sources)
+
+ # Subaction has the same nested file
+ sub_input = _build_source_tree(cas, {"src/lib/util.c": v1})
+ sub_digest = _build_action(cas, sub_input)
+
+ generator = SpeculativeActionsGenerator(cas)
+ spec_actions = generator.generate_speculative_actions(element, [sub_digest], [])
+ assert len(spec_actions.actions) == 1
+
+ weak_key = "nested"
+ artifact = FakeArtifact(element=element)
+ artifactcache.store_speculative_actions(artifact, spec_actions, weak_key=weak_key)
+
+ # Change the nested file
+ v2 = b'nested file v2'
+ source_root_v2 = _build_source_tree(cas, {"src/lib/util.c": v2})
+ sources_v2 = FakeSources(FakeSourceDir(source_root_v2))
+ element_v2 = FakeElement("app.bst", sources=sources_v2)
+ artifact_v2 = FakeArtifact(element=element_v2)
+
+ retrieved = artifactcache.get_speculative_actions(artifact_v2, weak_key=weak_key)
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+ result = instantiator.instantiate_action(retrieved.actions[0], element_v2, {})
+
+ assert result is not None
+ assert result.hash != sub_digest.hash
+
+ # Verify nested file was updated
+ new_action = cas.fetch_action(result)
+ all_files = {}
+ self._collect_files(cas, cas.fetch_directory_proto(new_action.input_root_digest), "", all_files)
+ assert all_files["src/lib/util.c"] == _make_digest(v2).hash
+
+ def test_weak_key_isolation(self, tmp_path):
+ """
+ Different weak keys should store and retrieve independent SA sets,
+ modeling how different element configurations get separate SA entries.
+ """
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache(cas, str(tmp_path))
+
+ content_a = b'content for config A'
+ content_b = b'content for config B'
+
+ # Store SA under key A
+ source_root_a = _build_source_tree(cas, {"file.c": content_a})
+ sources_a = FakeSources(FakeSourceDir(source_root_a))
+ element_a = FakeElement("app.bst", sources=sources_a)
+ sub_a = _build_action(cas, _build_source_tree(cas, {"file.c": content_a}))
+
+ generator = SpeculativeActionsGenerator(cas)
+ sa_a = generator.generate_speculative_actions(element_a, [sub_a], [])
+ artifact_a = FakeArtifact(element=element_a)
+ artifactcache.store_speculative_actions(artifact_a, sa_a, weak_key="key-A")
+
+ # Store SA under key B
+ source_root_b = _build_source_tree(cas, {"file.c": content_b})
+ sources_b = FakeSources(FakeSourceDir(source_root_b))
+ element_b = FakeElement("app.bst", sources=sources_b)
+ sub_b = _build_action(cas, _build_source_tree(cas, {"file.c": content_b}))
+
+ sa_b = generator.generate_speculative_actions(element_b, [sub_b], [])
+ artifact_b = FakeArtifact(element=element_b)
+ artifactcache.store_speculative_actions(artifact_b, sa_b, weak_key="key-B")
+
+ # Retrieve each independently
+ ret_a = artifactcache.get_speculative_actions(artifact_a, weak_key="key-A")
+ ret_b = artifactcache.get_speculative_actions(artifact_b, weak_key="key-B")
+
+ assert ret_a is not None
+ assert ret_b is not None
+
+ # They should reference different base actions
+ assert ret_a.actions[0].base_action_digest.hash != ret_b.actions[0].base_action_digest.hash
+
+ # Key A should not return key B's data
+ ret_missing = artifactcache.get_speculative_actions(artifact_a, weak_key="key-nonexistent")
+ assert ret_missing is None
+
+ def test_priming_scenario(self, tmp_path):
+ """
+ Models the priming queue's core scenario:
+
+ 1. Element app.bst depends on dep.bst
+ 2. app.bst is built with dep v1 — subactions recorded, SA generated
+ and stored under app's weak key
+ 3. dep.bst is rebuilt with new content (v2)
+ 4. app.bst needs rebuilding (strict key changed), but its weak key
+ is stable (only dep names, not cache keys)
+ 5. Priming: retrieve SA by weak key, instantiate each action with
+ dep v2's artifact digests, verify the adapted actions have the
+ correct updated digests
+
+ This is the core value of speculative actions: adapting cached
+ build actions to new dependency versions without rebuilding.
+ """
+ cas = FakeCAS()
+ artifactcache = FakeArtifactCache(cas, str(tmp_path))
+
+ # --- Initial build: app depends on dep v1 ---
+ app_src = b'#include "dep.h"\nint main() { return dep(); }'
+ dep_header_v1 = b'int dep(void); /* v1 */'
+ dep_lib_v1 = b'dep-object-code-v1'
+
+ app_source_root = _build_source_tree(cas, {"main.c": app_src})
+ app_sources = FakeSources(FakeSourceDir(app_source_root))
+
+ dep_artifact_root_v1 = _build_source_tree(cas, {
+ "include/dep.h": dep_header_v1,
+ "lib/libdep.o": dep_lib_v1,
+ })
+ dep_artifact_v1 = FakeArtifact(FakeSourceDir(dep_artifact_root_v1))
+ dep_element_v1 = FakeElement("dep.bst", artifact=dep_artifact_v1)
+
+ app_element = FakeElement("app.bst", sources=app_sources)
+
+ # Subactions from app's build: compile (uses main.c + dep.h) and
+ # link (uses main.o + libdep.o)
+ compile_input = _build_source_tree(cas, {
+ "main.c": app_src,
+ "include/dep.h": dep_header_v1,
+ })
+ compile_action = _build_action(cas, compile_input)
+
+ link_input = _build_source_tree(cas, {
+ "main.o": b'app-object-code',
+ "lib/libdep.o": dep_lib_v1,
+ })
+ link_action = _build_action(cas, link_input)
+
+ # Generate SA from both subactions
+ generator = SpeculativeActionsGenerator(cas)
+ spec_actions = generator.generate_speculative_actions(
+ app_element, [compile_action, link_action], [dep_element_v1]
+ )
+
+ assert len(spec_actions.actions) == 2, (
+ f"Expected 2 speculative actions (compile + link), got {len(spec_actions.actions)}"
+ )
+
+ # Store under app's weak key
+ weak_key = "app-weak-key"
+ app_artifact = FakeArtifact(element=app_element)
+ artifactcache.store_speculative_actions(
+ app_artifact, spec_actions, weak_key=weak_key
+ )
+
+ # --- dep.bst rebuilt with v2 ---
+ dep_header_v2 = b'int dep(void); /* v2 - added feature */'
+ dep_lib_v2 = b'dep-object-code-v2'
+
+ dep_artifact_root_v2 = _build_source_tree(cas, {
+ "include/dep.h": dep_header_v2,
+ "lib/libdep.o": dep_lib_v2,
+ })
+ dep_artifact_v2 = FakeArtifact(FakeSourceDir(dep_artifact_root_v2))
+ dep_element_v2 = FakeElement("dep.bst", artifact=dep_artifact_v2)
+
+ # app's sources unchanged, weak key stable
+ app_element_v2 = FakeElement("app.bst", sources=app_sources)
+ app_artifact_v2 = FakeArtifact(element=app_element_v2)
+
+ # --- Priming: retrieve and instantiate ---
+ retrieved = artifactcache.get_speculative_actions(
+ app_artifact_v2, weak_key=weak_key
+ )
+ assert retrieved is not None
+ assert len(retrieved.actions) == 2
+
+ element_lookup = {"dep.bst": dep_element_v2}
+ instantiator = SpeculativeActionInstantiator(cas, artifactcache)
+
+ adapted_actions = []
+ for spec_action in retrieved.actions:
+ result = instantiator.instantiate_action(
+ spec_action, app_element_v2, element_lookup
+ )
+ assert result is not None
+ adapted_actions.append(result)
+
+ # Verify compile action: main.c unchanged, dep.h updated to v2
+ compile_result = cas.fetch_action(adapted_actions[0])
+ compile_files = {}
+ self._collect_files(
+ cas,
+ cas.fetch_directory_proto(compile_result.input_root_digest),
+ "", compile_files,
+ )
+ assert compile_files["main.c"] == _make_digest(app_src).hash
+ assert compile_files["include/dep.h"] == _make_digest(dep_header_v2).hash
+
+ # Verify link action: libdep.o updated to v2
+ link_result = cas.fetch_action(adapted_actions[1])
+ link_files = {}
+ self._collect_files(
+ cas,
+ cas.fetch_directory_proto(link_result.input_root_digest),
+ "", link_files,
+ )
+ assert link_files["lib/libdep.o"] == _make_digest(dep_lib_v2).hash
+
+ @staticmethod
+ def _collect_files(cas, directory, prefix, result):
+ """Recursively collect {path: digest_hash} from a Directory proto."""
+ if directory is None:
+ return
+ for f in directory.files:
+ path = f.name if not prefix else "{}/{}".format(prefix, f.name)
+ result[path] = f.digest.hash
+ for d in directory.directories:
+ subpath = d.name if not prefix else "{}/{}".format(prefix, d.name)
+ subdir = cas.fetch_directory_proto(d.digest)
+ TestGenerateStoreRetrieveInstantiate._collect_files(cas, subdir, subpath, result)
diff --git a/tests/speculative_actions/test_weak_key.py b/tests/speculative_actions/test_weak_key.py
new file mode 100644
index 000000000..25672d791
--- /dev/null
+++ b/tests/speculative_actions/test_weak_key.py
@@ -0,0 +1,211 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Tests for the speculative actions weak key lookup.
+
+The weak cache key is used for speculative actions lookup because it is:
+- Stable across dependency version changes (only dep names, not cache keys)
+- Changing when the element's own sources change
+- Changing when build commands change
+- Changing when environment changes
+- Changing when sandbox config changes
+
+This mirrors Element._calculate_cache_key() with weak-mode dependencies
+(only [project_name, name] per dependency).
+"""
+
+import pytest
+from buildstream._cachekey import generate_key
+
+
+# These helpers mirror the structure of Element._calculate_cache_key() to
+# verify the properties of the weak key as used for speculative actions.
+# The actual weak key is computed by Element.__update_cache_keys() using
+# _calculate_cache_key(dependencies) where dependencies are [project, name]
+# pairs (in non-strict mode).
+
+def _make_weak_key_dict(
+ plugin_name="autotools",
+ plugin_key=None,
+ sources_key="abc123",
+ dep_names=None,
+ sandbox=None,
+ environment=None,
+ public=None,
+):
+ """Helper to construct a dict that mirrors the weak cache key inputs.
+
+ This doesn't replicate _calculate_cache_key exactly, but captures the
+ same structural properties for testing key stability/invalidation.
+ """
+ if plugin_key is None:
+ plugin_key = {
+ "build-commands": ["make"],
+ "install-commands": ["make install"],
+ }
+ if dep_names is None:
+ dep_names = [["project", "base.bst"], ["project", "dep-a.bst"]]
+ if environment is None:
+ environment = {"PATH": "/usr/bin"}
+ if public is None:
+ public = {}
+
+ cache_key_dict = {
+ "core-artifact-version": 1,
+ "element-plugin-key": plugin_key,
+ "element-plugin-name": plugin_name,
+ "element-plugin-version": 0,
+ "sources": sources_key,
+ "public": public,
+ "fatal-warnings": [],
+ }
+ if sandbox is not None:
+ cache_key_dict["sandbox"] = sandbox
+ cache_key_dict["environment"] = environment
+
+ # Weak dependencies: only [project, name] pairs (no cache keys)
+ cache_key_dict["dependencies"] = sorted(dep_names)
+
+ return cache_key_dict
+
+
+class TestWeakKeyStability:
+ """Verify key stability: same inputs produce same key."""
+
+ def test_same_inputs_same_key(self):
+ """Identical inputs must produce the same key."""
+ dict1 = _make_weak_key_dict()
+ dict2 = _make_weak_key_dict()
+ assert generate_key(dict1) == generate_key(dict2)
+
+ def test_stable_across_dependency_version_changes(self):
+ """Key uses dependency names only, not their cache keys.
+
+ When a dependency is rebuilt with different content, the weak key
+ remains stable because it only records [project, name] pairs.
+ """
+ # Same dep names → same key, regardless of what version was built
+ dict1 = _make_weak_key_dict(dep_names=[["proj", "dep.bst"]])
+ dict2 = _make_weak_key_dict(dep_names=[["proj", "dep.bst"]])
+ assert generate_key(dict1) == generate_key(dict2)
+
+ def test_dependency_order_irrelevant(self):
+ """Dependency names are sorted, so ordering doesn't matter."""
+ dict1 = _make_weak_key_dict(dep_names=[["proj", "a.bst"], ["proj", "b.bst"]])
+ dict2 = _make_weak_key_dict(dep_names=[["proj", "b.bst"], ["proj", "a.bst"]])
+ assert generate_key(dict1) == generate_key(dict2)
+
+
+class TestWeakKeyInvalidation:
+ """Verify key changes when element configuration changes."""
+
+ def test_changes_when_source_changes(self):
+ """Different source content must produce a different key.
+
+ Unlike the old structural key, the weak key includes source
+ digests, so changing source code correctly invalidates it.
+ """
+ key1 = generate_key(_make_weak_key_dict(sources_key="source-v1"))
+ key2 = generate_key(_make_weak_key_dict(sources_key="source-v2"))
+ assert key1 != key2
+
+ def test_changes_when_build_commands_change(self):
+ """Different build commands must produce a different key."""
+ key1 = generate_key(
+ _make_weak_key_dict(plugin_key={"build-commands": ["make"]})
+ )
+ key2 = generate_key(
+ _make_weak_key_dict(plugin_key={"build-commands": ["cmake --build ."]})
+ )
+ assert key1 != key2
+
+ def test_changes_when_install_commands_change(self):
+ """Different install commands must produce a different key."""
+ key1 = generate_key(
+ _make_weak_key_dict(plugin_key={"install-commands": ["make install"]})
+ )
+ key2 = generate_key(
+ _make_weak_key_dict(plugin_key={"install-commands": ["make install DESTDIR=/foo"]})
+ )
+ assert key1 != key2
+
+ def test_changes_when_dependency_names_change(self):
+ """Adding a dependency must change the key."""
+ key1 = generate_key(
+ _make_weak_key_dict(dep_names=[["proj", "base.bst"]])
+ )
+ key2 = generate_key(
+ _make_weak_key_dict(dep_names=[["proj", "base.bst"], ["proj", "extra.bst"]])
+ )
+ assert key1 != key2
+
+ def test_changes_when_dependency_removed(self):
+ """Removing a dependency must change the key."""
+ key1 = generate_key(
+ _make_weak_key_dict(dep_names=[["proj", "base.bst"], ["proj", "dep.bst"]])
+ )
+ key2 = generate_key(
+ _make_weak_key_dict(dep_names=[["proj", "base.bst"]])
+ )
+ assert key1 != key2
+
+ def test_changes_when_plugin_name_changes(self):
+ """Different plugin type must produce a different key."""
+ key1 = generate_key(_make_weak_key_dict(plugin_name="autotools"))
+ key2 = generate_key(_make_weak_key_dict(plugin_name="cmake"))
+ assert key1 != key2
+
+ def test_changes_when_sandbox_config_changes(self):
+ """Different sandbox configuration must change the key."""
+ key1 = generate_key(
+ _make_weak_key_dict(sandbox={"build-os": "linux", "build-arch": "x86_64"})
+ )
+ key2 = generate_key(
+ _make_weak_key_dict(sandbox={"build-os": "linux", "build-arch": "aarch64"})
+ )
+ assert key1 != key2
+
+ def test_changes_when_environment_changes(self):
+ """Different environment must change the key."""
+ key1 = generate_key(
+ _make_weak_key_dict(
+ sandbox={"build-os": "linux"},
+ environment={"PATH": "/usr/bin"},
+ )
+ )
+ key2 = generate_key(
+ _make_weak_key_dict(
+ sandbox={"build-os": "linux"},
+ environment={"PATH": "/usr/bin", "CC": "gcc"},
+ )
+ )
+ assert key1 != key2
+
+ def test_no_sandbox_vs_sandbox(self):
+ """Having sandbox config vs not having it must change the key."""
+ key1 = generate_key(_make_weak_key_dict(sandbox=None))
+ key2 = generate_key(
+ _make_weak_key_dict(sandbox={"build-os": "linux"})
+ )
+ assert key1 != key2
+
+
+class TestWeakKeyFormat:
+ """Verify key format properties."""
+
+ def test_key_is_hex_digest(self):
+ """Key should be a valid sha256 hex digest."""
+ key = generate_key(_make_weak_key_dict())
+ assert len(key) == 64
+ assert all(c in "0123456789abcdef" for c in key)