You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/01/22 08:49:45 UTC
[buildstream] branch tristan/change-remote-config updated:
plugins/elements/junction.py: Remove ambiguous cache related junction
parameters
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/change-remote-config
in repository https://gitbox.apache.org/repos/asf/buildstream.git
The following commit(s) were added to refs/heads/tristan/change-remote-config by this push:
new 021a78a plugins/elements/junction.py: Remove ambiguous cache related junction parameters
021a78a is described below
commit 021a78a502c79fdfe44416f689e192968731874e
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Fri Jan 22 17:41:32 2021 +0900
plugins/elements/junction.py: Remove ambiguous cache related junction parameters
This removes the "cache-junction-elements" and "ignore-junction-remotes"
parameters from the junction YAML configuration.
As referenced by the mailing list thread[0], these parameters do not belong
in element configuration, fine grained configuration of these parameters
are not needed in the context of a pipeline with many subprojects in play,
and the documentation and expected behavior with deep and complex subproject
hierarches is unclear, especially when considering the ability to override
junction configurations. Further, these currently cannot even be overridden
by user configuration.
This will be replaced by a simpler configuration in the regular way, with
recommendations (not hard decisions) in project.conf and the final decision
made by user configuration files.
Summary of changes:
* plugins/elements/junction.py: Removed the configuration and docs
* _project.py: Stop observing these configurations
* tests/artifactcache/junctions.py: Stop testing these configurations
[0]: https://lists.apache.org/thread.html/rf2da9830e2fa918357f99a6021e55fc43df876f0b19d43f68802f083%40%3Cdev.buildstream.apache.org%3E
---
src/buildstream/_project.py | 10 --
src/buildstream/plugins/elements/junction.py | 13 +--
tests/artifactcache/junctions.py | 157 ---------------------------
3 files changed, 1 insertion(+), 179 deletions(-)
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index f1d1fa6..6dbc120 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -867,16 +867,6 @@ class Project:
# Load artifacts pull/push configuration for this project
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
- # If there is a junction Element which specifies that we want to remotely cache
- # its elements, append the junction's remotes to the artifact cache specs list
- if self.junction:
- parent = self.junction._get_project()
- if self.junction.ignore_junction_remotes:
- self.artifact_cache_specs = []
-
- if self.junction.cache_junction_elements:
- self.artifact_cache_specs = parent.artifact_cache_specs + self.artifact_cache_specs
-
# Load source caches with pull/push config
self.source_cache_specs = SourceCache.specs_from_config_node(config, self.directory)
diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py
index cc980be..a8e3f22 100644
--- a/src/buildstream/plugins/elements/junction.py
+++ b/src/buildstream/plugins/elements/junction.py
@@ -53,15 +53,6 @@ Overview
overrides:
subproject-junction.bst: local-junction.bst
- # Optionally declare whether elements within the junction project
- # should interact with project remotes (default: False).
- cache-junction-elements: False
-
- # Optionally ignore junction remotes, this means that BuildStream
- # will not attempt to pull artifacts from the junction project's
- # remote(s) (default: False).
- ignore-junction-remotes: False
-
With a junction element in place, local elements can depend on elements in
the other BuildStream project using :ref:`element paths <format_element_names>`.
For example, if you have a ``toolchain.bst`` junction element referring to
@@ -350,12 +341,10 @@ class JunctionElement(Element):
def configure(self, node):
- node.validate_keys(["path", "options", "cache-junction-elements", "ignore-junction-remotes", "overrides"])
+ node.validate_keys(["path", "options", "overrides"])
self.path = node.get_str("path", default="")
self.options = node.get_mapping("options", default={})
- self.cache_junction_elements = node.get_bool("cache-junction-elements", default=False)
- self.ignore_junction_remotes = node.get_bool("ignore-junction-remotes", default=False)
# The overrides dictionary has the target junction
# to override as a key, and the ScalarNode of the
diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py
index 18c48b4..e1b7dbf 100644
--- a/tests/artifactcache/junctions.py
+++ b/tests/artifactcache/junctions.py
@@ -82,160 +82,3 @@ def test_push_pull(cli, tmpdir, datafiles):
assert state == "cached"
state = cli.get_element_state(base_project, "base-element.bst")
assert state == "cached"
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_caching_junction_elements(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), "parent")
- base_project = os.path.join(str(project), "base")
-
- # Load the junction element
- junction_element = os.path.join(project, "base.bst")
- junction_data = _yaml.roundtrip_load(junction_element)
-
- # Add the "cache-junction-elements" boolean to the junction Element
- junction_data["config"] = {"cache-junction-elements": True}
- _yaml.roundtrip_dump(junction_data, junction_element)
-
- with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share(
- os.path.join(str(tmpdir), "artifactshare-base")
- ) as base_share:
-
- # First build it without the artifact cache configured
- result = cli.run(project=project, args=["build", "target.bst"])
- assert result.exit_code == 0
-
- # Assert that we are now cached locally
- state = cli.get_element_state(project, "target.bst")
- assert state == "cached"
- state = cli.get_element_state(base_project, "base-element.bst")
- assert state == "cached"
-
- project_set_artifacts(project, share.repo)
- project_set_artifacts(base_project, base_share.repo)
-
- # Now try bst artifact push
- result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"])
- assert result.exit_code == 0
-
- # And finally assert that the artifacts are in the right shares
- #
- # The parent project's cache should *also* contain elements from the junction
- assert_shared(cli, share, project, "target.bst", project_name="parent")
- assert_shared(cli, share, project, "app.bst", project_name="parent")
- assert_shared(cli, share, base_project, "base-element.bst", project_name="base")
-
- # The junction project's cache should only contain elements in the junction project
- assert_not_shared(cli, base_share, project, "target.bst", project_name="parent")
- assert_not_shared(cli, base_share, project, "app.bst", project_name="parent")
- assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base")
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_ignore_junction_remotes(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), "parent")
- base_project = os.path.join(str(project), "base")
-
- # Load the junction element
- junction_element = os.path.join(project, "base.bst")
- junction_data = _yaml.roundtrip_load(junction_element)
-
- with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share(
- os.path.join(str(tmpdir), "artifactshare-base")
- ) as base_share:
-
- # Immediately declare the artifact caches in the appropriate project configs
- project_set_artifacts(project, share.repo)
- project_set_artifacts(base_project, base_share.repo)
-
- # Build and populate the project remotes with their respective elements
- result = cli.run(project=project, args=["build", "target.bst"])
- assert result.exit_code == 0
-
- # And finally assert that the artifacts are in the right shares
- #
- # The parent project's cache should only contain project elements
- assert_shared(cli, share, project, "target.bst", project_name="parent")
- assert_shared(cli, share, project, "app.bst", project_name="parent")
- assert_not_shared(cli, share, base_project, "base-element.bst", project_name="base")
-
- # The junction project's cache should only contain elements in the junction project
- assert_not_shared(cli, base_share, project, "target.bst", project_name="parent")
- assert_not_shared(cli, base_share, project, "app.bst", project_name="parent")
- assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base")
-
- # Ensure that, from now on, we ignore junction element remotes
- junction_data["config"] = {"ignore-junction-remotes": True}
- _yaml.roundtrip_dump(junction_data, junction_element)
-
- # Now delete everything from the local cache and try to
- # redownload from the shares.
- #
- cas = os.path.join(cli.directory, "cas")
- shutil.rmtree(cas)
- artifact_dir = os.path.join(cli.directory, "artifacts")
- shutil.rmtree(artifact_dir)
-
- # Assert that nothing is cached locally anymore
- state = cli.get_element_state(project, "target.bst")
- assert state != "cached"
- state = cli.get_element_state(base_project, "base-element.bst")
- assert state != "cached"
-
- # Now try bst artifact pull
- result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"])
- assert result.exit_code == 0
-
- # And assert that they are again in the local cache, without having built
- state = cli.get_element_state(project, "target.bst")
- assert state == "cached"
- # We shouldn't be able to download base-element!
- state = cli.get_element_state(base_project, "base-element.bst")
- assert state != "cached"
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_caching_elements_ignoring_remotes(cli, tmpdir, datafiles):
- project = os.path.join(str(datafiles), "parent")
- base_project = os.path.join(str(project), "base")
-
- # Load the junction element
- junction_element = os.path.join(project, "base.bst")
- junction_data = _yaml.roundtrip_load(junction_element)
-
- # Configure to push everything to the project's remote and nothing to the junction's
- junction_data["config"] = {"cache-junction-elements": True, "ignore-junction-remotes": True}
- _yaml.roundtrip_dump(junction_data, junction_element)
-
- with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share(
- os.path.join(str(tmpdir), "artifactshare-base")
- ) as base_share:
-
- # First build it without the artifact cache configured
- result = cli.run(project=project, args=["build", "target.bst"])
- assert result.exit_code == 0
-
- # Assert that we are now cached locally
- state = cli.get_element_state(project, "target.bst")
- assert state == "cached"
- state = cli.get_element_state(base_project, "base-element.bst")
- assert state == "cached"
-
- project_set_artifacts(project, share.repo)
- project_set_artifacts(base_project, base_share.repo)
-
- # Push to the remote(s))
- result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"])
- assert result.exit_code == 0
-
- # And finally assert that the artifacts are in the right shares
- #
- # The parent project's cache should *also* contain elements from the junction
- assert_shared(cli, share, project, "target.bst", project_name="parent")
- assert_shared(cli, share, project, "app.bst", project_name="parent")
- assert_shared(cli, share, base_project, "base-element.bst", project_name="base")
-
- # The junction project's cache should be empty
- assert_not_shared(cli, base_share, project, "target.bst", project_name="parent")
- assert_not_shared(cli, base_share, project, "app.bst", project_name="parent")
- assert_not_shared(cli, base_share, base_project, "base-element.bst", project_name="base")