You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:51:55 UTC
[buildstream] branch tristan/shell-artifacts created (now 9ab6073)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 9ab6073 tests/integration/shellbuildtrees.py: Refactor test cases
This branch includes the following new commits:
new 39e219e doc/source/format_declaring.rst: Minor corrections and added references.
new d33b29f doc: Documenting "artifact names".
new 1cb5d69 plugins/elements/stack.py: Require all dependencies be build & run.
new 43b6b2b NEWS: Document breaking change, stack element dependencies must be build + run
new 35639c7 tests/format/stack.py: Test stack dependency types
new 5c08f31 _stream.py: Added missing machine readable error codes
new a8e6f32 _frontend/cli.py: Propagate machine readable error codes in `bst shell`
new 9ab6073 tests/integration/shellbuildtrees.py: Refactor test cases
The 8 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 08/08: tests/integration/shellbuildtrees.py: Refactor
test cases
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9ab607318c7deb070cb9413847674ca9e9851f12
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Dec 9 18:00:40 2020 +0900
tests/integration/shellbuildtrees.py: Refactor test cases
These tests were quite hard to follow what they were testing, and
did not cover the error cases thoroughly either. Instead, use some
test parametrization to implement more succinct tests which cover
the API surface more thoroughly.
---
tests/integration/shellbuildtrees.py | 304 ++++++++++++++---------------------
1 file changed, 125 insertions(+), 179 deletions(-)
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 5164c02..082e7e6 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -19,9 +19,12 @@ pytestmark = pytest.mark.integration
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
+#
+# Ensure that we didn't get a build tree if we didn't ask for one
+#
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_staged(cli_integration, datafiles):
+def test_buildtree_unused(cli_integration, datafiles):
# We can only test the non interacitve case
# The non interactive case defaults to not using buildtrees
# for `bst shell --build`
@@ -35,9 +38,12 @@ def test_buildtree_staged(cli_integration, datafiles):
res.assert_shell_error()
+#
+# Ensure we can use a buildtree from a successful build
+#
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_staged_forced_true(cli_integration, datafiles):
+def test_buildtree_from_success(cli_integration, datafiles):
# Test that if we ask for a build tree it is there.
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
@@ -52,27 +58,9 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
assert "Hi" in res.output
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
- # Test that if we stage a cached and empty buildtree, we warn the user.
- project = str(datafiles)
- element_name = "build-shell/buildtree.bst"
-
- # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
- # without caching a buildtree which is the default bst behaviour
- cli_integration.configure({"cachedir": str(tmpdir)})
-
- res = cli_integration.run(project=project, args=["build", element_name])
- res.assert_success()
-
- res = cli_integration.run(
- project=project, args=["shell", "--build", "--use-buildtree", element_name, "--", "cat", "test"]
- )
- res.assert_main_error(ErrorDomain.APP, None)
- assert "Error launching shell: Artifact was created without buildtree" in res.stderr
-
-
+#
+# Ensure we can use a buildtree from a failed build
+#
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
def test_buildtree_from_failure(cli_integration, datafiles):
@@ -92,198 +80,151 @@ def test_buildtree_from_failure(cli_integration, datafiles):
assert "Hi" in res.output
+#
+# Test behavior of launching a shell and requesting to use a buildtree, with
+# various states of local cache (ranging from nothing cached to everything cached)
+#
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles):
-
- project = str(datafiles)
- element_name = "build-shell/buildtree-fail.bst"
-
- # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
- # without caching a buildtree explicitly
- cli_integration.configure({"cachedir": str(tmpdir)})
-
- res = cli_integration.run(project=project, args=["--cache-buildtrees", "never", "build", element_name])
- res.assert_main_error(ErrorDomain.STREAM, None)
-
- res = cli_integration.run(
- project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]
- )
- res.assert_main_error(ErrorDomain.APP, None)
- assert "Error launching shell: Artifact was created without buildtree" in res.stderr
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles):
-
- project = str(datafiles)
- element_name = "build-shell/buildtree-fail.bst"
-
- # build with --cache-buildtrees set to 'always', behaviour should match
- # default behaviour (which is always) as the buildtree will explicitly have been
- # cached with content.
- cli_integration.configure({"cachedir": str(tmpdir)})
-
- res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
- res.assert_main_error(ErrorDomain.STREAM, None)
-
- res = cli_integration.run(
- project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]
- )
- res.assert_success()
- assert "WARNING using a buildtree from a failed build" in res.stderr
- assert "Hi" in res.output
-
-
-# Check that build shells work when pulled from a remote cache
-# This is to roughly simulate remote execution
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_pulled(cli, tmpdir, datafiles):
+@pytest.mark.parametrize(
+ "pull,pull_deps,pull_buildtree,cache_buildtree",
+ [
+ # Don't pull at all
+ (False, "build", False, False),
+ # Pull only dependencies
+ (True, "build", False, True),
+ # Pull all elements including the shell element, but without the buildtree
+ (True, "all", False, True),
+ # Pull all elements including the shell element, and pull buildtrees
+ (True, "all", True, True),
+ # Pull all elements including the shell element, and pull buildtrees, but buildtree was never cached
+ (True, "all", True, False),
+ ],
+ ids=["no-pull", "pull-only-deps", "pull-without-buildtree", "pull-with-buildtree", "created-without-buildtree"],
+)
+def test_shell_use_pulled_buildtree(cli, tmpdir, datafiles, pull, pull_deps, pull_buildtree, cache_buildtree):
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Build the element to push it to cache
cli.configure({"artifacts": {"url": share.repo, "push": True}})
- result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
- result.assert_success()
- assert cli.get_element_state(project, element_name) == "cached"
-
- # Discard the cache
- shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
- shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
- assert cli.get_element_state(project, element_name) != "cached"
- # Pull from cache, ensuring cli options is set to pull the buildtree
- result = cli.run(
- project=project, args=["--pull-buildtrees", "artifact", "pull", "--deps", "all", element_name]
- )
+ # Build it, optionally caching the build tree
+ args = []
+ if cache_buildtree:
+ args += ["--cache-buildtrees", "always"]
+ args += ["--on-error", "continue", "build", element_name]
+ result = cli.run(project=project, args=args)
result.assert_success()
- # Check it's using the cached build tree
- res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"])
- res.assert_success()
-
-
-# This test checks for correct behaviour if a buildtree is not present in the local cache.
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_buildtree_options(cli, tmpdir, datafiles):
- project = str(datafiles)
- element_name = "build-shell/buildtree.bst"
-
- with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
- # Build the element to push it to cache
- cli.configure({"artifacts": {"url": share.repo, "push": True}})
- result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
- result.assert_success()
assert cli.get_element_state(project, element_name) == "cached"
assert share.get_artifact(cli.get_artifact_name(project, "test", element_name))
- # Discard the cache
+ # Discard the local cache
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
assert cli.get_element_state(project, element_name) != "cached"
- # Pull from cache, but do not include buildtrees.
- result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name])
- result.assert_success()
+ # Optionally pull the buildtree along with `bst artifact pull`
+ if pull:
+ args = []
+ if pull_buildtree:
+ args += ["--pull-buildtrees"]
+ args += ["artifact", "pull", "--deps", pull_deps, element_name]
- # Check it's not using the cached build tree
- res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"])
- res.assert_shell_error()
- assert "Hi" not in res.output
-
- # Check it's not using the cached build tree, default is to ask, and fall back to not
- # for non interactive behavior
- res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"])
- res.assert_shell_error()
- assert "Hi" not in res.output
-
- # Check correctly handling the lack of buildtree, with '--use-buildtree' attempting and succeeding
- # to pull the buildtree as the user context allow the pulling of buildtrees and it is
- # available in the remote and --pull given
- res = cli.run(
- project=project,
- args=[
- "--pull-buildtrees",
- "shell",
- "--build",
- element_name,
- "--pull",
- "--use-buildtree",
- "--",
- "cat",
- "test",
- ],
- )
- assert "Hi" in res.output
- shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "cas")))
- shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "artifacts")))
- assert cli.get_element_state(project, element_name) != "cached"
+ # Pull from cache
+ result = cli.run(project=project, args=args)
+ result.assert_success()
- # Check it's not loading the shell at all with `--use-buildtree`, when the
- # user context does not allow for buildtree pulling and --pull is not given
- result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name])
- result.assert_success()
- res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"])
- res.assert_main_error(ErrorDomain.APP, None)
- assert "Buildtree is not cached locally" in res.stderr
- assert "Hi" not in res.output
-
- # Check that when user context is set to pull buildtrees and a remote has the buildtree,
- # '--use-buildtree' will attempt and succeed at pulling the missing buildtree with --pull set.
- res = cli.run(
- project=project,
- args=[
- "--pull-buildtrees",
- "shell",
- "--build",
- element_name,
- "--pull",
- "--use-buildtree",
- "--",
- "cat",
- "test",
- ],
+ # Run the shell without asking it to pull any buildtree, just asking to use a buildtree
+ result = cli.run(
+ project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]
)
- assert "Hi" in res.output
- assert res.get_pulled_elements() == [element_name]
-
-# Tests running pull and pull-buildtree options at the same time.
+ # If we did pull the buildtree, expect success, otherwise fail
+ if pull:
+ if pull_deps == "all":
+ if pull_buildtree:
+
+ if cache_buildtree:
+ result.assert_success()
+ assert "Hi" in result.output
+ else:
+ # Sorry, a buildtree was never cached for this element
+ result.assert_main_error(
+ ErrorDomain.APP, "missing-buildtree-artifact-created-without-buildtree"
+ )
+ else:
+ # We just didn't pull the buildtree
+ result.assert_main_error(ErrorDomain.APP, "missing-buildtree-artifact-buildtree-not-cached")
+ else:
+ # The artifact we're shelling into is missing
+ result.assert_main_error(ErrorDomain.APP, "missing-buildtree-artifact-not-cached")
+ else:
+ # The dependencies are missing, cannot stage anything even
+ result.assert_main_error(ErrorDomain.APP, "shell-missing-deps")
+
+
+#
+# Test behavior of launching a shell and requesting to use and pull a buildtree, with
+# various states of local cache (ranging from nothing cached to everything cached)
+#
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
+@pytest.mark.parametrize(
+ "pull,pull_deps,pull_buildtree,cache_buildtree",
+ [
+ # Don't pull at all
+ (False, "build", False, True),
+ # Pull only dependencies
+ (True, "build", False, True),
+ # Pull all elements including the shell element, but without the buildtree
+ (True, "all", False, True),
+ # Pull all elements including the shell element, and pull buildtrees
+ (True, "all", True, True),
+ # Pull all elements including the shell element, and pull buildtrees, but buildtree was never cached
+ (True, "all", True, False),
+ ],
+ ids=["no-pull", "pull-only-deps", "pull-without-buildtree", "pull-with-buildtree", "created-without-buildtree"],
+)
+def test_shell_pull_buildtree(cli, tmpdir, datafiles, pull, pull_deps, pull_buildtree, cache_buildtree):
project = str(datafiles)
element_name = "build-shell/buildtree.bst"
with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
# Build the element to push it to cache
cli.configure({"artifacts": {"url": share.repo, "push": True}})
- result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
+
+ # Build it, optionally caching the build tree
+ args = []
+ if cache_buildtree:
+ args += ["--cache-buildtrees", "always"]
+ args += ["build", element_name]
+ result = cli.run(project=project, args=args)
result.assert_success()
+
assert cli.get_element_state(project, element_name) == "cached"
+ assert share.get_artifact(cli.get_artifact_name(project, "test", element_name))
- # Discard the cache
+ # Discard the local cache
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas")))
shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts")))
assert cli.get_element_state(project, element_name) != "cached"
- # Check it's not using the cached build tree, because --pull
- # and pull-buildtrees were not both set
- res = cli.run(
- project=project,
- args=["shell", "--build", element_name, "--pull", "--use-buildtree", "--", "cat", "test",],
- )
- res.assert_main_error(ErrorDomain.APP, None)
- assert "Buildtree is not cached locally" in res.stderr
+ # Optionally pull the buildtree along with `bst artifact pull`
+ if pull:
+ args = []
+ if pull_buildtree:
+ args += ["--pull-buildtrees"]
+ args += ["artifact", "pull", "--deps", pull_deps, element_name]
+
+ # Pull from cache
+ result = cli.run(project=project, args=args)
+ result.assert_success()
- # Check it's using the cached build tree, because --pull
- # and pull-buildtrees were both set
- res = cli.run(
+ # Run the shell without asking it to pull any buildtree, just asking to use a buildtree
+ result = cli.run(
project=project,
args=[
"--pull-buildtrees",
@@ -297,5 +238,10 @@ def test_pull_buildtree_pulled(cli, tmpdir, datafiles):
"test",
],
)
- result.assert_success()
- assert "Hi" in res.output
+
+ if cache_buildtree:
+ result.assert_success()
+ assert "Hi" in result.output
+ else:
+ # Sorry, a buildtree was never cached for this element
+ result.assert_main_error(ErrorDomain.APP, "missing-buildtree-artifact-created-without-buildtree")
[buildstream] 07/08: _frontend/cli.py: Propagate machine readable
error codes in `bst shell`
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a8e6f32d52230eb2a5e4cfe890583f2763f99498
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Dec 9 16:05:32 2020 +0900
_frontend/cli.py: Propagate machine readable error codes in `bst shell`
This command prefixes a reported error, so it rewraps the error into
an AppError, this needs to propagate the originating machine readable
error.
---
src/buildstream/_frontend/cli.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 5e42bda..ab06e8a 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -640,7 +640,7 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command):
pull_=pull_,
)
except BstError as e:
- raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e
+ raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e
# If there were no errors, we return the shell's exit code here.
sys.exit(exitcode)
[buildstream] 01/08: doc/source/format_declaring.rst: Minor
corrections and added references.
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 39e219e69788e9bcef65fa83a0d6df8b120d6d0d
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Dec 8 15:46:27 2020 +0900
doc/source/format_declaring.rst: Minor corrections and added references.
* When documenting dependency types, add references to the shorthand
convenience lists `build-depends` and `runtime-depends`.
* When documenting the `type` attribute of dependencies, correct the
language referring to the convenience lists to specify `build-depends`
and `runtime-depends` instead of the incorrectly worded
`Build-Depends` and `Runtime-Depends`, which would be invalid keys.
---
doc/source/format_declaring.rst | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/doc/source/format_declaring.rst b/doc/source/format_declaring.rst
index 380f367..1c6a5b3 100644
--- a/doc/source/format_declaring.rst
+++ b/doc/source/format_declaring.rst
@@ -450,8 +450,8 @@ Attributes:
* ``type``
This attribute is used to express the :ref:`dependency type <format_dependencies_types>`.
- This field is not permitted in :ref:`Build-Depends <format_build_depends>` or
- :ref:`Runtime-Depends <format_runtime_depends>`.
+ This field is not permitted in the :ref:`build-depends <format_build_depends>` or
+ :ref:`runtime-depends <format_runtime_depends>` lists.
* ``strict``
@@ -540,12 +540,18 @@ There are three types which one can specify for a dependency:
which has ``build`` dependencies will not implicitly depend on that element's
``build`` dependencies.
+ For convenience, these can be specified under the :ref:`build-depends <format_build_depends>`
+ list.
+
* ``runtime``
A ``runtime`` dependency type states that the given element's product
must be present for the depending element to function. An element's
``runtime`` dependencies are not available to the element at build time.
+ For convenience, these can be specified under the :ref:`runtime-depends <format_runtime_depends>`
+ list.
+
* ``all``
An ``all`` dependency is the default dependency type. If ``all`` is specified,
[buildstream] 06/08: _stream.py: Added missing machine readable
error codes
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5c08f314f68fbb8c17c21ba49ead2f1fe53f3f8d
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Dec 9 16:04:56 2020 +0900
_stream.py: Added missing machine readable error codes
In errors pertaining to failing to launch a shell with a buildtree.
---
src/buildstream/_stream.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index a539156..a3e4ba5 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -221,6 +221,7 @@ class Stream:
raise StreamError(
"Elements need to be built or downloaded before staging a shell environment",
detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))),
+ reason="shell-missing-deps",
)
# Check if we require a pull queue attempt, with given artifact state and context
@@ -229,11 +230,14 @@ class Stream:
remotes_message = " or in available remotes" if pull_ else ""
if not element._cached():
message = "Artifact not cached locally" + remotes_message
+ reason = "missing-buildtree-artifact-not-cached"
elif element._buildtree_exists():
message = "Buildtree is not cached locally" + remotes_message
+ reason = "missing-buildtree-artifact-buildtree-not-cached"
else:
message = "Artifact was created without buildtree"
- raise StreamError(message)
+ reason = "missing-buildtree-artifact-created-without-buildtree"
+ raise StreamError(message, reason=reason)
# Raise warning if the element is cached in a failed state
if element._cached_failure():
[buildstream] 04/08: NEWS: Document breaking change,
stack element dependencies must be build + run
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 43b6b2b48f9481e43c52761192e10ec0fc7f2c00
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Dec 8 16:59:50 2020 +0900
NEWS: Document breaking change, stack element dependencies must be build + run
---
NEWS | 3 +++
1 file changed, 3 insertions(+)
diff --git a/NEWS b/NEWS
index 4104630..ce93d29 100644
--- a/NEWS
+++ b/NEWS
@@ -11,6 +11,9 @@ Format
BuildElement plugins to also stage dependencies into custom locations in
the sandbox.
+ o BREAKING CHANGE: Stack element dependencies are now hard required to be
+ both build and runtime dependencies.
+
Core
----
[buildstream] 02/08: doc: Documenting "artifact names".
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d33b29f27d0d22c55dd96157efaad7ef28a2d036
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Dec 8 16:51:35 2020 +0900
doc: Documenting "artifact names".
Added a description about artifact names at the beginning of the
artifact commands section, along with a new glossary entry for
artifact names which refers to the description.
---
doc/source/main_glossary.rst | 4 ++++
doc/source/using_commands.rst | 23 +++++++++++++++++++++++
2 files changed, 27 insertions(+)
diff --git a/doc/source/main_glossary.rst b/doc/source/main_glossary.rst
index aa0c6da..f1bf4cf 100644
--- a/doc/source/main_glossary.rst
+++ b/doc/source/main_glossary.rst
@@ -18,6 +18,10 @@ Glossary
Artifacts can be built from :term:`Sources <Source>`, or pulled from a
:term:`Remote Cache <Remote Cache>`, if available.
+ Artifact name
+ The :ref:`name of an artifact <artifact_names>`, which can be used
+ in various commands to operate directly on artifacts, without requiring
+ the use of a :term:`Project`.
Cache
BuildStream leverages various caching techniques in order to avoid
diff --git a/doc/source/using_commands.rst b/doc/source/using_commands.rst
index 27fe692..0ec39fc 100644
--- a/doc/source/using_commands.rst
+++ b/doc/source/using_commands.rst
@@ -77,6 +77,29 @@ Top-level commands
Artifact subcommands
--------------------
+
+.. _artifact_names:
+
+Artifact names
+~~~~~~~~~~~~~~
+Various artifact subcommands accept either :ref:`element names <format_element_names>`,
+which will operate on artifacts by deriving the artifact from local project state,
+or :term:`artifact names <Artifact name>` interchangeably as targets. Artifact names allow
+the user to operate directly on cached artifacts, without requiring local project data.
+
+An artifact name is composed of the following identifiers:
+
+* The :ref:`project name <project_format_name>`
+
+* The :ref:`element name <format_element_names>`, without any trailing ``.bst`` extension
+
+* The cache key of the element at the time it was built.
+
+To compose an artifact name, simply join these using a forward slash (``/``) charachter, like so: ``<project-name>/<element-name>/<cache-key>``.
+
+An artifact name might look like: ``project/target/788da21e7c1b5818b7e7b60f7eb75841057ff7e45d362cc223336c606fe47f27``
+
+
.. _invoking_artifact_checkout:
.. click:: buildstream._frontend.cli:artifact_checkout
[buildstream] 03/08: plugins/elements/stack.py: Require all
dependencies be build & run.
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1cb5d696b845b492ec6fbebf4b35071efe3151cb
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Dec 8 16:52:44 2020 +0900
plugins/elements/stack.py: Require all dependencies be build & run.
Stack elements cannot be build-only dependencies, as this would defeat
the purpose of using stack elements in order to directly build-depend on
them.
Stack element dependencies must all be built in order to build depend
on them, and as such we gain no build parallelism by allowing runtime-only
dependencies on stack elements. Declaring a runtime-only dependency on
a stack element as a whole might still be useful, but still requires the
entire stack to be built at the time we need that stack.
Instead, it is more useful to ensure that a stack element is a logical
group of all dependencies, including runtime dependencies, such that we
can guarantee cache key alignment with all stack dependencies.
This allows for stronger reliability in commands such as
`bst artifact checkout`, which can now reliably download and checkout
a fully built stack as a result, without any uncertainty about possible
runtime-only dependencies which might exist in the project where that
artifact was created.
This consequently closes #1075
This also fixes the following tests such that the no longer
require build-depends or runtime-depends to work in stack elements:
* tests/frontend/default_target.py: Was not necessary to check results of show,
these stacks were set to runtime-depends so that they would have the same
buildable state as their dependencies when shown.
* tests/format/dependencies.py: tests/frontend/pull.py, test/frontend/show.py,
tests/integration/compose.py:
These tests were using specific build/runtime dependencies in stacks, but
for no particular reason.
---
src/buildstream/plugins/elements/stack.py | 95 +++++++++++++++++++++-
.../dependencies1/elements/builddep-list.bst | 4 +-
.../format/dependencies1/elements/list-combine.bst | 2 +-
.../dependencies1/elements/runtimedep-list.bst | 4 +-
.../default-target/elements/dummy_stack.bst | 2 +-
tests/frontend/default_target.py | 44 +++++-----
tests/frontend/pull.py | 2 +-
.../strict-depends/elements/non-strict-depends.bst | 2 +-
.../strict-depends/elements/strict-depends.bst | 2 +-
.../project/elements/compose/test-integration.bst | 2 +-
10 files changed, 126 insertions(+), 33 deletions(-)
diff --git a/src/buildstream/plugins/elements/stack.py b/src/buildstream/plugins/elements/stack.py
index b15f670..bd914ed 100644
--- a/src/buildstream/plugins/elements/stack.py
+++ b/src/buildstream/plugins/elements/stack.py
@@ -1,5 +1,5 @@
#
-# Copyright (C) 2016 Codethink Limited
+# Copyright (C) 2020 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -22,9 +22,85 @@ stack - Symbolic Element for dependency grouping
================================================
Stack elements are simply a symbolic element used for representing
a logical group of elements.
+
+All dependencies declared in stack elements must always be both
+:ref:`build and runtime dependencies <format_dependencies_types>`.
+
+**Example:**
+
+.. code:: yaml
+
+ kind: stack
+
+ # Declare all of your dependencies in the `depends` list.
+ depends:
+ - libc.bst
+ - coreutils.bst
+
+.. note::
+
+ Unlike other elements, whose cache keys are a unique identifier
+ of the contents of the artifacts they produce, stack elements do
+ not produce any artifact content. Instead, the cache key of an artifact
+ is a unique identifier for the assembly of its own dependencies.
+
+
+Using intermediate stacks
+-------------------------
+Using a stack element at intermediate levels of your build graph
+allows you to abstract away some parts of your project into logical
+subsystems which elements can more conveniently depend on as a whole.
+
+In addition to the added convenience, it will allow you to more
+easily change the implementation of a subsystem later on, without needing
+to update many reverse dependencies to depend on new elements, or even
+allow you to conditionally implement a subsystem with various implementations
+depending on what :ref:`project options <project_options>` were specified at
+build time.
+
+
+Using toplevel stacks
+---------------------
+Stack elements can also be useful as toplevel targets in your build graph
+to simply indicate all of the components which need to be built for a given
+system to be complete, or for your integration pipeline to be successful.
+
+
+Checking out and deploying toplevel stacks
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+In case that your software is built remotely, it is possible to checkout
+the built content of a stack on your own machine for the purposes of
+inspection or further deployment.
+
+To accomplish this, you will need to know the cache key of the stack element
+which was built remotely, possibly by inspecting the remote build log or by
+deriving it with an equally configured BuildStream project, and you will
+need read access to the artifact cache server which the build was uploaded to,
+this should be configured in your :ref:`user configuration file <config_artifacts>`.
+
+You can then checkout the remotely built stack using the
+:ref:`bst artifact checkout <invoking_artifact_checkout>` command and providing
+it with the :ref:`artifact name <artifact_names>`:
+
+**Example:**
+
+.. code:: shell
+
+ bst artifact checkout --deps build --pull --integrate \\
+ --directory `pwd`/checkout \\
+ project/stack/788da21e7c1b5818b7e7b60f7eb75841057ff7e45d362cc223336c606fe47f27
+
+.. note::
+
+ It is possible to checkout other elements in the same way, however stack
+ elements are uniquely suited to this purpose, as they cannot have
+ :ref:`runtime only dependencies <format_dependencies_types>`, and consequently
+ their cache keys are always a unique representation of their collective
+ dependencies.
"""
-from buildstream import Element
+from buildstream import Element, ElementError
+from buildstream.types import _Scope
# Element implementation for the 'stack' kind.
@@ -46,7 +122,20 @@ class StackElement(Element):
pass
def preflight(self):
- pass
+
+ # Assert that all dependencies are both build and runtime dependencies.
+ #
+ all_deps = list(self._dependencies(_Scope.ALL, recurse=False))
+ run_deps = list(self._dependencies(_Scope.RUN, recurse=False))
+ build_deps = list(self._dependencies(_Scope.BUILD, recurse=False))
+ if any(dep not in run_deps for dep in all_deps) or any(dep not in build_deps for dep in all_deps):
+ # There is no need to specify the `self` provenance here in preflight() errors, as the base class
+ # will take care of prefixing these for plugin author convenience.
+ raise ElementError(
+ "All dependencies of 'stack' elements must be both build and runtime dependencies",
+ detail="Make sure you declare all dependencies in the `depends` list, without specifying any `type`.",
+ reason="stack-requires-build-and-run",
+ )
def get_unique_key(self):
# We do not add anything to the build, only our dependencies
diff --git a/tests/format/dependencies1/elements/builddep-list.bst b/tests/format/dependencies1/elements/builddep-list.bst
index a0cbcaf..eb216a1 100644
--- a/tests/format/dependencies1/elements/builddep-list.bst
+++ b/tests/format/dependencies1/elements/builddep-list.bst
@@ -1,4 +1,4 @@
-kind: stack
+kind: manual
description: This element has a build-only dependency specified via build-depends
build-depends:
- - firstdep.bst
+- firstdep.bst
diff --git a/tests/format/dependencies1/elements/list-combine.bst b/tests/format/dependencies1/elements/list-combine.bst
index ed34522..d39ddd3 100644
--- a/tests/format/dependencies1/elements/list-combine.bst
+++ b/tests/format/dependencies1/elements/list-combine.bst
@@ -1,4 +1,4 @@
-kind: stack
+kind: manual
description: This element depends on three elements in different ways
build-depends:
- firstdep.bst
diff --git a/tests/format/dependencies1/elements/runtimedep-list.bst b/tests/format/dependencies1/elements/runtimedep-list.bst
index 1207a49..eaa0cd2 100644
--- a/tests/format/dependencies1/elements/runtimedep-list.bst
+++ b/tests/format/dependencies1/elements/runtimedep-list.bst
@@ -1,4 +1,4 @@
-kind: stack
+kind: manual
description: This element has a runtime-only dependency
runtime-depends:
- - firstdep.bst
+- firstdep.bst
diff --git a/tests/frontend/default-target/elements/dummy_stack.bst b/tests/frontend/default-target/elements/dummy_stack.bst
index 5f92166..3ea51b0 100644
--- a/tests/frontend/default-target/elements/dummy_stack.bst
+++ b/tests/frontend/default-target/elements/dummy_stack.bst
@@ -1,5 +1,5 @@
kind: stack
-runtime-depends:
+depends:
- dummy_1.bst
- dummy_2.bst
diff --git a/tests/frontend/default_target.py b/tests/frontend/default_target.py
index bb7a495..60578bb 100644
--- a/tests/frontend/default_target.py
+++ b/tests/frontend/default_target.py
@@ -13,27 +13,27 @@ from tests.testutils import create_artifact_share
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "default-target",)
-###################################
-# build/show operations #
-###################################
-
-
+#
+# When no target is default, then expect all targets to be built
+#
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("operation,expected_state", [("show", "buildable"), ("build", "cached")])
-def test_no_default(cli, datafiles, operation, expected_state):
+def test_no_default(cli, datafiles):
project = str(datafiles)
all_targets = ["dummy_1.bst", "dummy_2.bst", "dummy_3.bst", "dummy_stack.bst"]
- result = cli.run(project=project, args=[operation])
+ result = cli.run(project=project, args=["build"])
result.assert_success()
states = cli.get_element_states(project, all_targets)
- assert all(states[e] == expected_state for e in all_targets)
+ assert all(states[e] == "cached" for e in all_targets)
+#
+# When the stack is specified as the default target, then
+# expect only it and it's dependencies to be built
+#
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("operation,expected_state", [("show", "buildable"), ("build", "cached")])
-def test_default_target(cli, datafiles, operation, expected_state):
+def test_default_target(cli, datafiles):
project = str(datafiles)
project_path = os.path.join(project, "project.conf")
@@ -49,19 +49,23 @@ def test_default_target(cli, datafiles, operation, expected_state):
# dummy_stack only depends on dummy_1 and dummy_2, but not dummy_3
all_targets = ["dummy_1.bst", "dummy_2.bst", "dummy_stack.bst"]
- result = cli.run(project=project, args=[operation])
+ result = cli.run(project=project, args=["build"])
result.assert_success()
states = cli.get_element_states(project, all_targets)
- assert all(states[e] == expected_state for e in all_targets)
+ assert all(states[e] == "cached" for e in all_targets)
# assert that dummy_3 isn't included in the output
assert "dummy_3.bst" not in states
+#
+# Even when there is a junction, expect that the elements in the
+# subproject referred to by the toplevel project are built when
+# calling `bst build` and no default is specified.
+#
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("operation,expected_state", [("show", "buildable"), ("build", "cached")])
-def test_no_default_with_junction(cli, datafiles, operation, expected_state):
+def test_no_default_with_junction(cli, datafiles):
project = str(datafiles)
junction_path = os.path.join(project, "elements", "junction.bst")
target_path = os.path.join(project, "elements", "junction-target.bst")
@@ -71,16 +75,16 @@ def test_no_default_with_junction(cli, datafiles, operation, expected_state):
_yaml.roundtrip_dump(junction_config, junction_path)
# Then, create a stack element with dependency on cross junction element
- target_config = {"kind": "stack", "runtime-depends": ["junction.bst:dummy_subproject.bst"]}
+ target_config = {"kind": "stack", "depends": ["junction.bst:dummy_subproject.bst"]}
_yaml.roundtrip_dump(target_config, target_path)
- # Now try to perform the specified operation.
+ # Now try to perform a build
# This should automatically fetch the junction at load time.
- result = cli.run(project=project, args=[operation])
+ result = cli.run(project=project, args=["build"])
result.assert_success()
- assert cli.get_element_state(project, "junction.bst:dummy_subproject.bst") == expected_state
- assert cli.get_element_state(project, "junction-target.bst") == expected_state
+ assert cli.get_element_state(project, "junction.bst:dummy_subproject.bst") == "cached"
+ assert cli.get_element_state(project, "junction-target.bst") == "cached"
###################################
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index c1fa76a..f873ad9 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -338,7 +338,7 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles):
_yaml.roundtrip_dump(input_config, input_file)
depends_name = "depends.bst"
- depends_config = {"kind": "stack", "depends": [{"filename": input_name, "type": "build"}]}
+ depends_config = {"kind": "stack", "depends": [input_name]}
depends_file = os.path.join(element_dir, depends_name)
_yaml.roundtrip_dump(depends_config, depends_file)
diff --git a/tests/frontend/strict-depends/elements/non-strict-depends.bst b/tests/frontend/strict-depends/elements/non-strict-depends.bst
index 9ab119b..e0ed21e 100644
--- a/tests/frontend/strict-depends/elements/non-strict-depends.bst
+++ b/tests/frontend/strict-depends/elements/non-strict-depends.bst
@@ -1,4 +1,4 @@
kind: stack
-build-depends:
+depends:
- base.bst
diff --git a/tests/frontend/strict-depends/elements/strict-depends.bst b/tests/frontend/strict-depends/elements/strict-depends.bst
index 1e4e294..10f3b94 100644
--- a/tests/frontend/strict-depends/elements/strict-depends.bst
+++ b/tests/frontend/strict-depends/elements/strict-depends.bst
@@ -1,5 +1,5 @@
kind: stack
-build-depends:
+depends:
- filename: base.bst
strict: true
diff --git a/tests/integration/project/elements/compose/test-integration.bst b/tests/integration/project/elements/compose/test-integration.bst
index 2f9faf1..ef47b3b 100644
--- a/tests/integration/project/elements/compose/test-integration.bst
+++ b/tests/integration/project/elements/compose/test-integration.bst
@@ -1,6 +1,6 @@
kind: stack
-runtime-depends:
+depends:
- base.bst
public:
[buildstream] 05/08: tests/format/stack.py: Test stack dependency
types
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/shell-artifacts
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 35639c7d6af3d258826b8e55af6dfe8bf3d3e59f
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Dec 8 17:42:38 2020 +0900
tests/format/stack.py: Test stack dependency types
Assert that errors are raised when stack dependencies are declared as
build-only or runtime-only dependencies.
---
tests/format/stack.py | 22 ++++++++++++++++++++++
tests/format/stack/elements/build-only-stack.bst | 4 ++++
tests/format/stack/elements/dependency.bst | 2 ++
tests/format/stack/elements/runtime-only-stack.bst | 4 ++++
tests/format/stack/project.conf | 4 ++++
5 files changed, 36 insertions(+)
diff --git a/tests/format/stack.py b/tests/format/stack.py
new file mode 100644
index 0000000..b014e9b
--- /dev/null
+++ b/tests/format/stack.py
@@ -0,0 +1,22 @@
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+import pytest
+
+from buildstream.exceptions import ErrorDomain
+from buildstream.testing import cli # pylint: disable=unused-import
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack")
+
+
+#
+# Assert that we have errors when trying to have runtime-only or
+# build-only dependencies.
+#
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("target", ["build-only-stack.bst", "runtime-only-stack.bst",])
+def test_require_build_and_run(cli, datafiles, target):
+ project = str(datafiles)
+ result = cli.run(project=project, args=["show", target])
+ result.assert_main_error(ErrorDomain.ELEMENT, "stack-requires-build-and-run")
diff --git a/tests/format/stack/elements/build-only-stack.bst b/tests/format/stack/elements/build-only-stack.bst
new file mode 100644
index 0000000..92f1c80
--- /dev/null
+++ b/tests/format/stack/elements/build-only-stack.bst
@@ -0,0 +1,4 @@
+kind: stack
+
+build-depends:
+- dependency.bst
diff --git a/tests/format/stack/elements/dependency.bst b/tests/format/stack/elements/dependency.bst
new file mode 100644
index 0000000..8c231b6
--- /dev/null
+++ b/tests/format/stack/elements/dependency.bst
@@ -0,0 +1,2 @@
+kind: manual
+description: This is a dependency
diff --git a/tests/format/stack/elements/runtime-only-stack.bst b/tests/format/stack/elements/runtime-only-stack.bst
new file mode 100644
index 0000000..b5dd9a7
--- /dev/null
+++ b/tests/format/stack/elements/runtime-only-stack.bst
@@ -0,0 +1,4 @@
+kind: stack
+
+runtime-depends:
+- dependency.bst
diff --git a/tests/format/stack/project.conf b/tests/format/stack/project.conf
new file mode 100644
index 0000000..9a5f11e
--- /dev/null
+++ b/tests/format/stack/project.conf
@@ -0,0 +1,4 @@
+# Basic project
+name: test
+min-version: 2.0
+element-path: elements