You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ju...@apache.org on 2021/05/20 08:09:25 UTC
[buildstream] branch juerg/remote-cache-ci created (now b12c13f)
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a change to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at b12c13f TEMP: drop local tests
This branch includes the following new commits:
new 5b75982 .github: Rename remote service tests in CI
new e5cf255 _artifact.py: Add cached and strong_key parameters to set_cached()
new bf2eecd element.py: Move get_logs() to internal API
new 7ecbc41 Drop pull-artifact-files config option for remote execution
new d20ed3d Always require artifact files
new 8ef8081 _sandboxremote.py: Drop unused output_files_required parameter
new b44beaf _cas: Add support for remote cache
new 977180a _context.py: Add support for remote-cache configuration
new 0de2a8a _sandboxremote.py: Make storage-service optional with remote cache
new 1bd00f1 _scheduler: Add CacheQueryQueue to support cache query in job threads
new dfd9f07 tests/frontend/remote-caches.py: Add test for remote-cache
new f92ddcc tests/remoteexecution/remotecache.py: Add test for remote-cache
new 74c8a90 _stream.py: Parallelize cache queries if a remote cache is configured
new b12c13f TEMP: drop local tests
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 05/14: Always require artifact files
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d20ed3d87862c6ad8d1568131a85f563d3c778fd
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Mon Oct 26 18:26:17 2020 +0100
Always require artifact files
In line with the previous commit, also require artifact files for build
dependencies with remote execution.
Local artifact files will become optional with the upcoming remote cache
support.
---
src/buildstream/_artifact.py | 13 +------
src/buildstream/_context.py | 15 ---------
src/buildstream/_stream.py | 9 -----
src/buildstream/element.py | 29 ----------------
tests/remoteexecution/partial.py | 73 ----------------------------------------
5 files changed, 1 insertion(+), 138 deletions(-)
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index 51e1f57..98f8250 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -551,24 +551,13 @@ class Artifact:
# (bool): Whether artifact is in local cache
#
def query_cache(self):
- context = self._context
-
artifact = self._load_proto()
if not artifact:
self._cached = False
return False
- # Determine whether directories are required
- require_directories = context.require_artifact_directories
- # Determine whether file contents are required as well
- require_files = context.require_artifact_files or self._element._artifact_files_required()
-
# Check whether 'files' subdirectory is available, with or without file contents
- if (
- require_directories
- and str(artifact.files)
- and not self._cas.contains_directory(artifact.files, with_files=require_files)
- ):
+ if str(artifact.files) and not self._cas.contains_directory(artifact.files, with_files=True):
self._cached = False
return False
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 2c1f25a..d5d0669 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -181,12 +181,6 @@ class Context:
# Whether or not to cache build trees on artifact creation
self.cache_buildtrees: Optional[str] = None
- # Whether directory trees are required for all artifacts in the local cache
- self.require_artifact_directories: bool = True
-
- # Whether file contents are required for all artifacts in the local cache
- self.require_artifact_files: bool = True
-
# Don't shoot the messenger
self.messenger: Messenger = Messenger()
@@ -654,15 +648,6 @@ class Context:
# value which we cache here too.
return self._strict_build_plan
- # set_artifact_files_optional()
- #
- # This indicates that the current context (command or configuration)
- # does not require file contents of all artifacts to be available in the
- # local cache.
- #
- def set_artifact_files_optional(self) -> None:
- self.require_artifact_files = False
-
def get_cascache(self) -> CASCache:
if self._cascache is None:
if self.log_debug:
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 0559d8d..e22b93c 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -378,15 +378,6 @@ class Stream:
# Assert that the elements are consistent
_pipeline.assert_consistent(self._context, elements)
- if self._context.remote_execution_specs:
- # Remote execution is configured.
- # Require artifact files only for target elements and their runtime dependencies.
- self._context.set_artifact_files_optional()
-
- scope = _Scope.ALL if selection == _PipelineSelection.ALL else _Scope.RUN
- for element in self.targets:
- element._set_artifact_files_required(scope=scope)
-
source_push_enabled = self._sourcecache.has_push_remotes()
# If source push is enabled, the source cache status of all elements
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index bba3506..38240bb 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -296,7 +296,6 @@ class Element(Plugin):
self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap
self.__tainted = None # Whether the artifact is tainted and should not be shared
self.__required = False # Whether the artifact is required in the current session
- self.__artifact_files_required = False # Whether artifact files are required in the local cache
self.__build_result = None # The result of assembling this Element (success, description, detail)
# Artifact class for direct artifact composite interaction
self.__artifact = None # type: Optional[Artifact]
@@ -1555,31 +1554,6 @@ class Element(Plugin):
def _is_required(self):
return self.__required
- # _set_artifact_files_required():
- #
- # Mark artifact files for this element and its runtime dependencies as
- # required in the local cache.
- #
- def _set_artifact_files_required(self, scope=_Scope.RUN):
- assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread"
-
- if self.__artifact_files_required:
- # Already done
- return
-
- self.__artifact_files_required = True
-
- # Request artifact files of runtime dependencies
- for dep in self._dependencies(scope, recurse=False):
- dep._set_artifact_files_required(scope=scope)
-
- # _artifact_files_required():
- #
- # Returns whether artifact files for this element have been marked as required.
- #
- def _artifact_files_required(self):
- return self.__artifact_files_required
-
# __should_schedule()
#
# Returns:
@@ -2797,8 +2771,6 @@ class Element(Plugin):
self.info("Using a remote sandbox for artifact {} with directory '{}'".format(self.name, directory))
- output_files_required = context.require_artifact_files or self._artifact_files_required()
-
sandbox = SandboxRemote(
context,
project,
@@ -2807,7 +2779,6 @@ class Element(Plugin):
stdout=stdout,
stderr=stderr,
config=config,
- output_files_required=output_files_required,
output_node_properties=output_node_properties,
)
yield sandbox
diff --git a/tests/remoteexecution/partial.py b/tests/remoteexecution/partial.py
deleted file mode 100644
index cd87293..0000000
--- a/tests/remoteexecution/partial.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-import pytest
-
-from buildstream.exceptions import ErrorDomain
-from buildstream.testing import cli_remote_execution as cli # pylint: disable=unused-import
-from buildstream.testing.integration import assert_contains
-
-from tests.testutils.artifactshare import create_artifact_share
-
-
-pytestmark = pytest.mark.remoteexecution
-
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
-
-
-# Test that `bst build` does not download file blobs of a build-only dependency
-# to the local cache.
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("build_all", [True, False])
-def test_build_dependency_partial_local_cas(cli, datafiles, build_all):
- project = str(datafiles)
- element_name = "no-runtime-deps.bst"
- builddep_element_name = "autotools/amhello.bst"
- checkout = os.path.join(cli.directory, "checkout")
- builddep_checkout = os.path.join(cli.directory, "builddep-checkout")
-
- services = cli.ensure_services()
- assert set(services) == set(["action-cache", "execution", "storage"])
-
- # configure pull blobs
- if build_all:
- cli.configure({"build": {"dependencies": "all"}})
-
- result = cli.run(project=project, args=["build", element_name])
- result.assert_success()
-
- # Verify artifact is pulled bar files when ensure artifact files is set
- result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
- result.assert_success()
- assert_contains(checkout, ["/test"])
-
- # Verify build dependencies is pulled for ALL and BUILD
- result = cli.run(
- project=project, args=["artifact", "checkout", builddep_element_name, "--directory", builddep_checkout]
- )
- if build_all:
- result.assert_success()
- else:
- result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_build_partial_push(cli, tmpdir, datafiles):
- project = str(datafiles)
- share_dir = os.path.join(str(tmpdir), "artifactshare")
- element_name = "no-runtime-deps.bst"
- builddep_element_name = "autotools/amhello.bst"
-
- with create_artifact_share(share_dir) as share:
-
- services = cli.ensure_services()
- assert set(services) == set(["action-cache", "execution", "storage"])
-
- cli.config["artifacts"] = {"servers": [{"url": share.repo, "push": True,}]}
-
- res = cli.run(project=project, args=["build", element_name])
- res.assert_success()
-
- assert builddep_element_name in res.get_pushed_elements()
[buildstream] 11/14: tests/frontend/remote-caches.py: Add test for
remote-cache
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit dfd9f078e8de1e429142ce27409bbdb7ec5fd06e
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Wed May 19 11:03:30 2021 +0200
tests/frontend/remote-caches.py: Add test for remote-cache
This is a simple test for `remote-cache` without remote execution.
---
tests/frontend/remote-caches.py | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/tests/frontend/remote-caches.py b/tests/frontend/remote-caches.py
index 103af10..892190e 100644
--- a/tests/frontend/remote-caches.py
+++ b/tests/frontend/remote-caches.py
@@ -36,6 +36,36 @@ def message_handler(message, context):
@pytest.mark.datafiles(DATA_DIR)
+def test_build_checkout(cli, tmpdir, datafiles):
+ cachedir = os.path.join(str(tmpdir), "cache")
+ project = str(datafiles)
+ checkout = os.path.join(cli.directory, "checkout")
+
+ with create_artifact_share(os.path.join(str(tmpdir), "remote-cache")) as remote_cache:
+ # Enable remote cache
+ cli.configure({"cache": {"remote-cache": {"url": remote_cache.repo}}})
+
+ # First build it
+ result = cli.run(project=project, args=["build", "target.bst"])
+ result.assert_success()
+
+ # Discard the local CAS cache
+ shutil.rmtree(str(os.path.join(cachedir, "cas")))
+
+ # Now check it out, this should automatically fetch the necessary blobs
+ # from the remote cache
+ result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout])
+ result.assert_success()
+
+ # Check that the executable hello file is found in the checkout
+ filename = os.path.join(checkout, "usr", "bin", "hello")
+ assert os.path.exists(filename)
+
+ filename = os.path.join(checkout, "usr", "include", "pony.h")
+ assert os.path.exists(filename)
+
+
+@pytest.mark.datafiles(DATA_DIR)
def test_source_artifact_caches(cli, tmpdir, datafiles):
cachedir = os.path.join(str(tmpdir), "cache")
project_dir = str(datafiles)
[buildstream] 07/14: _cas: Add support for remote cache
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b44beafb2732e8bf1eb440affe3a28aa6bf6be0f
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Mon Oct 19 16:17:36 2020 +0200
_cas: Add support for remote cache
---
src/buildstream/_cas/cascache.py | 86 ++++++++++++++++++++++++++----
src/buildstream/_cas/casdprocessmanager.py | 13 ++++-
src/buildstream/_cas/casremote.py | 5 ++
src/buildstream/_cas/casserver.py | 2 +-
src/buildstream/_remote.py | 9 +++-
5 files changed, 102 insertions(+), 13 deletions(-)
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index b80460a..609feb5 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -37,7 +37,7 @@ from ..types import FastEnum, SourceRef
from .._exceptions import CASCacheError
from .casdprocessmanager import CASDProcessManager
-from .casremote import _CASBatchRead, _CASBatchUpdate, BlobNotFound
+from .casremote import CASRemote, _CASBatchRead, _CASBatchUpdate, BlobNotFound
_BUFFER_SIZE = 65536
@@ -69,6 +69,7 @@ class CASCache:
*,
casd=True,
cache_quota=None,
+ remote_cache_spec=None,
protect_session_blobs=True,
log_level=CASLogLevel.WARNING,
log_directory=None
@@ -80,18 +81,25 @@ class CASCache:
self._cache_usage_monitor = None
self._cache_usage_monitor_forbidden = False
+ self._remote_cache = bool(remote_cache_spec)
+
self._casd_process_manager = None
self._casd_channel = None
if casd:
assert log_directory is not None, "log_directory is required when casd is True"
log_dir = os.path.join(log_directory, "_casd")
self._casd_process_manager = CASDProcessManager(
- path, log_dir, log_level, cache_quota, protect_session_blobs
+ path, log_dir, log_level, cache_quota, remote_cache_spec, protect_session_blobs
)
self._casd_channel = self._casd_process_manager.create_channel()
self._cache_usage_monitor = _CASCacheUsageMonitor(self._casd_channel)
self._cache_usage_monitor.start()
+ else:
+ assert not self._remote_cache
+
+ self._default_remote = CASRemote(None, self)
+ self._default_remote.init()
# get_cas():
#
@@ -142,6 +150,9 @@ class CASCache:
self._casd_process_manager.release_resources(messenger)
self._casd_process_manager = None
+ def get_default_remote(self):
+ return self._default_remote
+
# contains_files():
#
# Check whether file digests exist in the local CAS cache
@@ -168,13 +179,17 @@ class CASCache:
def contains_directory(self, digest, *, with_files):
local_cas = self.get_local_cas()
+ # Without a remote cache, `FetchTree` simply checks the local cache.
request = local_cas_pb2.FetchTreeRequest()
request.root_digest.CopyFrom(digest)
- request.fetch_file_blobs = with_files
+ # Always fetch Directory protos as they are needed to enumerate subdirectories and files.
+ # Don't implicitly fetch file blobs from the remote cache as we don't need them.
+ request.fetch_file_blobs = with_files and not self._remote_cache
try:
local_cas.FetchTree(request)
- return True
+ if not self._remote_cache:
+ return True
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.NOT_FOUND:
return False
@@ -182,6 +197,10 @@ class CASCache:
raise CASCacheError("Unsupported buildbox-casd version: FetchTree unimplemented") from e
raise
+ # Check whether everything is available in the remote cache.
+ missing_blobs = self.missing_blobs_for_directory(digest, remote=self._default_remote)
+ return not missing_blobs
+
# checkout():
#
# Checkout the specified directory digest.
@@ -191,7 +210,17 @@ class CASCache:
# tree (Digest): The directory digest to extract
# can_link (bool): Whether we can create hard links in the destination
#
- def checkout(self, dest, tree, *, can_link=False):
+ def checkout(self, dest, tree, *, can_link=False, _fetch=True):
+ if _fetch and self._remote_cache:
+ # We need the files in the local cache
+ local_cas = self.get_local_cas()
+
+ request = local_cas_pb2.FetchTreeRequest()
+ request.root_digest.CopyFrom(tree)
+ request.fetch_file_blobs = True
+
+ local_cas.FetchTree(request)
+
os.makedirs(dest, exist_ok=True)
directory = remote_execution_pb2.Directory()
@@ -229,7 +258,7 @@ class CASCache:
for dirnode in directory.directories:
fullpath = os.path.join(dest, dirnode.name)
- self.checkout(fullpath, dirnode.digest, can_link=can_link)
+ self.checkout(fullpath, dirnode.digest, can_link=can_link, _fetch=False)
for symlinknode in directory.symlinks:
# symlink
@@ -286,6 +315,11 @@ class CASCache:
objpath = self.objpath(digest)
+ if self._remote_cache and not os.path.exists(objpath):
+ batch = _CASBatchRead(self._default_remote)
+ batch.add(digest)
+ batch.send()
+
return open(objpath, mode=mode)
# add_object():
@@ -399,7 +433,7 @@ class CASCache:
if tree_response.status.code == code_pb2.RESOURCE_EXHAUSTED:
raise CASCacheError("Cache too full", reason="cache-too-full")
if tree_response.status.code != code_pb2.OK:
- raise CASCacheError("Failed to capture tree {}: {}".format(path, tree_response.status.code))
+ raise CASCacheError("Failed to capture tree {}: {}".format(path, tree_response.status))
treepath = self.objpath(tree_response.tree_digest)
tree = remote_execution_pb2.Tree()
@@ -469,10 +503,20 @@ class CASCache:
# Generator that returns the Digests of all blobs in the tree specified by
# the Digest of the toplevel Directory object.
#
- def required_blobs_for_directory(self, directory_digest, *, excluded_subdirs=None):
+ def required_blobs_for_directory(self, directory_digest, *, excluded_subdirs=None, _fetch_tree=True):
if not excluded_subdirs:
excluded_subdirs = []
+ if self._remote_cache and _fetch_tree:
+ # Ensure we have the directory protos in the local cache
+ local_cas = self.get_local_cas()
+
+ request = local_cas_pb2.FetchTreeRequest()
+ request.root_digest.CopyFrom(directory_digest)
+ request.fetch_file_blobs = False
+
+ local_cas.FetchTree(request)
+
# parse directory, and recursively add blobs
yield directory_digest
@@ -487,7 +531,7 @@ class CASCache:
for dirnode in directory.directories:
if dirnode.name not in excluded_subdirs:
- yield from self.required_blobs_for_directory(dirnode.digest)
+ yield from self.required_blobs_for_directory(dirnode.digest, _fetch_tree=False)
################################################
# Local Private Methods #
@@ -569,6 +613,10 @@ class CASCache:
# Returns: The Digests of the blobs that were not available on the remote CAS
#
def fetch_blobs(self, remote, digests, *, allow_partial=False):
+ if self._remote_cache:
+ # Determine blobs missing in the remote cache and only fetch those
+ digests = self.missing_blobs(digests)
+
missing_blobs = [] if allow_partial else None
remote.init()
@@ -581,6 +629,15 @@ class CASCache:
batch.send(missing_blobs=missing_blobs)
+ if self._remote_cache:
+ # Upload fetched blobs to the remote cache as we can't transfer
+ # blobs directly from another remote to the remote cache
+ batch = _CASBatchUpdate(self._default_remote)
+ for digest in digests:
+ if missing_blobs is None or digest not in missing_blobs: # pylint: disable=unsupported-membership-test
+ batch.add(digest)
+ batch.send()
+
return missing_blobs
# send_blobs():
@@ -592,6 +649,17 @@ class CASCache:
# digests (list): The Digests of Blobs to upload
#
def send_blobs(self, remote, digests):
+ if self._remote_cache:
+ # First fetch missing blobs from the remote cache as we can't
+ # transfer blobs directly from the remote cache to another remote.
+
+ remote_missing_blobs = self.missing_blobs(digests, remote=remote)
+
+ batch = _CASBatchRead(self._default_remote)
+ for digest in remote_missing_blobs:
+ batch.add(digest)
+ batch.send()
+
batch = _CASBatchUpdate(remote)
for digest in digests:
diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py
index 11a16f2..0a7d768 100644
--- a/src/buildstream/_cas/casdprocessmanager.py
+++ b/src/buildstream/_cas/casdprocessmanager.py
@@ -51,10 +51,11 @@ _CASD_TIMEOUT = 300 # in seconds
# log_dir (str): The directory for the logs
# log_level (LogLevel): Log level to give to buildbox-casd for logging
# cache_quota (int): User configured cache quota
+# remote_cache_spec (RemoteSpec): Optional remote cache server
# protect_session_blobs (bool): Disable expiry for blobs used in the current session
#
class CASDProcessManager:
- def __init__(self, path, log_dir, log_level, cache_quota, protect_session_blobs):
+ def __init__(self, path, log_dir, log_level, cache_quota, remote_cache_spec, protect_session_blobs):
self._log_dir = log_dir
self._socket_path = self._make_socket_path(path)
@@ -71,6 +72,16 @@ class CASDProcessManager:
if protect_session_blobs:
casd_args.append("--protect-session-blobs")
+ if remote_cache_spec:
+ casd_args.append("--cas-remote={}".format(remote_cache_spec.url))
+ if remote_cache_spec.instance_name:
+ casd_args.append("--cas-instance={}".format(remote_cache_spec.instance_name))
+ if remote_cache_spec.server_cert:
+ casd_args.append("--cas-server-cert={}".format(remote_cache_spec.server_cert))
+ if remote_cache_spec.client_key:
+ casd_args.append("--cas-client-key={}".format(remote_cache_spec.client_key))
+ casd_args.append("--cas-client-cert={}".format(remote_cache_spec.client_cert))
+
casd_args.append(path)
self._start_time = time.time()
diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py
index 3799f95..b9ae3d7 100644
--- a/src/buildstream/_cas/casremote.py
+++ b/src/buildstream/_cas/casremote.py
@@ -55,6 +55,11 @@ class CASRemote(BaseRemote):
# be called outside of init().
#
def _configure_protocols(self):
+ if not self.spec:
+ # Remote cache (handled by default instance in casd)
+ self.local_cas_instance_name = ""
+ return
+
local_cas = self.cascache.get_local_cas()
request = local_cas_pb2.GetInstanceNameForRemotesRequest()
cas_endpoint = request.content_addressable_storage
diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py
index 04c5eb8..fd66103 100644
--- a/src/buildstream/_cas/casserver.py
+++ b/src/buildstream/_cas/casserver.py
@@ -106,7 +106,7 @@ def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.Le
logger.addHandler(handler)
casd_manager = CASDProcessManager(
- os.path.abspath(repo), os.path.join(os.path.abspath(repo), "logs"), log_level, quota, False
+ os.path.abspath(repo), os.path.join(os.path.abspath(repo), "logs"), log_level, quota, None, False
)
casd_channel = casd_manager.create_channel()
diff --git a/src/buildstream/_remote.py b/src/buildstream/_remote.py
index 0d47921..42314eb 100644
--- a/src/buildstream/_remote.py
+++ b/src/buildstream/_remote.py
@@ -52,7 +52,10 @@ class BaseRemote:
return False
def __str__(self):
- return self.spec.url
+ if self.spec:
+ return self.spec.url
+ else:
+ return "(default remote)"
####################################################
# Remote API #
@@ -68,7 +71,9 @@ class BaseRemote:
if self._initialized:
return
- self.channel = self.spec.open_channel()
+ if self.spec:
+ self.channel = self.spec.open_channel()
+
self._configure_protocols()
self._initialized = True
[buildstream] 12/14: tests/remoteexecution/remotecache.py: Add test
for remote-cache
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f92ddcceb08fde2aa1ae691249b683bf5cba9d54
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu May 20 10:06:18 2021 +0200
tests/remoteexecution/remotecache.py: Add test for remote-cache
This tests `remote-cache` with remote execution. It verifies that build
outputs are not downloaded if not necessary.
---
tests/remoteexecution/remotecache.py | 58 ++++++++++++++++++++++++++++++++++++
1 file changed, 58 insertions(+)
diff --git a/tests/remoteexecution/remotecache.py b/tests/remoteexecution/remotecache.py
new file mode 100644
index 0000000..02759c2
--- /dev/null
+++ b/tests/remoteexecution/remotecache.py
@@ -0,0 +1,58 @@
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+import pytest
+
+from buildstream.testing import cli_remote_execution as cli # pylint: disable=unused-import
+from buildstream.testing.integration import assert_contains
+
+
+pytestmark = pytest.mark.remoteexecution
+
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
+
+
+# Test building an executable with remote-execution and remote-cache enabled
+@pytest.mark.datafiles(DATA_DIR)
+def test_remote_autotools_build(cli, datafiles, remote_services):
+ project = str(datafiles)
+ checkout = os.path.join(cli.directory, "checkout")
+ element_name = "autotools/amhello.bst"
+
+ services = cli.ensure_services()
+ assert set(services) == set(["action-cache", "execution", "storage"])
+
+ # Enable remote cache and remove explicit remote execution CAS configuration.
+ cli.configure({"cache": {"remote-cache": {"url": remote_services.storage_service}}})
+ del cli.config["remote-execution"]["storage-service"]
+
+ # Build element with remote execution.
+ result = cli.run(project=project, args=["build", element_name])
+ result.assert_success()
+
+ # Attempt checkout from local cache by temporarily disabling remote cache.
+ # This should fail as the build result shouldn't have been downloaded to the local cache.
+ del cli.config["cache"]["remote-cache"]
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
+ result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
+
+ # Re-enable remote cache and attempt checkout again.
+ cli.configure({"cache": {"remote-cache": {"url": remote_services.storage_service}}})
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
+ result.assert_success()
+
+ assert_contains(
+ checkout,
+ [
+ "/usr",
+ "/usr/lib",
+ "/usr/bin",
+ "/usr/share",
+ "/usr/bin/hello",
+ "/usr/share/doc",
+ "/usr/share/doc/amhello",
+ "/usr/share/doc/amhello/README",
+ ],
+ )
[buildstream] 01/14: .github: Rename remote service tests in CI
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5b7598248a103bed125b85bd8d60a346e09749c4
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Tue May 18 11:47:21 2021 +0200
.github: Rename remote service tests in CI
There is one CI job for each remote service, which should run all
applicable tests against that particular service. Consistently use the
name of the remote service as CI job name to clarify this and prepare
for additional tests.
---
.../{ci.buildstream-remote-cache.yml => ci.bst-artifact-server.yml} | 4 ++--
.github/compose/{ci.buildbarn-remote-cache.yml => ci.buildbarn.yml} | 4 ++--
.github/compose/{ci.remote-execution.yml => ci.buildgrid.yml} | 4 ++--
.github/compose/ci.docker-compose.yml | 6 +++---
.github/workflows/ci.yml | 6 +++---
5 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/.github/compose/ci.buildstream-remote-cache.yml b/.github/compose/ci.bst-artifact-server.yml
similarity index 82%
rename from .github/compose/ci.buildstream-remote-cache.yml
rename to .github/compose/ci.bst-artifact-server.yml
index 9f853fa..866f73b 100644
--- a/.github/compose/ci.buildstream-remote-cache.yml
+++ b/.github/compose/ci.bst-artifact-server.yml
@@ -11,8 +11,8 @@
# push: true
#
# Basic usage:
-# - docker-compose -f ci.buildstream-remote-cache.yml up
-# - docker-compose -f ci.buildstream-remote-cache.yml down
+# - docker-compose -f ci.bst-artifact-server.yml up
+# - docker-compose -f ci.bst-artifact-server.yml down
#
version: "3.2"
diff --git a/.github/compose/ci.buildbarn-remote-cache.yml b/.github/compose/ci.buildbarn.yml
similarity index 90%
rename from .github/compose/ci.buildbarn-remote-cache.yml
rename to .github/compose/ci.buildbarn.yml
index 349a289..c1e90e9 100644
--- a/.github/compose/ci.buildbarn-remote-cache.yml
+++ b/.github/compose/ci.buildbarn.yml
@@ -16,8 +16,8 @@
# push: true
#
# Basic usage:
-# - docker-compose -f ci.buildbarn-remote-cache.yml up
-# - docker-compose -f ci.buildbarn-remote-cache.yml down
+# - docker-compose -f ci.buildbarn.yml up
+# - docker-compose -f ci.buildbarn.yml down
version: '3.4'
diff --git a/.github/compose/ci.remote-execution.yml b/.github/compose/ci.buildgrid.yml
similarity index 94%
rename from .github/compose/ci.remote-execution.yml
rename to .github/compose/ci.buildgrid.yml
index e58a2a3..211c320 100644
--- a/.github/compose/ci.remote-execution.yml
+++ b/.github/compose/ci.buildgrid.yml
@@ -19,8 +19,8 @@
# url: http://localhost:50051
#
# Basic usage:
-# - docker-compose -f ci.remote-execution.yml up
-# - docker-compose -f ci.remote-execution.yml down
+# - docker-compose -f ci.buildgrid.yml up
+# - docker-compose -f ci.buildgrid.yml down
#
version: "3.2"
diff --git a/.github/compose/ci.docker-compose.yml b/.github/compose/ci.docker-compose.yml
index 2ef1a24..ee8d8c8 100644
--- a/.github/compose/ci.docker-compose.yml
+++ b/.github/compose/ci.docker-compose.yml
@@ -58,7 +58,7 @@ services:
environment:
BST_PLUGINS_EXPERIMENTAL_VERSION: master
- remote-execution:
+ buildgrid:
<<: *tests-template
command: tox -vvvvv -- --color=yes --remote-execution
environment:
@@ -72,7 +72,7 @@ services:
#
network_mode: host
- buildstream-remote-cache:
+ bst-artifact-server:
<<: *tests-template
command: tox -vvvvv -- --color=yes --remote-cache
environment:
@@ -84,7 +84,7 @@ services:
#
network_mode: host
- buildbarn-remote-cache:
+ buildbarn:
<<: *tests-template
command: tox -vvvvv -- --color=yes --remote-cache
environment:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d9401b4..894caaa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -87,9 +87,9 @@ jobs:
# and they also map to corresponding filenames of services which are expected
# to be run in the background
test-name:
- - remote-execution
- - buildstream-remote-cache
- - buildbarn-remote-cache
+ - bst-artifact-server
+ - buildbarn
+ - buildgrid
steps:
- name: Check out repository
[buildstream] 09/14: _sandboxremote.py: Make storage-service
optional with remote cache
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0de2a8ab8f26b24cc7a6f48077ef77046545f811
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Tue Oct 20 09:34:29 2020 +0200
_sandboxremote.py: Make storage-service optional with remote cache
---
src/buildstream/_context.py | 2 +-
src/buildstream/_remotespec.py | 22 +++++--
src/buildstream/sandbox/_sandboxremote.py | 102 ++++++++++++++++--------------
3 files changed, 72 insertions(+), 54 deletions(-)
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 8c409ec..88ac785 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -749,4 +749,4 @@ class Context:
os.environ["XDG_DATA_HOME"] = os.path.expanduser("~/.local/share")
def _load_remote_execution(self, node: MappingNode) -> Optional[RemoteExecutionSpec]:
- return RemoteExecutionSpec.new_from_node(node)
+ return RemoteExecutionSpec.new_from_node(node, remote_cache=bool(self.remote_cache_spec))
diff --git a/src/buildstream/_remotespec.py b/src/buildstream/_remotespec.py
index a37698e..ea8125a 100644
--- a/src/buildstream/_remotespec.py
+++ b/src/buildstream/_remotespec.py
@@ -454,9 +454,11 @@ class RemoteSpec:
# communicate with various components of an RE build cluster.
#
class RemoteExecutionSpec:
- def __init__(self, exec_spec: RemoteSpec, storage_spec: RemoteSpec, action_spec: Optional[RemoteSpec]) -> None:
+ def __init__(
+ self, exec_spec: RemoteSpec, storage_spec: Optional[RemoteSpec], action_spec: Optional[RemoteSpec]
+ ) -> None:
self.exec_spec: RemoteSpec = exec_spec
- self.storage_spec: RemoteSpec = storage_spec
+ self.storage_spec: Optional[RemoteSpec] = storage_spec
self.action_spec: Optional[RemoteSpec] = action_spec
# new_from_node():
@@ -474,15 +476,25 @@ class RemoteExecutionSpec:
# LoadError: If the node is malformed.
#
@classmethod
- def new_from_node(cls, node: MappingNode, basedir: Optional[str] = None) -> "RemoteExecutionSpec":
+ def new_from_node(
+ cls, node: MappingNode, basedir: Optional[str] = None, *, remote_cache: bool = False
+ ) -> "RemoteExecutionSpec":
node.validate_keys(["execution-service", "storage-service", "action-cache-service"])
exec_node = node.get_mapping("execution-service")
- storage_node = node.get_mapping("storage-service")
+ if remote_cache:
+ storage_node = node.get_mapping("storage-service", default=None)
+ else:
+ storage_node = node.get_mapping("storage-service")
action_node = node.get_mapping("action-cache-service", default=None)
exec_spec = RemoteSpec.new_from_node(exec_node, basedir, remote_execution=True)
- storage_spec = RemoteSpec.new_from_node(storage_node, basedir, remote_execution=True)
+
+ storage_spec: Optional[RemoteSpec]
+ if storage_node:
+ storage_spec = RemoteSpec.new_from_node(storage_node, basedir, remote_execution=True)
+ else:
+ storage_spec = None
action_spec: Optional[RemoteSpec]
if action_node:
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index e0a3a2d..d1e1c8f 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -42,6 +42,8 @@ class SandboxRemote(SandboxREAPI):
super().__init__(*args, **kwargs)
context = self._get_context()
+ cascache = context.get_cascache()
+
specs = context.remote_execution_specs
if specs is None:
return
@@ -51,6 +53,17 @@ class SandboxRemote(SandboxREAPI):
self.action_spec = specs.action_spec
self.operation_name = None
+ if self.storage_spec:
+ self.storage_remote = CASRemote(self.storage_spec, cascache)
+ try:
+ self.storage_remote.init()
+ except grpc.RpcError as e:
+ raise SandboxError(
+ "Failed to contact remote execution CAS endpoint at {}: {}".format(self.storage_spec.url, e)
+ ) from e
+ else:
+ self.storage_remote = cascache.get_default_remote()
+
def run_remote_command(self, channel, action_digest):
# Sends an execution request to the remote execution server.
#
@@ -141,13 +154,13 @@ class SandboxRemote(SandboxREAPI):
cascache = context.get_cascache()
# Fetch the file blobs
- dir_digest = vdir._get_digest()
- required_blobs = cascache.required_blobs_for_directory(dir_digest)
+ if self.storage_spec:
+ dir_digest = vdir._get_digest()
+ required_blobs = cascache.required_blobs_for_directory(dir_digest)
- local_missing_blobs = cascache.missing_blobs(required_blobs)
- if local_missing_blobs:
- with CASRemote(self.storage_spec, cascache) as casremote:
- cascache.fetch_blobs(casremote, local_missing_blobs)
+ local_missing_blobs = cascache.missing_blobs(required_blobs)
+ if local_missing_blobs:
+ cascache.fetch_blobs(self.storage_remote, local_missing_blobs)
def _execute_action(self, action, flags):
stdout, stderr = self._get_output()
@@ -159,46 +172,40 @@ class SandboxRemote(SandboxREAPI):
action_digest = cascache.add_object(buffer=action.SerializeToString())
+ casremote = self.storage_remote
+
# check action cache download and download if there
action_result = self._check_action_cache(action_digest)
if not action_result:
- with CASRemote(self.storage_spec, cascache) as casremote:
+ with self._get_context().messenger.timed_activity(
+ "Uploading input root", element_name=self._get_element_name()
+ ):
+ # Determine blobs missing on remote
try:
- casremote.init()
+ input_root_digest = action.input_root_digest
+ missing_blobs = list(cascache.missing_blobs_for_directory(input_root_digest, remote=casremote))
except grpc.RpcError as e:
- raise SandboxError(
- "Failed to contact remote execution CAS endpoint at {}: {}".format(self.storage_spec.url, e)
- ) from e
+ raise SandboxError("Failed to determine missing blobs: {}".format(e)) from e
- with self._get_context().messenger.timed_activity(
- "Uploading input root", element_name=self._get_element_name()
- ):
- # Determine blobs missing on remote
- try:
- input_root_digest = action.input_root_digest
- missing_blobs = list(cascache.missing_blobs_for_directory(input_root_digest, remote=casremote))
- except grpc.RpcError as e:
- raise SandboxError("Failed to determine missing blobs: {}".format(e)) from e
-
- # Check if any blobs are also missing locally (partial artifact)
- # and pull them from the artifact cache.
- try:
- local_missing_blobs = cascache.missing_blobs(missing_blobs)
- if local_missing_blobs:
- artifactcache.fetch_missing_blobs(project, local_missing_blobs)
- except (grpc.RpcError, BstError) as e:
- raise SandboxError("Failed to pull missing blobs from artifact cache: {}".format(e)) from e
-
- # Add command and action messages to blob list to push
- missing_blobs.append(action.command_digest)
- missing_blobs.append(action_digest)
-
- # Now, push the missing blobs to the remote.
- try:
- cascache.send_blobs(casremote, missing_blobs)
- except grpc.RpcError as e:
- raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e
+ # Check if any blobs are also missing locally (partial artifact)
+ # and pull them from the artifact cache.
+ try:
+ local_missing_blobs = cascache.missing_blobs(missing_blobs)
+ if local_missing_blobs:
+ artifactcache.fetch_missing_blobs(project, local_missing_blobs)
+ except (grpc.RpcError, BstError) as e:
+ raise SandboxError("Failed to pull missing blobs from artifact cache: {}".format(e)) from e
+
+ # Add command and action messages to blob list to push
+ missing_blobs.append(action.command_digest)
+ missing_blobs.append(action_digest)
+
+ # Now, push the missing blobs to the remote.
+ try:
+ cascache.send_blobs(casremote, missing_blobs)
+ except grpc.RpcError as e:
+ raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e
# Now request to execute the action
channel = self.exec_spec.open_channel()
@@ -207,17 +214,16 @@ class SandboxRemote(SandboxREAPI):
action_result = self._extract_action_result(operation)
# Fetch outputs
- with CASRemote(self.storage_spec, cascache) as casremote:
- for output_directory in action_result.output_directories:
- tree_digest = output_directory.tree_digest
- if tree_digest is None or not tree_digest.hash:
- raise SandboxError("Output directory structure had no digest attached.")
+ for output_directory in action_result.output_directories:
+ tree_digest = output_directory.tree_digest
+ if tree_digest is None or not tree_digest.hash:
+ raise SandboxError("Output directory structure had no digest attached.")
- # Now do a pull to ensure we have the full directory structure.
- cascache.pull_tree(casremote, tree_digest)
+ # Now do a pull to ensure we have the full directory structure.
+ cascache.pull_tree(casremote, tree_digest)
- # Fetch stdout and stderr blobs
- cascache.fetch_blobs(casremote, [action_result.stdout_digest, action_result.stderr_digest])
+ # Fetch stdout and stderr blobs
+ cascache.fetch_blobs(casremote, [action_result.stdout_digest, action_result.stderr_digest])
# Forward remote stdout and stderr
if stdout:
[buildstream] 06/14: _sandboxremote.py: Drop unused
output_files_required parameter
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8ef80817e71e3f324f875b9ad5c63703dd33a7e8
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Mon Oct 26 20:03:36 2020 +0100
_sandboxremote.py: Drop unused output_files_required parameter
---
src/buildstream/sandbox/_sandboxremote.py | 29 +++++++----------------------
1 file changed, 7 insertions(+), 22 deletions(-)
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index af50051..e0a3a2d 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -41,8 +41,6 @@ class SandboxRemote(SandboxREAPI):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self._output_files_required = kwargs.get("output_files_required", True)
-
context = self._get_context()
specs = context.remote_execution_specs
if specs is None:
@@ -140,29 +138,16 @@ class SandboxRemote(SandboxREAPI):
def _fetch_missing_blobs(self, vdir):
context = self._get_context()
- project = self._get_project()
cascache = context.get_cascache()
- artifactcache = context.artifactcache
- # Fetch the file blobs if needed
- if self._output_files_required or artifactcache.has_push_remotes():
- dir_digest = vdir._get_digest()
- required_blobs = cascache.required_blobs_for_directory(dir_digest)
+ # Fetch the file blobs
+ dir_digest = vdir._get_digest()
+ required_blobs = cascache.required_blobs_for_directory(dir_digest)
- local_missing_blobs = cascache.missing_blobs(required_blobs)
- if local_missing_blobs:
- if self._output_files_required:
- # Fetch all blobs from Remote Execution CAS server
- blobs_to_fetch = local_missing_blobs
- else:
- # Output files are not required in the local cache,
- # however, artifact push remotes will need them.
- # Only fetch blobs that are missing on one or multiple
- # artifact servers.
- blobs_to_fetch = artifactcache.find_missing_blobs(project, local_missing_blobs)
-
- with CASRemote(self.storage_spec, cascache) as casremote:
- cascache.fetch_blobs(casremote, blobs_to_fetch)
+ local_missing_blobs = cascache.missing_blobs(required_blobs)
+ if local_missing_blobs:
+ with CASRemote(self.storage_spec, cascache) as casremote:
+ cascache.fetch_blobs(casremote, local_missing_blobs)
def _execute_action(self, action, flags):
stdout, stderr = self._get_output()
[buildstream] 02/14: _artifact.py: Add cached and strong_key
parameters to set_cached()
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e5cf255cf89439c7ae8cef725e522a2829e7036c
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu Sep 10 19:14:50 2020 +0200
_artifact.py: Add cached and strong_key parameters to set_cached()
---
src/buildstream/_artifact.py | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index 28e2f59..51e1f57 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -616,10 +616,20 @@ class Artifact:
# Mark the artifact as cached without querying the filesystem.
# This is used as optimization when we know the artifact is available.
#
- def set_cached(self):
- self._proto = self._load_proto()
- assert self._proto
- self._cached = True
+ def set_cached(self, cached=True, *, strong_key=None):
+ if strong_key:
+ assert not self._cache_key or self._cache_key == strong_key
+ self._cache_key = strong_key
+
+ if cached:
+ self._proto = self._load_proto()
+ assert self._proto
+ if self._cache_key:
+ assert self._cache_key == self._proto.strong_key
+ else:
+ self._cache_key = self._proto.strong_key
+
+ self._cached = cached
# pull()
#
[buildstream] 10/14: _scheduler: Add CacheQueryQueue to support
cache query in job threads
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1bd00f1e64bc96aaf67f8edc198b45dbf3c717e7
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Tue Mar 2 11:36:33 2021 +0100
_scheduler: Add CacheQueryQueue to support cache query in job threads
This prepares parallelization of cache queries but does not enable it
yet.
---
src/buildstream/_scheduler/__init__.py | 1 +
.../_scheduler/queues/cachequeryqueue.py | 66 ++++++++++++++++++++++
2 files changed, 67 insertions(+)
diff --git a/src/buildstream/_scheduler/__init__.py b/src/buildstream/_scheduler/__init__.py
index d2f458f..fcde00d 100644
--- a/src/buildstream/_scheduler/__init__.py
+++ b/src/buildstream/_scheduler/__init__.py
@@ -25,6 +25,7 @@ from .queues.trackqueue import TrackQueue
from .queues.buildqueue import BuildQueue
from .queues.artifactpushqueue import ArtifactPushQueue
from .queues.pullqueue import PullQueue
+from .queues.cachequeryqueue import CacheQueryQueue
from .scheduler import Scheduler, SchedStatus
from .jobs import ElementJob, JobStatus
diff --git a/src/buildstream/_scheduler/queues/cachequeryqueue.py b/src/buildstream/_scheduler/queues/cachequeryqueue.py
new file mode 100644
index 0000000..b650a91
--- /dev/null
+++ b/src/buildstream/_scheduler/queues/cachequeryqueue.py
@@ -0,0 +1,66 @@
+#
+# Copyright (C) 2020 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+from . import Queue, QueueStatus
+from ..resources import ResourceType
+from ..jobs import JobStatus
+from ...types import _KeyStrength
+
+
+# A queue which queries the cache for artifacts and sources
+#
+class CacheQueryQueue(Queue):
+
+ action_name = "Cache-query"
+ complete_name = "Cache queried"
+ resources = [ResourceType.PROCESS, ResourceType.CACHE]
+
+ def __init__(self, scheduler, *, sources=False):
+ super().__init__(scheduler)
+
+ self._sources = sources
+
+ def get_process_func(self):
+ if not self._sources:
+ return CacheQueryQueue._query_artifacts_or_sources
+ else:
+ return CacheQueryQueue._query_sources
+
+ def status(self, element):
+ if not element._get_cache_key(strength=_KeyStrength.WEAK):
+ # Strict and weak cache keys are unavailable if the element or
+ # a dependency has an unresolved source
+ return QueueStatus.SKIP
+
+ return QueueStatus.READY
+
+ def done(self, _, element, result, status):
+ if status is JobStatus.FAIL:
+ return
+
+ if not self._sources:
+ if not element._pull_pending():
+ element._load_artifact_done()
+
+ @staticmethod
+ def _query_artifacts_or_sources(element):
+ element._load_artifact(pull=False)
+ if not element._can_query_cache() or not element._cached_success():
+ element._query_source_cache()
+
+ @staticmethod
+ def _query_sources(element):
+ element._query_source_cache()
[buildstream] 03/14: element.py: Move get_logs() to internal API
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bf2eecd467eb0f4e8afb9f5ec57d7c79307f96ab
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Wed Oct 28 11:10:24 2020 +0100
element.py: Move get_logs() to internal API
---
src/buildstream/_elementproxy.py | 3 ---
src/buildstream/_stream.py | 2 +-
src/buildstream/element.py | 18 ++++++++++--------
3 files changed, 11 insertions(+), 12 deletions(-)
diff --git a/src/buildstream/_elementproxy.py b/src/buildstream/_elementproxy.py
index a7b1f09..53780ea 100644
--- a/src/buildstream/_elementproxy.py
+++ b/src/buildstream/_elementproxy.py
@@ -146,9 +146,6 @@ class ElementProxy(PluginProxy):
def get_variable(self, varname: str) -> Optional[str]:
return cast("Element", self._plugin).get_variable(varname)
- def get_logs(self) -> List[str]:
- return cast("Element", self._plugin).get_logs()
-
##############################################################
# Element Internal APIs #
##############################################################
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index d03480e..520483b 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -796,7 +796,7 @@ class Stream:
self._context.messenger.warn("{} is cached without log files".format(ref))
continue
- artifact_logs[obj.name] = obj.get_logs()
+ artifact_logs[obj.name] = obj._get_logs()
return artifact_logs
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 8a484a2..bba3506 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -839,14 +839,6 @@ class Element(Plugin):
self.__batch_prepare_assemble_flags = flags
self.__batch_prepare_assemble_collect = collect
- def get_logs(self) -> List[str]:
- """Obtain a list of log file paths
-
- Returns:
- A list of log file paths
- """
- return cast(Artifact, self.__artifact).get_logs()
-
#############################################################
# Private Methods used in BuildStream #
#############################################################
@@ -2498,6 +2490,16 @@ class Element(Plugin):
self.__whitelist_regex = re.compile(expression)
return self.__whitelist_regex.match(os.path.join(os.sep, path))
+ # _get_logs()
+ #
+ # Obtain a list of log file paths
+ #
+ # Returns:
+ # A list of log file paths
+ #
+ def _get_logs(self) -> List[str]:
+ return cast(Artifact, self.__artifact).get_logs()
+
#############################################################
# Private Local Methods #
#############################################################
[buildstream] 04/14: Drop pull-artifact-files config option for
remote execution
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7ecbc41b9c5126a8fa47dbd5193a2d29d0aebf49
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Tue Oct 20 16:29:39 2020 +0200
Drop pull-artifact-files config option for remote execution
With `pull-artifact-files` disabled, artifact file blobs are completely
ignored in cache checks. While this allows remote execution builds
without downloading build outputs, it can result in build failures when
blobs expire on the server side. It also does not download missing blobs
on demand. I.e. `bst artifact checkout` and other commands don't work
with `pull-artifact-files` disabled.
This commit removes the `pull-artifact-files` config option due to the
issues above. It will be replaced by remote cache support, which only
downloads blobs as needed but still considers all file blobs in cache
checks for reliable builds. It will also support download of missing
blobs on demand.
---
doc/source/using_config.rst | 12 ------------
src/buildstream/_context.py | 26 +++++---------------------
src/buildstream/_frontend/widget.py | 1 -
src/buildstream/_stream.py | 8 +++-----
tests/remoteexecution/partial.py | 13 ++++---------
5 files changed, 12 insertions(+), 48 deletions(-)
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index acfff95..6e9fe24 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -900,7 +900,6 @@ using the ``remote-execution`` key, like so:
.. code:: yaml
remote-execution:
- pull-artifact-files: True
execution-service:
url: http://execution.fallback.example.com:50051
instance-name: main
@@ -918,16 +917,6 @@ using the ``remote-execution`` key, like so:
Attributes
''''''''''
-* ``pull-artifact-files``
-
- This determines whether you want the artifacts which were built remotely
- to be downloaded into the local CAS, so that it is ready for checkout
- directly after a built completes.
-
- If this is set to ``false``, then you will need to download the artifacts
- you intend to use with :ref:`bst artifact checkout <invoking_artifact_checkout>`
- after your build completes.
-
* ``execution-service``
A :ref:`service configuration <user_config_remote_execution_service>` specifying
@@ -1095,7 +1084,6 @@ the global configuration can be overridden on a per project basis in this projec
# remote execution configuration.
#
remote-execution:
- pull-artifact-files: True
execution-service:
url: http://execution.example.com:50051
instance-name: main
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 7a46caa..2c1f25a 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -175,9 +175,6 @@ class Context:
# User specified cache quota, used for display messages
self.config_cache_quota_string: Optional[str] = None
- # Whether to pull the files of an artifact when doing remote execution
- self.pull_artifact_files: bool = True
-
# Whether or not to attempt to pull build trees globally
self.pull_buildtrees: Optional[bool] = None
@@ -376,10 +373,10 @@ class Context:
cache_config = defaults.get_mapping("source-caches", default={})
self._global_source_cache_config = _CacheConfig.new_from_node(cache_config)
- # Load the global remote execution config including pull-artifact-files setting
+ # Load the global remote execution config
remote_execution = defaults.get_mapping("remote-execution", default=None)
if remote_execution:
- self.pull_artifact_files, self.remote_execution_specs = self._load_remote_execution(remote_execution)
+ self.remote_execution_specs = self._load_remote_execution(remote_execution)
# Load pull build trees configuration
self.pull_buildtrees = cache.get_bool("pull-buildtrees")
@@ -549,7 +546,7 @@ class Context:
override_node = self.get_overrides(project.name)
remote_execution = override_node.get_mapping("remote-execution", default=None)
if remote_execution:
- self.pull_artifact_files, self.remote_execution_specs = self._load_remote_execution(remote_execution)
+ self.remote_execution_specs = self._load_remote_execution(remote_execution)
#
# Maintain our list of remote specs for artifact and source caches
@@ -758,18 +755,5 @@ class Context:
if not os.environ.get("XDG_DATA_HOME"):
os.environ["XDG_DATA_HOME"] = os.path.expanduser("~/.local/share")
- def _load_remote_execution(self, node: MappingNode) -> Tuple[bool, Optional[RemoteExecutionSpec]]:
- # The pull_artifact_files attribute is special, it is allowed to
- # be set to False even if there is no remote execution service configured.
- #
- pull_artifact_files: bool = node.get_bool("pull-artifact-files", default=True)
- node.safe_del("pull-artifact-files")
-
- # Don't pass the remote execution settings if that was the only option
- remote_execution_specs: Optional[RemoteExecutionSpec]
- if node.keys():
- remote_execution_specs = RemoteExecutionSpec.new_from_node(node)
- else:
- remote_execution_specs = None
-
- return pull_artifact_files, remote_execution_specs
+ def _load_remote_execution(self, node: MappingNode) -> Optional[RemoteExecutionSpec]:
+ return RemoteExecutionSpec.new_from_node(node)
diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py
index 589e548..9994818 100644
--- a/src/buildstream/_frontend/widget.py
+++ b/src/buildstream/_frontend/widget.py
@@ -500,7 +500,6 @@ class LogLine(Widget):
values["Storage Service"] = format_spec(specs.storage_spec)
if specs.action_spec:
values["Action Cache Service"] = format_spec(specs.action_spec)
- values["Pull artifact files"] = context.pull_artifact_files
text += self._format_values(values)
# Print information about each loaded project
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 520483b..0559d8d 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -383,11 +383,9 @@ class Stream:
# Require artifact files only for target elements and their runtime dependencies.
self._context.set_artifact_files_optional()
- # fetch blobs of targets if options set
- if self._context.pull_artifact_files:
- scope = _Scope.ALL if selection == _PipelineSelection.ALL else _Scope.RUN
- for element in self.targets:
- element._set_artifact_files_required(scope=scope)
+ scope = _Scope.ALL if selection == _PipelineSelection.ALL else _Scope.RUN
+ for element in self.targets:
+ element._set_artifact_files_required(scope=scope)
source_push_enabled = self._sourcecache.has_push_remotes()
diff --git a/tests/remoteexecution/partial.py b/tests/remoteexecution/partial.py
index a47abee..cd87293 100644
--- a/tests/remoteexecution/partial.py
+++ b/tests/remoteexecution/partial.py
@@ -20,9 +20,8 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
# Test that `bst build` does not download file blobs of a build-only dependency
# to the local cache.
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("pull_artifact_files", [True, False])
@pytest.mark.parametrize("build_all", [True, False])
-def test_build_dependency_partial_local_cas(cli, datafiles, pull_artifact_files, build_all):
+def test_build_dependency_partial_local_cas(cli, datafiles, build_all):
project = str(datafiles)
element_name = "no-runtime-deps.bst"
builddep_element_name = "autotools/amhello.bst"
@@ -35,24 +34,20 @@ def test_build_dependency_partial_local_cas(cli, datafiles, pull_artifact_files,
# configure pull blobs
if build_all:
cli.configure({"build": {"dependencies": "all"}})
- cli.config["remote-execution"]["pull-artifact-files"] = pull_artifact_files
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
# Verify artifact is pulled bar files when ensure artifact files is set
result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
- if pull_artifact_files:
- result.assert_success()
- assert_contains(checkout, ["/test"])
- else:
- result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
+ result.assert_success()
+ assert_contains(checkout, ["/test"])
# Verify build dependencies is pulled for ALL and BUILD
result = cli.run(
project=project, args=["artifact", "checkout", builddep_element_name, "--directory", builddep_checkout]
)
- if build_all and pull_artifact_files:
+ if build_all:
result.assert_success()
else:
result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt")
[buildstream] 14/14: TEMP: drop local tests
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b12c13f6dbd85f86bae0ec6bc185968f2505118d
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu May 20 10:08:59 2021 +0200
TEMP: drop local tests
---
.github/workflows/ci.yml | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 894caaa..5dfb08b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -44,20 +44,9 @@ jobs:
# The names here should map to a valid service defined in
# "../compose/ci.docker-compose.yml"
test-name:
- - debian-10
- - fedora-32
- - fedora-33
- - ubuntu-18.04
- - centos-7.7.1908
- - fedora-missing-deps
- - no-usedevelop
- lint
- mypy
- include:
- - test-name: plugins-master
- allow-failure: true
-
steps:
- name: Check out repository
uses: actions/checkout@v2
[buildstream] 08/14: _context.py: Add support for remote-cache
configuration
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 977180ae3cca4d636599c82df27a7e5cac8f452d
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Mon Oct 19 16:18:34 2020 +0200
_context.py: Add support for remote-cache configuration
---
src/buildstream/_context.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index d5d0669..8c409ec 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -175,6 +175,9 @@ class Context:
# User specified cache quota, used for display messages
self.config_cache_quota_string: Optional[str] = None
+ # Remote cache server
+ self.remote_cache_spec: Optional[RemoteSpec] = None
+
# Whether or not to attempt to pull build trees globally
self.pull_buildtrees: Optional[bool] = None
@@ -343,7 +346,7 @@ class Context:
# We need to find the first existing directory in the path of our
# casdir - the casdir may not have been created yet.
cache = defaults.get_mapping("cache")
- cache.validate_keys(["quota", "pull-buildtrees", "cache-buildtrees"])
+ cache.validate_keys(["quota", "remote-cache", "pull-buildtrees", "cache-buildtrees"])
cas_volume = self.casdir
while not os.path.exists(cas_volume):
@@ -359,6 +362,10 @@ class Context:
LoadErrorReason.INVALID_DATA,
) from e
+ remote_cache = cache.get_mapping("remote-cache", default=None)
+ if remote_cache:
+ self.remote_cache_spec = RemoteSpec.new_from_node(remote_cache)
+
# Load global artifact cache configuration
cache_config = defaults.get_mapping("artifacts", default={})
self._global_artifact_cache_config = _CacheConfig.new_from_node(cache_config)
@@ -661,6 +668,7 @@ class Context:
self.cachedir,
casd=self.use_casd,
cache_quota=self.config_cache_quota,
+ remote_cache_spec=self.remote_cache_spec,
log_level=log_level,
log_directory=self.logdir,
)
[buildstream] 13/14: _stream.py: Parallelize cache queries if a
remote cache is configured
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/remote-cache-ci
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 74c8a9010263ed3cd6a1509c43a12c2573d2d050
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Thu Mar 4 13:29:53 2021 +0100
_stream.py: Parallelize cache queries if a remote cache is configured
---
src/buildstream/_stream.py | 38 +++++++++++++++++++++++++-------------
1 file changed, 25 insertions(+), 13 deletions(-)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index e22b93c..436da74 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -38,6 +38,7 @@ from ._scheduler import (
Scheduler,
SchedStatus,
TrackQueue,
+ CacheQueryQueue,
FetchQueue,
SourcePushQueue,
BuildQueue,
@@ -206,20 +207,31 @@ class Stream:
# Enqueue complete build plan as this is required to determine `buildable` status.
plan = list(_pipeline.dependencies(elements, _Scope.ALL))
- for element in plan:
- if element._can_query_cache():
- # Cache status already available.
- # This is the case for artifact elements, which load the
- # artifact early on.
- pass
- elif not only_sources and element._get_cache_key(strength=_KeyStrength.WEAK):
- element._load_artifact(pull=False)
- if sources_of_cached_elements or not element._can_query_cache() or not element._cached_success():
+ if self._context.remote_cache_spec:
+ # Parallelize cache queries if a remote cache is configured
+ self._reset()
+ self._add_queue(CacheQueryQueue(self._scheduler, sources=only_sources), track=True)
+ self._enqueue_plan(plan)
+ self._run()
+ else:
+ for element in plan:
+ if element._can_query_cache():
+ # Cache status already available.
+ # This is the case for artifact elements, which load the
+ # artifact early on.
+ pass
+ elif not only_sources and element._get_cache_key(strength=_KeyStrength.WEAK):
+ element._load_artifact(pull=False)
+ if (
+ sources_of_cached_elements
+ or not element._can_query_cache()
+ or not element._cached_success()
+ ):
+ element._query_source_cache()
+ if not element._pull_pending():
+ element._load_artifact_done()
+ elif element._has_all_sources_resolved():
element._query_source_cache()
- if not element._pull_pending():
- element._load_artifact_done()
- elif element._has_all_sources_resolved():
- element._query_source_cache()
# shell()
#