You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ak...@apache.org on 2025/09/16 20:57:14 UTC
(buildstream) branch abderrahim/cascache-cleanup created (now bbbe9ddcf)
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a change to branch abderrahim/cascache-cleanup
in repository https://gitbox.apache.org/repos/asf/buildstream.git
at bbbe9ddcf Use CASDProcessManager directly to access REAPI stubs
This branch includes the following new commits:
new 91877f5a6 tests/artifactcache/pull: stop using CASRemote.push_message()
new 6f5d61a59 casremote: drop the push_message method
new bbbe9ddcf Use CASDProcessManager directly to access REAPI stubs
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(buildstream) 02/03: casremote: drop the push_message method
Posted by ak...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/cascache-cleanup
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6f5d61a59ac294b3b850c042db144b089f2a94fa
Author: Abderrahim Kitouni <ak...@gnome.org>
AuthorDate: Sun Apr 27 16:59:10 2025 +0100
casremote: drop the push_message method
It's now unused
---
src/buildstream/_cas/casremote.py | 18 -------------
tests/artifactcache/push.py | 55 ---------------------------------------
2 files changed, 73 deletions(-)
diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py
index ae4aa9004..2db8033e4 100644
--- a/src/buildstream/_cas/casremote.py
+++ b/src/buildstream/_cas/casremote.py
@@ -61,24 +61,6 @@ class CASRemote(BaseRemote):
response = local_cas.GetInstanceNameForRemotes(request)
self.local_cas_instance_name = response.instance_name
- # push_message():
- #
- # Push the given protobuf message to a remote.
- #
- # Args:
- # message (Message): A protobuf message to push.
- #
- # Raises:
- # (CASRemoteError): if there was an error
- #
- def push_message(self, message):
-
- message_buffer = message.SerializeToString()
-
- self.init()
-
- return self.cascache.add_object(buffer=message_buffer, instance_name=self.local_cas_instance_name)
-
# Represents a batch of blobs queued for fetching.
#
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 7c2160f6e..2b6f9274a 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -131,58 +131,3 @@ def test_push_split(cli, tmpdir, datafiles):
cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key)
)
assert storage.get_cas_files(proto) is not None
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_push_message(tmpdir, datafiles):
- project_dir = str(datafiles)
-
- # Set up an artifact cache.
- artifactshare = os.path.join(str(tmpdir), "artifactshare")
- with create_artifact_share(artifactshare) as share:
- # Configure artifact share
- rootcache_dir = os.path.join(str(tmpdir), "cache")
- user_config_file = str(tmpdir.join("buildstream.conf"))
- user_config = {
- "scheduler": {"pushers": 1},
- "artifacts": {
- "servers": [
- {
- "url": share.repo,
- "push": True,
- }
- ]
- },
- "cachedir": rootcache_dir,
- }
-
- # Write down the user configuration file
- _yaml.roundtrip_dump(user_config, file=user_config_file)
-
- with dummy_context(config=user_config_file) as context:
- # Load the project manually
- project = Project(project_dir, context)
- project.ensure_fully_loaded()
-
- # Create a local artifact cache handle
- artifactcache = context.artifactcache
-
- # Initialize remotes
- context.initialize_remotes(True, True, None, None)
- assert artifactcache.has_push_remotes()
-
- command = remote_execution_pb2.Command(
- arguments=["/usr/bin/gcc", "--help"],
- working_directory="/buildstream-build",
- output_directories=["/buildstream-install"],
- )
-
- # Push the message object
- _, remotes = artifactcache.get_remotes(project.name, True)
- assert len(remotes) == 1
- command_digest = remotes[0].push_message(command)
- message_hash, message_size = command_digest.hash, command_digest.size_bytes
-
- assert message_hash and message_size
- message_digest = remote_execution_pb2.Digest(hash=message_hash, size_bytes=message_size)
- assert share.has_object(message_digest)
(buildstream) 03/03: Use CASDProcessManager directly to access REAPI stubs
Posted by ak...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/cascache-cleanup
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bbbe9ddcf97f19fcf84533d020db897c6d33ffb6
Author: Abderrahim Kitouni <ak...@gnome.org>
AuthorDate: Sun Apr 27 19:51:52 2025 +0100
Use CASDProcessManager directly to access REAPI stubs
This removes some indirections through CASCache, now that
CASDProcessManager is directly available from the Context
---
src/buildstream/_assetcache.py | 8 ++---
src/buildstream/_cas/cascache.py | 45 ++++++--------------------
src/buildstream/_cas/casremote.py | 10 +++---
src/buildstream/_stream.py | 2 +-
src/buildstream/sandbox/_reremote.py | 13 ++++----
src/buildstream/sandbox/_sandboxbuildboxrun.py | 7 ++--
src/buildstream/sandbox/_sandboxremote.py | 4 +--
7 files changed, 30 insertions(+), 59 deletions(-)
diff --git a/src/buildstream/_assetcache.py b/src/buildstream/_assetcache.py
index df70fc2ff..8e883fdfd 100644
--- a/src/buildstream/_assetcache.py
+++ b/src/buildstream/_assetcache.py
@@ -20,7 +20,7 @@ from typing import List, Dict, Tuple, Iterable, Optional
import grpc
from . import utils
-from ._cas import CASRemote, CASCache, CASDProcessManager
+from ._cas import CASRemote, CASDProcessManager
from ._exceptions import AssetCacheError, RemoteError
from ._remotespec import RemoteSpec, RemoteType
from ._remote import BaseRemote
@@ -264,7 +264,7 @@ class AssetRemote(BaseRemote):
# to establish a connection to this remote at initialization time.
#
class RemotePair:
- def __init__(self, casd: CASDProcessManager, cas: CASCache, spec: RemoteSpec):
+ def __init__(self, casd: CASDProcessManager, spec: RemoteSpec):
self.index: Optional[AssetRemote] = None
self.storage: Optional[CASRemote] = None
self.error: Optional[str] = None
@@ -275,7 +275,7 @@ class RemotePair:
index.check()
self.index = index
if spec.remote_type in [RemoteType.STORAGE, RemoteType.ALL]:
- storage = CASRemote(spec, cas)
+ storage = CASRemote(spec, casd)
storage.check()
self.storage = storage
except RemoteError as e:
@@ -322,7 +322,7 @@ class AssetCache:
if spec in self._remotes:
continue
- remote = RemotePair(casd, self.cas, spec)
+ remote = RemotePair(casd, spec)
if remote.error:
self.context.messenger.warn("Failed to initialize remote {}: {}".format(spec.url, remote.error))
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index fbfc150f3..68fd4b610 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -71,25 +71,9 @@ class CASCache:
else:
assert not self._remote_cache
- self._default_remote = CASRemote(None, self)
+ self._default_remote = CASRemote(None, casd)
self._default_remote.init()
- # get_cas():
- #
- # Return ContentAddressableStorage stub for buildbox-casd channel.
- #
- def get_cas(self):
- assert self._casd, "CASCache was created without buildbox-casd"
- return self._casd.get_cas()
-
- # get_local_cas():
- #
- # Return LocalCAS stub for buildbox-casd channel.
- #
- def get_local_cas(self):
- assert self._casd, "CASCache was created without buildbox-casd"
- return self._casd.get_local_cas()
-
# preflight():
#
# Preflight check.
@@ -133,7 +117,7 @@ class CASCache:
# Returns: True if the directory is available in the local cache
#
def contains_directory(self, digest):
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
# Without a remote cache, `FetchTree` simply checks the local cache.
request = local_cas_pb2.FetchTreeRequest()
@@ -241,7 +225,7 @@ class CASCache:
#
def ensure_tree(self, tree):
if self._remote_cache:
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.FetchTreeRequest()
request.root_digest.CopyFrom(tree)
@@ -260,7 +244,7 @@ class CASCache:
# dir_digest (Digest): Digest object for the directory to fetch.
#
def fetch_directory(self, remote, dir_digest):
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.FetchTreeRequest()
request.instance_name = remote.local_cas_instance_name
@@ -399,7 +383,7 @@ class CASCache:
for path in paths:
request.path.append(path)
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
response = local_cas.CaptureFiles(request)
@@ -432,7 +416,7 @@ class CASCache:
# (Digest): The digest of the imported directory
#
def import_directory(self, path: str, properties: Optional[List[str]] = None) -> SourceRef:
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.CaptureTreeRequest()
request.path.append(path)
@@ -478,7 +462,7 @@ class CASCache:
#
@contextlib.contextmanager
def stage_directory(self, directory_digest):
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.StageTreeRequest()
request.root_digest.CopyFrom(directory_digest)
@@ -535,7 +519,7 @@ class CASCache:
# Returns: List of missing Digest objects
#
def missing_blobs(self, blobs, *, remote=None):
- cas = self.get_cas()
+ cas = self._casd.get_cas()
if remote:
instance_name = remote.local_cas_instance_name
@@ -576,7 +560,7 @@ class CASCache:
if self._remote_cache and _fetch_tree:
# Ensure we have the directory protos in the local cache
- local_cas = self.get_local_cas()
+ local_cas = self._casd.get_local_cas()
request = local_cas_pb2.FetchTreeRequest()
request.root_digest.CopyFrom(directory_digest)
@@ -719,17 +703,6 @@ class CASCache:
def get_cache_usage(self):
return self._cache_usage_monitor.get_cache_usage()
- # get_casd()
- #
- # Get the underlying buildbox-casd process
- #
- # Returns:
- # (subprocess.Process): The casd process that is used for the current cascache
- #
- def get_casd(self):
- assert self._casd is not None, "Only call this with a running buildbox-casd process"
- return self._casd
-
# _CASCacheUsage
#
diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py
index 2db8033e4..3fefb9d47 100644
--- a/src/buildstream/_cas/casremote.py
+++ b/src/buildstream/_cas/casremote.py
@@ -37,10 +37,10 @@ class BlobNotFound(CASRemoteError):
# Represents a single remote CAS cache.
#
class CASRemote(BaseRemote):
- def __init__(self, spec, cascache, **kwargs):
+ def __init__(self, spec, casd, **kwargs):
super().__init__(spec, **kwargs)
- self.cascache = cascache
+ self.casd = casd
self.local_cas_instance_name = None
# check_remote
@@ -55,7 +55,7 @@ class CASRemote(BaseRemote):
self.local_cas_instance_name = ""
return
- local_cas = self.cascache.get_local_cas()
+ local_cas = self.casd.get_local_cas()
request = local_cas_pb2.GetInstanceNameForRemotesRequest()
self.spec.to_localcas_remote(request.content_addressable_storage)
response = local_cas.GetInstanceNameForRemotes(request)
@@ -89,7 +89,7 @@ class _CASBatchRead:
if not self._requests:
return
- local_cas = self._remote.cascache.get_local_cas()
+ local_cas = self._remote.casd.get_local_cas()
for request in self._requests:
batch_response = local_cas.FetchMissingBlobs(request)
@@ -143,7 +143,7 @@ class _CASBatchUpdate:
if not self._requests:
return
- local_cas = self._remote.cascache.get_local_cas()
+ local_cas = self._remote.casd.get_local_cas()
for request in self._requests:
batch_response = local_cas.UploadMissingBlobs(request)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index da8d48550..5b7de307e 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -1810,7 +1810,7 @@ class Stream:
self._session_start_callback()
self._running = True
- status = self._scheduler.run(self.queues, self._context.get_cascache().get_casd())
+ status = self._scheduler.run(self.queues, self._context.get_casd())
self._running = False
if status == SchedStatus.ERROR:
diff --git a/src/buildstream/sandbox/_reremote.py b/src/buildstream/sandbox/_reremote.py
index 79853fa34..923a93efd 100644
--- a/src/buildstream/sandbox/_reremote.py
+++ b/src/buildstream/sandbox/_reremote.py
@@ -22,8 +22,8 @@ from .._exceptions import RemoteError
class RERemote(CASRemote):
- def __init__(self, cas_spec, remote_execution_specs, cascache):
- super().__init__(cas_spec, cascache)
+ def __init__(self, cas_spec, remote_execution_specs, casd):
+ super().__init__(cas_spec, casd)
self.remote_execution_specs = remote_execution_specs
self.exec_service = None
@@ -31,7 +31,7 @@ class RERemote(CASRemote):
self.ac_service = None
def _configure_protocols(self):
- local_cas = self.cascache.get_local_cas()
+ local_cas = self.casd.get_local_cas()
request = local_cas_pb2.GetInstanceNameForRemotesRequest()
if self.remote_execution_specs.storage_spec:
self.remote_execution_specs.storage_spec.to_localcas_remote(request.content_addressable_storage)
@@ -50,10 +50,9 @@ class RERemote(CASRemote):
response = local_cas.GetInstanceNameForRemotes(request)
self.local_cas_instance_name = response.instance_name
- casd = self.cascache.get_casd()
- self.exec_service = casd.get_exec_service()
- self.operations_service = casd.get_operations_service()
- self.ac_service = casd.get_ac_service()
+ self.exec_service = self.casd.get_exec_service()
+ self.operations_service = self.casd.get_operations_service()
+ self.ac_service = self.casd.get_ac_service()
def _check(self):
super()._check()
diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py b/src/buildstream/sandbox/_sandboxbuildboxrun.py
index c87642f6b..007ae95b4 100644
--- a/src/buildstream/sandbox/_sandboxbuildboxrun.py
+++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py
@@ -38,11 +38,11 @@ class SandboxBuildBoxRun(SandboxREAPI):
super().__init__(*args, **kwargs)
context = self._get_context()
- cascache = context.get_cascache()
+ casd = context.get_casd()
re_specs = context.remote_execution_specs
if re_specs and re_specs.action_spec:
- self.re_remote = RERemote(context.remote_cache_spec, re_specs, cascache)
+ self.re_remote = RERemote(context.remote_cache_spec, re_specs, casd)
try:
self.re_remote.init()
self.re_remote.check()
@@ -110,8 +110,7 @@ class SandboxBuildBoxRun(SandboxREAPI):
stdout, stderr = self._get_output()
context = self._get_context()
- cascache = context.get_cascache()
- casd = cascache.get_casd()
+ casd = context.get_casd()
config = self._get_config()
if config.remote_apis_socket_path and context.remote_cache_spec and not self.re_remote:
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index 8072938bf..28d6fcb32 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -37,7 +37,7 @@ class SandboxRemote(SandboxREAPI):
super().__init__(*args, **kwargs)
context = self._get_context()
- cascache = context.get_cascache()
+ casd = context.get_casd()
specs = context.remote_execution_specs
if specs is None or specs.exec_spec is None:
@@ -48,7 +48,7 @@ class SandboxRemote(SandboxREAPI):
self.action_spec = specs.action_spec
self.operation_name = None
- self.re_remote = RERemote(context.remote_cache_spec, specs, cascache)
+ self.re_remote = RERemote(context.remote_cache_spec, specs, casd)
try:
self.re_remote.init()
except grpc.RpcError as e:
(buildstream) 01/03: tests/artifactcache/pull: stop using CASRemote.push_message()
Posted by ak...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
akitouni pushed a commit to branch abderrahim/cascache-cleanup
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 91877f5a6e13b088fd9f859193d2c65d74c9ee2d
Author: Abderrahim Kitouni <ak...@gnome.org>
AuthorDate: Sun Apr 27 16:57:52 2025 +0100
tests/artifactcache/pull: stop using CASRemote.push_message()
This is the only thing that uses this function
---
tests/artifactcache/pull.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index 114957b85..882134be3 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -195,7 +195,10 @@ def test_pull_tree(cli, tmpdir, datafiles):
# Push the Tree as a regular message
_, remotes = artifactcache.get_remotes(project.name, True)
assert len(remotes) == 1
- tree_digest = remotes[0].push_message(tree)
+
+ remotes[0].init()
+ tree_digest = cas.add_object(buffer=tree.SerializeToString(), instance_name=remotes[0].local_cas_instance_name)
+
tree_hash, tree_size = tree_digest.hash, tree_digest.size_bytes
assert tree_hash and tree_size