You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ju...@apache.org on 2025/05/23 10:45:22 UTC
(buildstream) branch jbilleter/nested-reapi updated (d8b5e5003 -> 403d580e2)
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a change to branch jbilleter/nested-reapi
in repository https://gitbox.apache.org/repos/asf/buildstream.git
discard d8b5e5003 Enable local execution in buildbox-casd to support nested execution
discard 820c8515c _context.py: Add `effective_build_max_jobs` helper
discard 842129e21 sandbox: Add support for `remote-apis-socket`
new 32d527f5f sandbox: Add support for `remote-apis-socket`
new 467030f65 _context.py: Add `effective_build_max_jobs` helper
new 403d580e2 Enable local execution in buildbox-casd to support nested execution
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d8b5e5003)
\
N -- N -- N refs/heads/jbilleter/nested-reapi (403d580e2)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
(buildstream) 02/03: _context.py: Add `effective_build_max_jobs` helper
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch jbilleter/nested-reapi
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 467030f655d30f5aa2a783a88035f40a817adf98
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Fri May 23 12:36:10 2025 +0200
_context.py: Add `effective_build_max_jobs` helper
---
src/buildstream/_context.py | 12 ++++++++++++
src/buildstream/_project.py | 11 +----------
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 6b0cf4123..e125e3c89 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -531,6 +531,18 @@ class Context:
return self._sourcecache
+ @property
+ def effective_build_max_jobs(self) -> int:
+ # Based on some testing (mainly on AWS), maximum effective
+ # max-jobs value seems to be around 8-10 if we have enough cores
+ # users should set values based on workload and build infrastructure
+ if self.build_max_jobs == 0:
+ # User requested automatic max-jobs
+ return self.platform.get_cpu_count(8)
+ else:
+ # User requested explicit max-jobs setting
+ return self.build_max_jobs
+
# add_project():
#
# Add a project to the context.
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index b55f257fa..334462df0 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -1075,16 +1075,7 @@ class Project:
# Extend variables with automatic variables and option exports
# Initialize it as a string as all variables are processed as strings.
- # Based on some testing (mainly on AWS), maximum effective
- # max-jobs value seems to be around 8-10 if we have enough cores
- # users should set values based on workload and build infrastructure
- if self._context.build_max_jobs == 0:
- # User requested automatic max-jobs
- platform = self._context.platform
- output.base_variables["max-jobs"] = str(platform.get_cpu_count(8))
- else:
- # User requested explicit max-jobs setting
- output.base_variables["max-jobs"] = str(self._context.build_max_jobs)
+ output.base_variables["max-jobs"] = str(self._context.effective_build_max_jobs)
# Export options into variables, if that was requested
output.options.export_variables(output.base_variables)
(buildstream) 03/03: Enable local execution in buildbox-casd to support nested execution
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch jbilleter/nested-reapi
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 403d580e25d5dc7e02bb9241889e26ee3ab4699e
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Fri Apr 11 14:12:09 2025 +0200
Enable local execution in buildbox-casd to support nested execution
This bumps the minimum version of buildbox-casd to 1.2.0.
---
src/buildstream/_cas/casdprocessmanager.py | 20 ++++++++++++++++----
src/buildstream/_context.py | 1 +
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py
index 899a5473d..78934b7cb 100644
--- a/src/buildstream/_cas/casdprocessmanager.py
+++ b/src/buildstream/_cas/casdprocessmanager.py
@@ -45,9 +45,9 @@ _CASD_TIMEOUT = 300 # in seconds
#
# Minimum required version of buildbox-casd
#
-_REQUIRED_CASD_MAJOR = 0
-_REQUIRED_CASD_MINOR = 0
-_REQUIRED_CASD_MICRO = 58
+_REQUIRED_CASD_MAJOR = 1
+_REQUIRED_CASD_MINOR = 2
+_REQUIRED_CASD_MICRO = 0
# CASDProcessManager
@@ -76,7 +76,8 @@ class CASDProcessManager:
messenger,
*,
reserved=None,
- low_watermark=None
+ low_watermark=None,
+ local_jobs=None
):
os.makedirs(path, exist_ok=True)
@@ -104,6 +105,17 @@ class CASDProcessManager:
if protect_session_blobs:
casd_args.append("--protect-session-blobs")
+ if local_jobs is not None:
+ try:
+ buildbox_run = utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox")
+ casd_args.append("--buildbox-run={}".format(buildbox_run))
+ casd_args.append("--jobs={}".format(local_jobs))
+ except utils.ProgramNotFoundError as e:
+ # Not fatal as buildbox-run is not needed for remote execution
+ # and buildbox-casd local execution will never be used if
+ # buildbox-run is not available.
+ pass
+
if remote_cache_spec:
casd_args.append("--cas-remote={}".format(remote_cache_spec.url))
if remote_cache_spec.instance_name:
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index e125e3c89..87675a3c8 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -749,6 +749,7 @@ class Context:
messenger=self.messenger,
reserved=self.config_cache_reserved,
low_watermark=self.config_cache_low_watermark,
+ local_jobs=self.sched_builders * self.effective_build_max_jobs,
)
return self._casd
(buildstream) 01/03: sandbox: Add support for `remote-apis-socket`
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch jbilleter/nested-reapi
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 32d527f5f1bd9e829cc2bbf750cb4eed2067f993
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Fri May 2 16:57:24 2025 +0200
sandbox: Add support for `remote-apis-socket`
Fixes #1985.
---
src/buildstream/sandbox/_config.py | 33 ++++++++++++++++++++++----
src/buildstream/sandbox/_sandboxbuildboxrun.py | 3 +++
src/buildstream/sandbox/_sandboxreapi.py | 3 +++
3 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/src/buildstream/sandbox/_config.py b/src/buildstream/sandbox/_config.py
index 654f16454..587e7a4ca 100644
--- a/src/buildstream/sandbox/_config.py
+++ b/src/buildstream/sandbox/_config.py
@@ -45,12 +45,19 @@ if TYPE_CHECKING:
#
class SandboxConfig:
def __init__(
- self, *, build_os: str, build_arch: str, build_uid: Optional[int] = None, build_gid: Optional[int] = None
+ self,
+ *,
+ build_os: str,
+ build_arch: str,
+ build_uid: Optional[int] = None,
+ build_gid: Optional[int] = None,
+ remote_apis_socket_path: Optional[str] = None
):
self.build_os = build_os
self.build_arch = build_arch
self.build_uid = build_uid
self.build_gid = build_gid
+ self.remote_apis_socket_path = remote_apis_socket_path
# to_dict():
#
@@ -66,7 +73,7 @@ class SandboxConfig:
# Returns:
# A dictionary representation of this SandboxConfig
#
- def to_dict(self) -> Dict[str, Union[str, int]]:
+ def to_dict(self) -> Dict[str, Union[str, int, bool]]:
# Assign mandatory portions of the sandbox configuration
#
@@ -74,7 +81,7 @@ class SandboxConfig:
# the sandbox configuration, as that would result in
# breaking cache key stability.
#
- sandbox_dict: Dict[str, Union[str, int]] = {"build-os": self.build_os, "build-arch": self.build_arch}
+ sandbox_dict: Dict[str, Union[str, int, bool]] = {"build-os": self.build_os, "build-arch": self.build_arch}
# Assign optional portions of the sandbox configuration
#
@@ -87,6 +94,9 @@ class SandboxConfig:
if self.build_gid is not None:
sandbox_dict["build-gid"] = self.build_gid
+ if self.remote_apis_socket_path is not None:
+ sandbox_dict["remote-apis-socket-path"] = self.remote_apis_socket_path
+
return sandbox_dict
# new_from_node():
@@ -108,7 +118,7 @@ class SandboxConfig:
#
@classmethod
def new_from_node(cls, config: "MappingNode[Node]", *, platform: Optional[Platform] = None) -> "SandboxConfig":
- config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch"])
+ config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch", "remote-apis-socket"])
build_os: str
build_arch: str
@@ -132,4 +142,17 @@ class SandboxConfig:
build_uid = config.get_int("build-uid", None)
build_gid = config.get_int("build-gid", None)
- return cls(build_os=build_os, build_arch=build_arch, build_uid=build_uid, build_gid=build_gid)
+ remote_apis_socket = config.get_mapping("remote-apis-socket", default=None)
+ if remote_apis_socket:
+ remote_apis_socket.validate_keys(["path"])
+ remote_apis_socket_path = remote_apis_socket.get_str("path")
+ else:
+ remote_apis_socket_path = None
+
+ return cls(
+ build_os=build_os,
+ build_arch=build_arch,
+ build_uid=build_uid,
+ build_gid=build_gid,
+ remote_apis_socket_path=remote_apis_socket_path,
+ )
diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py b/src/buildstream/sandbox/_sandboxbuildboxrun.py
index 25d200262..5ba3d7727 100644
--- a/src/buildstream/sandbox/_sandboxbuildboxrun.py
+++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py
@@ -83,6 +83,9 @@ class SandboxBuildBoxRun(SandboxREAPI):
if config.build_gid is not None and "platform:unixGID" not in cls._capabilities:
raise SandboxUnavailableError("Configuring sandbox GID is not supported by buildbox-run.")
+ if config.remote_apis_socket_path is not None and "platform:remoteApisSocketPath" not in cls._capabilities:
+ raise SandboxUnavailableError("Configuring Remote APIs socket path is not supported by buildbox-run.")
+
def _execute_action(self, action, flags):
stdout, stderr = self._get_output()
diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py
index 17997b355..262cb3f04 100644
--- a/src/buildstream/sandbox/_sandboxreapi.py
+++ b/src/buildstream/sandbox/_sandboxreapi.py
@@ -131,6 +131,9 @@ class SandboxREAPI(Sandbox):
if flags & _SandboxFlags.NETWORK_ENABLED:
platform_dict["network"] = "on"
+ if config.remote_apis_socket_path:
+ platform_dict["remoteApisSocketPath"] = config.remote_apis_socket_path.lstrip(os.path.sep)
+
# Create Platform message with properties sorted by name in code point order
platform = remote_execution_pb2.Platform()
for key, value in sorted(platform_dict.items()):