You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:44:02 UTC
[buildstream] 02/04: Add support for user remote execution
configuration
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch valentindavid/remote_execution_configuration_command_line
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8f0e52314769beb0d7b8cf461950996b4812b09c
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Dec 20 12:59:43 2018 +0100
Add support for user remote execution configuration
Fixes #631.
---
buildstream/_context.py | 11 ++++++--
buildstream/_project.py | 11 +++++++-
buildstream/sandbox/_sandboxremote.py | 2 +-
doc/source/format_project.rst | 4 +++
doc/source/using_config.rst | 48 +++++++++++++++++++++++++++++++++++
5 files changed, 72 insertions(+), 4 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index e0eea99..c62755c 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache
from ._artifactcache.cascache import CASCache
from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
from .plugin import _plugin_lookup
+from .sandbox import SandboxRemote
# Context()
@@ -72,6 +73,9 @@ class Context():
# The locations from which to push and pull prebuilt artifacts
self.artifact_cache_specs = None
+ # The global remote execution configuration
+ self.remote_execution_specs = None
+
# The directory to store build logs
self.logdir = None
@@ -187,7 +191,7 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
- 'cache', 'prompt', 'workspacedir',
+ 'cache', 'prompt', 'workspacedir', 'remote-execution'
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
@@ -212,6 +216,8 @@ class Context():
# Load artifact share configuration
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
+ self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
+
# Load pull build trees configuration
self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
@@ -271,7 +277,8 @@ class Context():
# Shallow validation of overrides, parts of buildstream which rely
# on the overrides are expected to validate elsewhere.
for _, overrides in _yaml.node_items(self._project_overrides):
- _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
+ _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror',
+ 'remote-execution'])
profile_end(Topics.LOAD_CONTEXT, 'load')
diff --git a/buildstream/_project.py b/buildstream/_project.py
index ef8d835..1492fde 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -507,7 +507,16 @@ class Project():
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
# Load remote-execution configuration for this project
- self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+ project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+ override_specs = SandboxRemote.specs_from_config_node(
+ self._context.get_overrides(self.name), self.directory)
+
+ if override_specs is not None:
+ self.remote_execution_specs = override_specs
+ elif project_specs is not None:
+ self.remote_execution_specs = project_specs
+ else:
+ self.remote_execution_specs = self._context.remote_execution_specs
# Load sandbox environment variables
self.base_environment = _yaml.node_get(config, Mapping, 'environment')
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py
index 111dc25..4b157b5 100644
--- a/buildstream/sandbox/_sandboxremote.py
+++ b/buildstream/sandbox/_sandboxremote.py
@@ -89,7 +89,7 @@ class SandboxRemote(Sandbox):
self._get_context().message(Message(None, MessageType.INFO, msg))
@staticmethod
- def specs_from_config_node(config_node, basedir):
+ def specs_from_config_node(config_node, basedir=None):
def require_node(config, keyname):
val = config.get(keyname)
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index bb66231..08e8a08 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and
push/pull access. For information regarding the server/client certificates
and keys, please see: :ref:`Key pair for the server <server_authentication>`.
+.. _project_remote_execution:
Remote execution
~~~~~~~~~~~~~~~~
@@ -268,6 +269,9 @@ instance names.
The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
+Remote execution configuration can be also provided in the `user
+configuration <user_config_remote_execution>`.
+
.. _project_essentials_mirrors:
Mirrors
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index 67574e5..71ca64a 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the
server/client certificates and keys, please see:
:ref:`Key pair for the server <server_authentication>`.
+.. _user_config_remote_execution:
+
+Remote execution
+~~~~~~~~~~~~~~~~
+
+The same configuration for :ref:`remote execution <project_remote_execution>`
+in ``project.conf`` can be provided in the user configuation.
+
+There is only one remote execution configuration used per project.
+
+The project overrides will be taken in priority. The global
+configuration will be used as fallback.
+
+1. Global remote execution fallback:
+
+.. code:: yaml
+
+ remote-execution:
+ execution-service:
+ url: http://execution.fallback.example.com:50051
+ instance-name: main
+ storage-service:
+ url: https://storage.fallback.example.com:11002/
+ server-cert: /keys/server.crt
+ client-cert: /keys/client.crt
+ client-key: /keys/client.key
+ instance-name: main
+ action-cache-service:
+ url: http://action.flalback.example.com:50052
+
+2. Project override:
+
+.. code:: yaml
+
+ projects:
+ some_project:
+ remote-execution:
+ execution-service:
+ url: http://execution.some_project.example.com:50051
+ instance-name: main
+ storage-service:
+ url: https://storage.some_project.example.com:11002/
+ server-cert: /some_project_keys/server.crt
+ client-cert: /some_project_keys/client.crt
+ client-key: /some_project_keys/client.key
+ instance-name: main
+ action-cache-service:
+ url: http://action.some_project.example.com:50052
Strict build plan