You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:49:46 UTC
[buildstream] branch
valentindavid/remote_execution_configuration_command_line created (now
a02fc69)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch valentindavid/remote_execution_configuration_command_line
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at a02fc69 Allow selecting user or project configuration for remote execution
This branch includes the following new commits:
new 7682939 Use relative path to project directory for remote execution certificates/keys
new 8f0e523 Add support for user remote execution configuration
new 51c9bb1 Add support for https channel to remote execution and actions servers
new a02fc69 Allow selecting user or project configuration for remote execution
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 03/04: Add support for https channel to remote
execution and actions servers
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch valentindavid/remote_execution_configuration_command_line
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 51c9bb1ea0b921dd7461635ad2a4da8102d0384f
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Dec 20 13:00:57 2018 +0100
Add support for https channel to remote execution and actions servers
Fixes #780.
---
buildstream/sandbox/_sandboxremote.py | 34 +++++++++++++++++++++++++++++-----
doc/source/format_project.rst | 3 ---
2 files changed, 29 insertions(+), 8 deletions(-)
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py
index 4b157b5..28b785f 100644
--- a/buildstream/sandbox/_sandboxremote.py
+++ b/buildstream/sandbox/_sandboxremote.py
@@ -68,10 +68,32 @@ class SandboxRemote(Sandbox):
self.storage_url = config.storage_service['url']
self.exec_url = config.exec_service['url']
+ exec_certs = {}
+ for key in ['client-cert', 'client-key', 'server-cert']:
+ if key in config.exec_service:
+ with open(resolve_path(config.exec_service[key]), 'rb') as f:
+ exec_certs[key] = f.read()
+
+ self.exec_credentials = grpc.ssl_channel_credentials(
+ root_certificates=exec_certs.get('server-cert'),
+ private_key=exec_certs.get('client-key'),
+ certificate_chain=exec_certs.get('client-cert'))
+
+ action_certs = {}
+ for key in ['client-cert', 'client-key', 'server-cert']:
+ if key in config.action_service:
+ with open(resolve_path(config.exec_service[key]), 'rb') as f:
+ action_certs[key] = f.read()
+
if config.action_service:
self.action_url = config.action_service['url']
+ self.action_credentials = grpc.ssl_channel_credentials(
+ root_certificates=action_certs.get('server-cert'),
+ private_key=action_certs.get('client-key'),
+ certificate_chain=action_certs.get('client-cert'))
else:
self.action_url = None
+ self.action_credentials = None
self.server_instance = config.exec_service.get('instance', None)
self.storage_instance = config.storage_service.get('instance', None)
@@ -117,7 +139,7 @@ class SandboxRemote(Sandbox):
remote_exec_storage_config = require_node(remote_config, 'storage-service')
remote_exec_action_config = remote_config.get('action-cache-service', {})
- _yaml.node_validate(remote_exec_service_config, ['url', 'instance'])
+ _yaml.node_validate(remote_exec_service_config, ['url', 'instance'] + tls_keys)
_yaml.node_validate(remote_exec_storage_config, ['url', 'instance'] + tls_keys)
if remote_exec_action_config:
_yaml.node_validate(remote_exec_action_config, ['url'])
@@ -304,6 +326,8 @@ class SandboxRemote(Sandbox):
"for example: http://buildservice:50051.")
if url.scheme == 'http':
channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
+ elif url.scheme == 'https':
+ channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.exec_credentials)
else:
raise SandboxError("Remote execution currently only supports the 'http' protocol "
"and '{}' was supplied.".format(url.scheme))
@@ -361,11 +385,11 @@ class SandboxRemote(Sandbox):
if not url.port:
raise SandboxError("You must supply a protocol and port number in the action-cache-service url, "
"for example: http://buildservice:50051.")
- if not url.scheme == "http":
- raise SandboxError("Currently only support http for the action cache"
- "and {} was supplied".format(url.scheme))
+ if url.scheme == 'http':
+ channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
+ elif url.scheme == 'https':
+ channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials)
- channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest)
stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
try:
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index 08e8a08..c3555e0 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -244,9 +244,6 @@ using the `remote-execution` option:
action-cache-service:
url: http://bar.action.com:50052
-The execution-service part of remote execution does not support encrypted
-connections yet, so the protocol must always be http.
-
storage-service specifies a remote CAS store and the parameters are the
same as those used to specify an :ref:`artifact server <artifacts>`.
[buildstream] 04/04: Allow selecting user or project configuration
for remote execution
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch valentindavid/remote_execution_configuration_command_line
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a02fc6963b977da825a69dac388bc8652fdf5634
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Dec 20 13:02:09 2018 +0100
Allow selecting user or project configuration for remote execution
---
buildstream/_context.py | 3 +++
buildstream/_frontend/app.py | 3 ++-
buildstream/_frontend/cli.py | 3 +++
buildstream/_project.py | 19 +++++++++++++++----
tests/completions/completions.py | 1 +
5 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index c62755c..b3c01f7 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -121,6 +121,9 @@ class Context():
# Whether or not to attempt to pull build trees globally
self.pull_buildtrees = None
+ # Which configured remote execution server to use
+ self.remote_execution = 'any'
+
# Boolean, whether we double-check with the user that they meant to
# remove a workspace directory.
self.prompt_workspace_close_remove_dir = None
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index 96e3c60..66bb975 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -183,7 +183,8 @@ class App():
'builders': 'sched_builders',
'pushers': 'sched_pushers',
'network_retries': 'sched_network_retries',
- 'pull_buildtrees': 'pull_buildtrees'
+ 'pull_buildtrees': 'pull_buildtrees',
+ 'remote_execution': 'remote_execution'
}
for cli_option, context_attr in override_map.items():
option_value = self._main_options.get(cli_option)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 29fc4cf..1d19408 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -239,6 +239,9 @@ def print_version(ctx, param, value):
help="The mirror to fetch from first, before attempting other mirrors")
@click.option('--pull-buildtrees', is_flag=True, default=None,
help="Include an element's build tree when pulling remote element artifacts")
+@click.option('--remote-execution', default='any',
+ type=click.Choice(['any', 'user', 'project', 'none']),
+ help='Select which remote execution server to use')
@click.pass_context
def cli(context, **kwargs):
"""Build and manipulate BuildStream projects
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 1492fde..fc3fb50 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -511,12 +511,23 @@ class Project():
override_specs = SandboxRemote.specs_from_config_node(
self._context.get_overrides(self.name), self.directory)
- if override_specs is not None:
- self.remote_execution_specs = override_specs
- elif project_specs is not None:
+ if self._context.remote_execution == 'any':
+ if override_specs is not None:
+ self.remote_execution_specs = override_specs
+ elif project_specs is not None:
+ self.remote_execution_specs = project_specs
+ else:
+ self.remote_execution_specs = self._context.remote_execution_specs
+ elif self._context.remote_execution == 'user':
+ if override_specs is not None:
+ self.remote_execution_specs = override_specs
+ else:
+ self.remote_execution_specs = self._context.remote_execution_specs
+ elif self._context.remote_execution == 'project':
self.remote_execution_specs = project_specs
else:
- self.remote_execution_specs = self._context.remote_execution_specs
+ assert self._context.remote_execution == 'none'
+ self.remote_execution_specs = None
# Load sandbox environment variables
self.base_environment = _yaml.node_get(config, Mapping, 'environment')
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index 372ed78..3f6a4d4 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -42,6 +42,7 @@ MAIN_OPTIONS = [
"--option ",
"--on-error ",
"--pull-buildtrees ",
+ "--remote-execution ",
"--pushers ",
"--strict ",
"--verbose ",
[buildstream] 01/04: Use relative path to project directory for
remote execution certificates/keys
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch valentindavid/remote_execution_configuration_command_line
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7682939ce3b8674e21b6d687ef0497dff551a8b5
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Dec 20 12:57:10 2018 +0100
Use relative path to project directory for remote execution certificates/keys
---
buildstream/sandbox/_sandboxremote.py | 23 ++++++++++++++++-------
1 file changed, 16 insertions(+), 7 deletions(-)
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py
index 503cf93..111dc25 100644
--- a/buildstream/sandbox/_sandboxremote.py
+++ b/buildstream/sandbox/_sandboxremote.py
@@ -41,7 +41,7 @@ from .._protos.google.longrunning import operations_pb2, operations_pb2_grpc
from .._artifactcache.cascache import CASRemote, CASRemoteSpec
-class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service action_service')):
+class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service action_service basedir')):
pass
@@ -59,6 +59,12 @@ class SandboxRemote(Sandbox):
if config is None:
return
+ def resolve_path(path):
+ if config.basedir and path:
+ return os.path.join(config.basedir, path)
+ else:
+ return path
+
self.storage_url = config.storage_service['url']
self.exec_url = config.exec_service['url']
@@ -70,11 +76,13 @@ class SandboxRemote(Sandbox):
self.server_instance = config.exec_service.get('instance', None)
self.storage_instance = config.storage_service.get('instance', None)
- self.storage_remote_spec = CASRemoteSpec(self.storage_url, push=True,
- server_cert=config.storage_service['server-cert'],
- client_key=config.storage_service['client-key'],
- client_cert=config.storage_service['client-cert'],
- instance_name=self.storage_instance)
+ self.storage_remote_spec = CASRemoteSpec(
+ self.storage_url, push=True,
+ server_cert=resolve_path(config.storage_service['server-cert']),
+ client_key=resolve_path(config.storage_service['client-key']),
+ client_cert=resolve_path(config.storage_service['client-cert']),
+ instance_name=self.storage_instance)
+
self.operation_name = None
def info(self, msg):
@@ -137,7 +145,8 @@ class SandboxRemote(Sandbox):
spec = RemoteExecutionSpec(remote_config['execution-service'],
remote_config['storage-service'],
- remote_exec_action_config)
+ remote_exec_action_config,
+ basedir)
return spec
def run_remote_command(self, channel, action_digest):
[buildstream] 02/04: Add support for user remote execution
configuration
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch valentindavid/remote_execution_configuration_command_line
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8f0e52314769beb0d7b8cf461950996b4812b09c
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Thu Dec 20 12:59:43 2018 +0100
Add support for user remote execution configuration
Fixes #631.
---
buildstream/_context.py | 11 ++++++--
buildstream/_project.py | 11 +++++++-
buildstream/sandbox/_sandboxremote.py | 2 +-
doc/source/format_project.rst | 4 +++
doc/source/using_config.rst | 48 +++++++++++++++++++++++++++++++++++
5 files changed, 72 insertions(+), 4 deletions(-)
diff --git a/buildstream/_context.py b/buildstream/_context.py
index e0eea99..c62755c 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache
from ._artifactcache.cascache import CASCache
from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
from .plugin import _plugin_lookup
+from .sandbox import SandboxRemote
# Context()
@@ -72,6 +73,9 @@ class Context():
# The locations from which to push and pull prebuilt artifacts
self.artifact_cache_specs = None
+ # The global remote execution configuration
+ self.remote_execution_specs = None
+
# The directory to store build logs
self.logdir = None
@@ -187,7 +191,7 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
- 'cache', 'prompt', 'workspacedir',
+ 'cache', 'prompt', 'workspacedir', 'remote-execution'
])
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
@@ -212,6 +216,8 @@ class Context():
# Load artifact share configuration
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
+ self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
+
# Load pull build trees configuration
self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
@@ -271,7 +277,8 @@ class Context():
# Shallow validation of overrides, parts of buildstream which rely
# on the overrides are expected to validate elsewhere.
for _, overrides in _yaml.node_items(self._project_overrides):
- _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
+ _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror',
+ 'remote-execution'])
profile_end(Topics.LOAD_CONTEXT, 'load')
diff --git a/buildstream/_project.py b/buildstream/_project.py
index ef8d835..1492fde 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -507,7 +507,16 @@ class Project():
self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
# Load remote-execution configuration for this project
- self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+ project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
+ override_specs = SandboxRemote.specs_from_config_node(
+ self._context.get_overrides(self.name), self.directory)
+
+ if override_specs is not None:
+ self.remote_execution_specs = override_specs
+ elif project_specs is not None:
+ self.remote_execution_specs = project_specs
+ else:
+ self.remote_execution_specs = self._context.remote_execution_specs
# Load sandbox environment variables
self.base_environment = _yaml.node_get(config, Mapping, 'environment')
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py
index 111dc25..4b157b5 100644
--- a/buildstream/sandbox/_sandboxremote.py
+++ b/buildstream/sandbox/_sandboxremote.py
@@ -89,7 +89,7 @@ class SandboxRemote(Sandbox):
self._get_context().message(Message(None, MessageType.INFO, msg))
@staticmethod
- def specs_from_config_node(config_node, basedir):
+ def specs_from_config_node(config_node, basedir=None):
def require_node(config, keyname):
val = config.get(keyname)
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index bb66231..08e8a08 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and
push/pull access. For information regarding the server/client certificates
and keys, please see: :ref:`Key pair for the server <server_authentication>`.
+.. _project_remote_execution:
Remote execution
~~~~~~~~~~~~~~~~
@@ -268,6 +269,9 @@ instance names.
The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
+Remote execution configuration can be also provided in the `user
+configuration <user_config_remote_execution>`.
+
.. _project_essentials_mirrors:
Mirrors
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index 67574e5..71ca64a 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the
server/client certificates and keys, please see:
:ref:`Key pair for the server <server_authentication>`.
+.. _user_config_remote_execution:
+
+Remote execution
+~~~~~~~~~~~~~~~~
+
+The same configuration for :ref:`remote execution <project_remote_execution>`
+in ``project.conf`` can be provided in the user configuation.
+
+There is only one remote execution configuration used per project.
+
+The project overrides will be taken in priority. The global
+configuration will be used as fallback.
+
+1. Global remote execution fallback:
+
+.. code:: yaml
+
+ remote-execution:
+ execution-service:
+ url: http://execution.fallback.example.com:50051
+ instance-name: main
+ storage-service:
+ url: https://storage.fallback.example.com:11002/
+ server-cert: /keys/server.crt
+ client-cert: /keys/client.crt
+ client-key: /keys/client.key
+ instance-name: main
+ action-cache-service:
+ url: http://action.flalback.example.com:50052
+
+2. Project override:
+
+.. code:: yaml
+
+ projects:
+ some_project:
+ remote-execution:
+ execution-service:
+ url: http://execution.some_project.example.com:50051
+ instance-name: main
+ storage-service:
+ url: https://storage.some_project.example.com:11002/
+ server-cert: /some_project_keys/server.crt
+ client-cert: /some_project_keys/client.crt
+ client-key: /some_project_keys/client.key
+ instance-name: main
+ action-cache-service:
+ url: http://action.some_project.example.com:50052
Strict build plan