You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/01/20 13:07:41 UTC
[buildstream] branch tristan/remote-cache-testing created (now
e8aa24b)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch tristan/remote-cache-testing
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at e8aa24b .github: Revived remote cache testing
This branch includes the following new commits:
new b51aa86 .github: Refactor how we run the remote execution test
new e8aa24b .github: Revived remote cache testing
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/02: .github: Revived remote cache testing
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/remote-cache-testing
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e8aa24b0beb63be0f788870429535b81c163b3a0
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Jan 20 22:03:33 2021 +0900
.github: Revived remote cache testing
This patch does the following things:
* ci.remote-execution-cluster.yml -> ci.remote-execution.yml
* .github/workflows/ci.yml: Refactored to add a second test matrix
The 'test-services' matrix runs tests which require adjacent services
run in parallel with the test defined in ci.docker-compose.yml
* .github/compose/ci.buildstream-remote-cache.yml: Added composition
file to run the buildstream artifact cache `bst-artifact-server`
* .github/compose/ci.buildbarn-remote-cache.yml: Added composition
file to run the buildbarn index and storage services.
* .github/compose/ci.docker-compose.yml: Add the test declarations
to run tests with `--remote-cache` to test against the artifact
cache servers.
---
.github/compose/buildbarn-config/asset.jsonnet | 32 ++++++++++++
.github/compose/buildbarn-config/storage.jsonnet | 26 ++++++++++
.github/compose/ci.buildbarn-remote-cache.yml | 59 ++++++++++++++++++++++
.github/compose/ci.buildstream-remote-cache.yml | 36 +++++++++++++
.github/compose/ci.docker-compose.yml | 29 +++++++++--
...ecution-cluster.yml => ci.remote-execution.yml} | 4 +-
.github/workflows/ci.yml | 24 ++++++---
7 files changed, 198 insertions(+), 12 deletions(-)
diff --git a/.github/compose/buildbarn-config/asset.jsonnet b/.github/compose/buildbarn-config/asset.jsonnet
new file mode 100644
index 0000000..aad06a2
--- /dev/null
+++ b/.github/compose/buildbarn-config/asset.jsonnet
@@ -0,0 +1,32 @@
+{
+ fetcher: {
+ caching: {
+ fetcher: {
+ // We should never be fetching anything which is not already returned by the caching fetcher.
+ 'error': {
+ code: 5,
+ message: "Asset Not Found",
+ }
+ }
+ }
+ },
+
+ assetStore: {
+ circular: {
+ directory: '/storage',
+ offsetFileSizeBytes: 1024 * 1024,
+ offsetCacheSize: 1000,
+ dataFileSizeBytes: 100 * 1024 * 1024,
+ dataAllocationChunkSizeBytes: 1048576,
+ instances: [''],
+ },
+ },
+ httpListenAddress: ':1111',
+ grpcServers: [{
+ listenAddresses: [':7981'],
+ authenticationPolicy: { allow: {} },
+ }],
+ allowUpdatesForInstances: [''],
+ maximumMessageSizeBytes: 16 * 1024 * 1024,
+}
+
diff --git a/.github/compose/buildbarn-config/storage.jsonnet b/.github/compose/buildbarn-config/storage.jsonnet
new file mode 100644
index 0000000..11fbdbe
--- /dev/null
+++ b/.github/compose/buildbarn-config/storage.jsonnet
@@ -0,0 +1,26 @@
+{
+ blobstore: {
+ contentAddressableStorage: {
+ circular: {
+ directory: '/cas',
+ offsetFileSizeBytes: 16 * 1024 * 1024,
+ offsetCacheSize: 10000,
+ dataFileSizeBytes: 10 * 1024 * 1024 * 1024,
+ dataAllocationChunkSizeBytes: 16 * 1024 * 1024,
+ },
+ },
+ actionCache: {
+ 'error': {
+ code: 12, # UNIMPLEMENTED
+ message: "AC requests are not supported for this endpoint.",
+ }
+ },
+ },
+ httpListenAddress: ':6981',
+ grpcServers: [{
+ listenAddresses: [':7982'],
+ authenticationPolicy: { allow: {} },
+ }],
+ allowAcUpdatesForInstanceNamePrefixes: [''],
+ maximumMessageSizeBytes: 16 * 1024 * 1024,
+}
diff --git a/.github/compose/ci.buildbarn-remote-cache.yml b/.github/compose/ci.buildbarn-remote-cache.yml
new file mode 100644
index 0000000..a7a8892
--- /dev/null
+++ b/.github/compose/ci.buildbarn-remote-cache.yml
@@ -0,0 +1,59 @@
+##
+# Buildbarn Compose manifest for BuildStream.
+#
+# Spins-up a unnamed and unauthenticated cache server:
+# - STORAGE at http://localhost:7982
+# - INDEX at: http://localhost:7981
+#
+# BuildStream configuration snippet:
+#
+# artifacts:
+# - url: https://localhost:7981
+# type: index
+# push: true
+# - url: https://localhost:7982
+# type: storage
+# push: true
+#
+# Basic usage:
+# - docker-compose -f ci.buildbarn-remote-cache.yml up
+# - docker-compose -f ci.buildbarn-remote-cache.yml down
+
+version: '3.4'
+
+services:
+ bb-asset:
+ image: buildbarn/bb-remote-asset:20200903T103837Z-90136c4
+ command: /config/asset.jsonnet
+ restart: unless-stopped
+ expose:
+ - "7981"
+ ports:
+ - "7981:7981"
+ volumes:
+ - type: volume
+ source: assets
+ target: /storage
+ - type: bind
+ source: ./buildbarn-config/
+ target: /config
+
+ bb-storage:
+ image: buildbarn/bb-storage:20200816T115912Z-d83e1f0
+ command: /config/storage.jsonnet
+ restart: unless-stopped
+ expose:
+ - "7982"
+ ports:
+ - "7982:7982"
+ volumes:
+ - type: volume
+ source: cas
+ target: /cas
+ - type: bind
+ source: ./buildbarn-config/
+ target: /config
+
+volumes:
+ assets:
+ cas:
diff --git a/.github/compose/ci.buildstream-remote-cache.yml b/.github/compose/ci.buildstream-remote-cache.yml
new file mode 100644
index 0000000..9f853fa
--- /dev/null
+++ b/.github/compose/ci.buildstream-remote-cache.yml
@@ -0,0 +1,36 @@
+##
+# A Builstream Artifact Server
+#
+# Spins-up a unnamed and unauthenticated grid:
+# - artifacts server at: http://localhost:50052
+#
+# BuildStream configuration snippet:
+#
+# artifacts:
+# url: http://localhost:50052
+# push: true
+#
+# Basic usage:
+# - docker-compose -f ci.buildstream-remote-cache.yml up
+# - docker-compose -f ci.buildstream-remote-cache.yml down
+#
+version: "3.2"
+
+services:
+ controller:
+ image: buildstream/buildstream:dev
+ command: ["bst-artifact-server","--port",
+ "50052",
+ "--enable-push",
+ "/artifacts"
+ ]
+ ports:
+ - 50052:50052
+ networks:
+ - grid
+networks:
+ grid:
+ driver: bridge
+
+volumes:
+ cache:
diff --git a/.github/compose/ci.docker-compose.yml b/.github/compose/ci.docker-compose.yml
index 88ab894..614a79c 100644
--- a/.github/compose/ci.docker-compose.yml
+++ b/.github/compose/ci.docker-compose.yml
@@ -67,12 +67,33 @@ services:
REMOTE_EXECUTION_SERVICE: http://localhost:50051
SOURCE_CACHE_SERVICE: http://localhost:50052
- # The ci.remote-execution-cluster.yml takes care of bringing up
- # a remote execution cluster, exposing the ports 50052 and 50051
- # on the localhost network.
+ # We need to use host networking mode in order to be able to
+ # properly resolve services exposed by adjacent containers.
+ #
+ network_mode: host
+
+ buildstream-remote-cache:
+ <<: *tests-template
+ command: tox -vvvvv -- --color=yes --remote-cache
+ environment:
+ TOXENV: ${CI_TOXENV_MAIN}
+ ARTIFACT_CACHE_SERVICE: http://localhost:50052
+
+ # We need to use host networking mode in order to be able to
+ # properly resolve services exposed by adjacent containers.
#
+ network_mode: host
+
+ buildbarn-remote-cache:
+ <<: *tests-template
+ command: tox -vvvvv -- --color=yes --remote-cache
+ environment:
+ TOXENV: ${CI_TOXENV_MAIN}
+ ARTIFACT_INDEX_SERVICE: http://localhost:7981
+ ARTIFACT_STORAGE_SERVICE: http://localhost:7982
+
# We need to use host networking mode in order to be able to
- # properly resolve these services.
+ # properly resolve services exposed by adjacent containers.
#
network_mode: host
diff --git a/.github/compose/ci.remote-execution-cluster.yml b/.github/compose/ci.remote-execution.yml
similarity index 94%
rename from .github/compose/ci.remote-execution-cluster.yml
rename to .github/compose/ci.remote-execution.yml
index 75d0ec7..a663362 100644
--- a/.github/compose/ci.remote-execution-cluster.yml
+++ b/.github/compose/ci.remote-execution.yml
@@ -19,8 +19,8 @@
# url: http://localhost:50051
#
# Basic usage:
-# - docker-compose -f buildgrid-compose.yml up
-# - docker-compose -f buildgrid-compose.yml down
+# - docker-compose -f ci.remote-execution.yml up
+# - docker-compose -f ci.remote-execution.yml down
#
version: "3.2"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 292e497..dcd353d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -76,8 +76,20 @@ jobs:
run \
${{ matrix.test-name }}
- remote-execution:
+ # Matrix of tests which run against remote services which we bring up adjacently
+ service-tests:
runs-on: ubuntu-20.04
+
+ strategy:
+ matrix:
+
+ # The names here map to valid services defined in "../compose/ci.docker-compose.yml",
+ # and they also map to corresponding filenames of services which are expected
+ # to be run in the background
+ test-name:
+ - remote-execution
+ - buildstream-remote-cache
+ - buildbarn-remote-cache
steps:
- name: Check out repository
uses: actions/checkout@v2
@@ -92,26 +104,26 @@ jobs:
run: |
docker-compose \
--env-file ${GITHUB_WORKSPACE}/.github/common.env \
- --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml \
+ --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml \
up --detach --renew-anon-volumes --remove-orphans
- name: Run the remote execution tests
run: |
docker-compose \
--env-file ${GITHUB_WORKSPACE}/.github/common.env \
- --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml run remote-execution
+ --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml run ${{ matrix.test-name }}
- name: Bring down the RE cluster
run: |
docker-compose \
--env-file ${GITHUB_WORKSPACE}/.github/common.env \
- --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml stop
+ --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml stop
docker-compose \
--env-file ${GITHUB_WORKSPACE}/.github/common.env \
- --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml logs
+ --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml logs
docker-compose \
--env-file ${GITHUB_WORKSPACE}/.github/common.env \
- --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-cluster.yml down
+ --file ${GITHUB_WORKSPACE}/.github/compose/ci.${{ matrix.test-name }}.yml down
docs:
runs-on: ubuntu-20.04
[buildstream] 01/02: .github: Refactor how we run the remote
execution test
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/remote-cache-testing
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b51aa860b181fe76a8e20a28d52e57f4ee8f45d5
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Jan 20 21:22:13 2021 +0900
.github: Refactor how we run the remote execution test
Add the remote execution client side service to ci.docker-compose.yml
instead of having a separate file for this, this improves code reuse.
---
.github/compose/ci.docker-compose.yml | 18 ++++++++++++++
.github/compose/ci.remote-execution-test.yml | 36 ----------------------------
.github/workflows/ci.yml | 2 +-
3 files changed, 19 insertions(+), 37 deletions(-)
diff --git a/.github/compose/ci.docker-compose.yml b/.github/compose/ci.docker-compose.yml
index 0a0da86..88ab894 100644
--- a/.github/compose/ci.docker-compose.yml
+++ b/.github/compose/ci.docker-compose.yml
@@ -58,6 +58,24 @@ services:
environment:
BST_PLUGINS_EXPERIMENTAL_VERSION: master
+ remote-execution:
+ <<: *tests-template
+ command: tox -vvvvv -- --color=yes --remote-execution
+ environment:
+ TOXENV: ${CI_TOXENV_MAIN}
+ ARTIFACT_CACHE_SERVICE: http://localhost:50052
+ REMOTE_EXECUTION_SERVICE: http://localhost:50051
+ SOURCE_CACHE_SERVICE: http://localhost:50052
+
+ # The ci.remote-execution-cluster.yml takes care of bringing up
+ # a remote execution cluster, exposing the ports 50052 and 50051
+ # on the localhost network.
+ #
+ # We need to use host networking mode in order to be able to
+ # properly resolve these services.
+ #
+ network_mode: host
+
docs:
<<: *tests-template
command: tox -e docs
diff --git a/.github/compose/ci.remote-execution-test.yml b/.github/compose/ci.remote-execution-test.yml
deleted file mode 100644
index 976142b..0000000
--- a/.github/compose/ci.remote-execution-test.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-version: '3.4'
-
-services:
-
- test:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${CI_IMAGE_VERSION:-latest}
- command: tox -vvvvv -- --color=yes --remote-execution
- environment:
- TOXENV: ${CI_TOXENV_MAIN}
- ARTIFACT_CACHE_SERVICE: http://localhost:50052
- REMOTE_EXECUTION_SERVICE: http://localhost:50051
- SOURCE_CACHE_SERVICE: http://localhost:50052
-
- # Some of the remote execution tests still require running a local sandbox
- # to run `bst shell` on some build results even though they were built
- # remotely, so we need to enable priviledged mode for this container as well.
- #
- privileged: true
- devices:
- - /dev/fuse:/dev/fuse
-
- # Mount the local directory and set the working directory
- # to run the tests from.
- #
- volumes:
- - ../..:/home/testuser/buildstream
- working_dir: /home/testuser/buildstream
-
- # The ci.remote-execution-cluster.yml takes care of bringing up
- # a remote execution cluster, exposing the ports 50052 and 50051
- # on the localhost network.
- #
- # We need to use host networking mode in order to be able to
- # properly resolve these services.
- #
- network_mode: host
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f2a44c8..292e497 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -99,7 +99,7 @@ jobs:
run: |
docker-compose \
--env-file ${GITHUB_WORKSPACE}/.github/common.env \
- --file ${GITHUB_WORKSPACE}/.github/compose/ci.remote-execution-test.yml run test
+ --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml run remote-execution
- name: Bring down the RE cluster
run: |