You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:28:41 UTC
[buildstream] branch qinusty/add-bb-remote-exec created (now
c48ef8a)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch qinusty/add-bb-remote-exec
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at c48ef8a TMP: LOGGING UNAVAILABLE
This branch includes the following new commits:
new 459f0a5 Add buildbarn remote-execution test case
new 9699079 TMP: Comment unrelated tests
new c48ef8a TMP: LOGGING UNAVAILABLE
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 03/03: TMP: LOGGING UNAVAILABLE
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch qinusty/add-bb-remote-exec
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c48ef8ad839a322f8d7f4cf179d4fbe8124e53b4
Author: Josh Smith <jo...@codethink.co.uk>
AuthorDate: Thu Sep 3 13:30:18 2020 +0100
TMP: LOGGING UNAVAILABLE
---
src/buildstream/sandbox/_sandboxremote.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index 41fb9e1..ed2ba2c 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -215,7 +215,7 @@ class SandboxRemote(SandboxREAPI):
except grpc.RpcError as e:
status_code = e.code()
if status_code == grpc.StatusCode.UNAVAILABLE:
- raise SandboxError("Failed contacting remote execution server at {}.".format(self.exec_url))
+ raise SandboxError("Failed contacting remote execution server at {}.\n\t{}".format(self.exec_url, e))
if status_code in (
grpc.StatusCode.INVALID_ARGUMENT,
[buildstream] 02/03: TMP: Comment unrelated tests
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch qinusty/add-bb-remote-exec
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9699079a55efd69d1acc132400a2ca62d2bde455
Author: Josh Smith <jo...@codethink.co.uk>
AuthorDate: Wed Sep 2 17:55:13 2020 +0100
TMP: Comment unrelated tests
---
.gitlab-ci.yml | 684 ++++++++++++++++++++++++++++-----------------------------
1 file changed, 342 insertions(+), 342 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f6c1217..e5b2ca8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -64,99 +64,99 @@ variables:
paths:
- .coverage-reports
-tests-debian-10:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:10-${DOCKER_IMAGE_VERSION}
- <<: *tests
+# tests-debian-10:
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:10-${DOCKER_IMAGE_VERSION}
+# <<: *tests
-tests-fedora-31:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
- <<: *tests
+# tests-fedora-31:
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
+# <<: *tests
-tests-fedora-32:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
- <<: *tests
+# tests-fedora-32:
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
+# <<: *tests
-tests-ubuntu-18.04:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:18.04-${DOCKER_IMAGE_VERSION}
- <<: *tests
+# tests-ubuntu-18.04:
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:18.04-${DOCKER_IMAGE_VERSION}
+# <<: *tests
-tests-centos-7.7:
- <<: *tests
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.7.1908-${DOCKER_IMAGE_VERSION}
+# tests-centos-7.7:
+# <<: *tests
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.7.1908-${DOCKER_IMAGE_VERSION}
-# Test the master version of some external plugins
-tests-plugins-master:
- <<: *tests
- allow_failure: true
+# # Test the master version of some external plugins
+# tests-plugins-master:
+# <<: *tests
+# allow_failure: true
- variables:
- BST_PLUGINS_EXPERIMENTAL_VERSION: master
+# variables:
+# BST_PLUGINS_EXPERIMENTAL_VERSION: master
-tests-userchroot:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
- <<: *tests
- variables:
- BST_CAS_STAGING_ROOT: "/builds/userchroot"
+# tests-userchroot:
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
+# <<: *tests
+# variables:
+# BST_CAS_STAGING_ROOT: "/builds/userchroot"
- script:
- - mkdir -p "${INTEGRATION_CACHE}"
- - useradd -Um buildstream
+# script:
+# - mkdir -p "${INTEGRATION_CACHE}"
+# - useradd -Um buildstream
- # Use buildbox-run-userchroot and hardlinking
- - ln -svf buildbox-run-userchroot /usr/local/bin/buildbox-run
- - rm -vf /usr/local/bin/buildbox-fuse
+# # Use buildbox-run-userchroot and hardlinking
+# - ln -svf buildbox-run-userchroot /usr/local/bin/buildbox-run
+# - rm -vf /usr/local/bin/buildbox-fuse
- # When using userchroot, buildbox-casd must run as a separate user
- - useradd -g buildstream buildbox-casd
- - chown buildbox-casd:buildstream /usr/local/bin/buildbox-casd
- - chmod u+s /usr/local/bin/buildbox-casd
+# # When using userchroot, buildbox-casd must run as a separate user
+# - useradd -g buildstream buildbox-casd
+# - chown buildbox-casd:buildstream /usr/local/bin/buildbox-casd
+# - chmod u+s /usr/local/bin/buildbox-casd
- # Set up staging root with permissions required by userchroot,
- # must be on same filesystem as current directory to support hardlinks
- - mkdir -p "${BST_CAS_STAGING_ROOT}"
- - chown -R buildbox-casd:buildstream "${BST_CAS_STAGING_ROOT}"
- # userchroot doesn't allow group/world-writable base directory
- - chmod go-w /builds
- - echo buildbox-casd:${BST_CAS_STAGING_ROOT} > /etc/userchroot.conf
+# # Set up staging root with permissions required by userchroot,
+# # must be on same filesystem as current directory to support hardlinks
+# - mkdir -p "${BST_CAS_STAGING_ROOT}"
+# - chown -R buildbox-casd:buildstream "${BST_CAS_STAGING_ROOT}"
+# # userchroot doesn't allow group/world-writable base directory
+# - chmod go-w /builds
+# - echo buildbox-casd:${BST_CAS_STAGING_ROOT} > /etc/userchroot.conf
- - chown -R buildstream:buildstream .
+# - chown -R buildstream:buildstream .
- # Run the tests as a simple user to test for permission issues
- - su buildstream -c "umask 002 && ${TEST_COMMAND}"
+# # Run the tests as a simple user to test for permission issues
+# - su buildstream -c "umask 002 && ${TEST_COMMAND}"
-tests-fedora-missing-deps:
- # Ensure that tests behave nicely while missing bwrap and ostree
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
- <<: *tests
+# tests-fedora-missing-deps:
+# # Ensure that tests behave nicely while missing bwrap and ostree
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
+# <<: *tests
- script:
- # We remove the Bubblewrap and OSTree packages here so that we catch any
- # codepaths that try to use them. Removing OSTree causes fuse-libs to
- # disappear unless we mark it as user-installed.
- - dnf mark install fuse-libs systemd-udev
- - dnf erase -y bubblewrap ostree
+# script:
+# # We remove the Bubblewrap and OSTree packages here so that we catch any
+# # codepaths that try to use them. Removing OSTree causes fuse-libs to
+# # disappear unless we mark it as user-installed.
+# - dnf mark install fuse-libs systemd-udev
+# - dnf erase -y bubblewrap ostree
- - useradd -Um buildstream
- - chown -R buildstream:buildstream .
+# - useradd -Um buildstream
+# - chown -R buildstream:buildstream .
- - su buildstream -c "${TEST_COMMAND}"
+# - su buildstream -c "${TEST_COMMAND}"
-tests-fedora-update-deps:
- # Check if the tests pass after updating requirements to their latest
- # allowed version.
- allow_failure: true
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
- <<: *tests
+# tests-fedora-update-deps:
+# # Check if the tests pass after updating requirements to their latest
+# # allowed version.
+# allow_failure: true
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
+# <<: *tests
- script:
- - useradd -Um buildstream
- - chown -R buildstream:buildstream .
+# script:
+# - useradd -Um buildstream
+# - chown -R buildstream:buildstream .
- - make --always-make --directory requirements
- - cat requirements/*.txt
+# - make --always-make --directory requirements
+# - cat requirements/*.txt
- - su buildstream -c "${TEST_COMMAND}"
+# - su buildstream -c "${TEST_COMMAND}"
# This template ensures that the server stack defined in COMPOSE_MANIFEST is spun up for script execution
# and cleaned up after the script is completed.
@@ -179,35 +179,35 @@ tests-fedora-update-deps:
# Required to be able to connect to the docker daemon. See https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
DOCKER_TLS_CERTDIR: ""
-tests-remote-execution:
- <<: *tests
- <<: *remote-test # Spin up server stack
- variables:
- <<: *docker-variables
- COMPOSE_MANIFEST: .gitlab-ci/buildgrid-remote-execution.yml # < *remote-test
- ARTIFACT_CACHE_SERVICE: http://docker:50052
- REMOTE_EXECUTION_SERVICE: http://docker:50051
- SOURCE_CACHE_SERVICE: http://docker:50052
- PYTEST_ARGS: "--color=yes --remote-execution"
-
-tests-remote-cache:
- <<: *tests
- <<: *remote-test # Spin up/down server stack
- variables:
- <<: *docker-variables
- COMPOSE_MANIFEST: .gitlab-ci/cache-server.yml # < *remote-test
- ARTIFACT_CACHE_SERVICE: http://docker:50052
- PYTEST_ARGS: "--color=yes --remote-cache"
-
-tests-bb-remote-cache:
- <<: *tests
- <<: *remote-test # Spin up/down server stack
- variables:
- <<: *docker-variables
- COMPOSE_MANIFEST: .gitlab-ci/buildbarn-remote-cache.yml # < *remote-test
- ARTIFACT_INDEX_SERVICE: http://docker:8979
- ARTIFACT_STORAGE_SERVICE: http://docker:8980
- PYTEST_ARGS: "--color=yes --remote-cache"
+# tests-remote-execution:
+# <<: *tests
+# <<: *remote-test # Spin up server stack
+# variables:
+# <<: *docker-variables
+# COMPOSE_MANIFEST: .gitlab-ci/buildgrid-remote-execution.yml # < *remote-test
+# ARTIFACT_CACHE_SERVICE: http://docker:50052
+# REMOTE_EXECUTION_SERVICE: http://docker:50051
+# SOURCE_CACHE_SERVICE: http://docker:50052
+# PYTEST_ARGS: "--color=yes --remote-execution"
+
+# tests-remote-cache:
+# <<: *tests
+# <<: *remote-test # Spin up/down server stack
+# variables:
+# <<: *docker-variables
+# COMPOSE_MANIFEST: .gitlab-ci/cache-server.yml # < *remote-test
+# ARTIFACT_CACHE_SERVICE: http://docker:50052
+# PYTEST_ARGS: "--color=yes --remote-cache"
+
+# tests-bb-remote-cache:
+# <<: *tests
+# <<: *remote-test # Spin up/down server stack
+# variables:
+# <<: *docker-variables
+# COMPOSE_MANIFEST: .gitlab-ci/buildbarn-remote-cache.yml # < *remote-test
+# ARTIFACT_INDEX_SERVICE: http://docker:8979
+# ARTIFACT_STORAGE_SERVICE: http://docker:8980
+# PYTEST_ARGS: "--color=yes --remote-cache"
tests-bb-remote-execution:
<<: *tests
@@ -221,245 +221,245 @@ tests-bb-remote-execution:
SOURCE_CACHE_SERVICE: http://docker:8980
PYTEST_ARGS: "--color=yes --remote-execution"
-tests-no-usedevelop:
- # Ensure that tests also pass without `--develop` flag.
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
- <<: *tests
- variables:
- TOXENV: py36-nocover,py37-nocover,py38-nocover
-
-
-# Run type checkers
-mypy:
- stage: test
-
- script:
- - tox -e mypy
- except:
- - schedules
-
-# Lint separately from testing
-lint:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
- stage: test
-
- before_script:
- # Diagnostics
- - python3 --version
-
- script:
- - tox -e format-check,lint
- except:
- - schedules
-
-# Catch regressions in native windows support
-tests-win32-master:
- stage: test
- variables:
- LC_ALL: C.UTF-8
- LANG: C.UTF-8
- tags:
- - win32
- script:
- - tox -e win32
- only:
- - master
- except:
- - schedules
-
-# Optional test to catch regressions in native windows support on non-master branches
-tests-win32-non-master:
- stage: test
- variables:
- LC_ALL: C.UTF-8
- LANG: C.UTF-8
- tags:
- - win32
- script:
- - tox -e win32
- except:
- - master
- when: manual
-
-# Automatically build documentation for every commit, we want to know
-# if building documentation fails even if we're not deploying it.
-docs:
- stage: test
- variables:
- BST_FORCE_SESSION_REBUILD: 1
- script:
- - env BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources" tox -e docs
- - mv doc/build/html public
- except:
- - schedules
- artifacts:
- paths:
- - public/
-
-.overnight-tests: &overnight-tests-template
- stage: test
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
- variables:
- FD_SDK_REF: freedesktop-sdk-20.08beta.1-buildstream2
- BST_EXT_REF: 1.93.4
- before_script:
- - |
- mkdir -p "${HOME}/.config"
- cat <<EOF >"${HOME}/.config/buildstream.conf"
- scheduler:
- fetchers: 2
- EOF
- - dnf install -y ostree
- # Install pinned BuildStream dependencies, BuildStream from the local repository
- # and bst-plugins-expeirmental from its repository
- - |
- pip3 install \
- -r requirements/requirements.txt . \
- git+https://gitlab.com/buildstream/bst-plugins-experimental.git@${BST_EXT_REF}#egg=bst_plugins_experimental[cargo] \
- .
- - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git
- - git -C freedesktop-sdk checkout ${FD_SDK_REF}
- artifacts:
- paths:
- - "${HOME}/.cache/buildstream/logs"
- only:
- - schedules
-
-overnight-tests:
- <<: *overnight-tests-template
- script:
- - |
- cat > ~/.config/buildstream.conf << EOF
- # Get a lot of output in case of errors
- logging:
- error-lines: 80
- #
- # Artifacts
- #
- artifacts:
- - url: https://bb-cache.buildstream.build:11002
- client-cert: $OVERNIGHT_CACHE_PUSH_CERT
- client-key: $OVERNIGHT_CACHE_PUSH_KEY
- push: true
- EOF
- - make -C freedesktop-sdk
- tags:
- - overnight-tests
-
-overnight-tests-no-cache:
- <<: *overnight-tests-template
- script:
- - sed -i '/artifacts:/,+1 d' freedesktop-sdk/project.conf
- - make -C freedesktop-sdk
- tags:
- - overnight-tests
-
-# These tests might be a bit more flaky since they randomize the order
-# They will also take longer to run. Hence run them only nightly
-overnight-randomized:
- image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
- <<: *tests
-
- script:
- - mkdir -p "${INTEGRATION_CACHE}"
- - useradd -Um buildstream
- - chown -R buildstream:buildstream .
-
- # Don't run tests multiprocessed here, the randomized order doesn't like that
- - su buildstream -c "tox -e py36-randomized,py37-randomized,py38-randomized -- --color=yes --integration"
- - su buildstream -c "tox -e py36-randomized-external,py37-randomized-external,py38-randomized-external -- --color=yes --integration"
-
- # We need to override the exclusion from the template in order to run on schedules
- except: []
- only:
- - schedules
-
-#####################################################
-# Post stage #
-#####################################################
-
-analysis:
- stage: post
- script:
- - |
- pip3 install radon
- mkdir analysis
-
- - |
- echo "Calculating Maintainability Index"
- radon mi -s -j src/buildstream > analysis/mi.json
- radon mi -s src/buildstream
-
- - |
- echo "Calculating Cyclomatic Complexity"
- radon cc -a -s -j src/buildstream > analysis/cc.json
- radon cc -a -s src/buildstream
-
- - |
- echo "Calculating Raw Metrics"
- radon raw -s -j src/buildstream > analysis/raw.json
- radon raw -s src/buildstream
-
- except:
- - schedules
- artifacts:
- paths:
- - analysis/
-
-# Collate coverage reports
-#
-coverage:
- stage: post
- coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
- script:
- - cp -a .coverage-reports/ ./coverage-sources
- - tox -e coverage
- - cp -a .coverage-reports/ ./coverage-report
- dependencies:
- - tests-centos-7.7
- - tests-debian-10
- - tests-fedora-31
- - tests-remote-execution
- - tests-ubuntu-18.04
- - tests-userchroot
- except:
- - schedules
- artifacts:
- paths:
- - coverage-sources/
- - coverage-report/
-
-# Deploy, only for merges which land on master branch.
-#
-pages:
- stage: publish
- dependencies:
- - coverage
- - docs
- variables:
- ACME_DIR: public/.well-known/acme-challenge
- COVERAGE_DIR: public/coverage
- script:
- - mkdir -p ${ACME_DIR}
- # Required to finish the creation of the Let's Encrypt certificate,
- # which allows using https://docs.buildstream.build/ for accessing
- # the documentation.
- - echo ${ACME_CHALLENGE} > ${ACME_DIR}/$(echo ${ACME_CHALLENGE} | cut -c1-43)
- - mkdir -p ${COVERAGE_DIR}
- - cp -a ./coverage-report/ ${COVERAGE_DIR}
- artifacts:
- paths:
- - public/
- only:
- #
- # FIXME:
- #
- # Ideally we want to publish to a different subdir of
- # pages depending on which stable branch we are building here,
- # not currently automatically supported but can be worked around.
- #
- # See https://gitlab.com/gitlab-org/gitlab-ce/issues/35141
- #
- - master
- except:
- - schedules
+# tests-no-usedevelop:
+# # Ensure that tests also pass without `--develop` flag.
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
+# <<: *tests
+# variables:
+# TOXENV: py36-nocover,py37-nocover,py38-nocover
+
+
+# # Run type checkers
+# mypy:
+# stage: test
+
+# script:
+# - tox -e mypy
+# except:
+# - schedules
+
+# # Lint separately from testing
+# lint:
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
+# stage: test
+
+# before_script:
+# # Diagnostics
+# - python3 --version
+
+# script:
+# - tox -e format-check,lint
+# except:
+# - schedules
+
+# # Catch regressions in native windows support
+# tests-win32-master:
+# stage: test
+# variables:
+# LC_ALL: C.UTF-8
+# LANG: C.UTF-8
+# tags:
+# - win32
+# script:
+# - tox -e win32
+# only:
+# - master
+# except:
+# - schedules
+
+# # Optional test to catch regressions in native windows support on non-master branches
+# tests-win32-non-master:
+# stage: test
+# variables:
+# LC_ALL: C.UTF-8
+# LANG: C.UTF-8
+# tags:
+# - win32
+# script:
+# - tox -e win32
+# except:
+# - master
+# when: manual
+
+# # Automatically build documentation for every commit, we want to know
+# # if building documentation fails even if we're not deploying it.
+# docs:
+# stage: test
+# variables:
+# BST_FORCE_SESSION_REBUILD: 1
+# script:
+# - env BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources" tox -e docs
+# - mv doc/build/html public
+# except:
+# - schedules
+# artifacts:
+# paths:
+# - public/
+
+# .overnight-tests: &overnight-tests-template
+# stage: test
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
+# variables:
+# FD_SDK_REF: freedesktop-sdk-20.08beta.1-buildstream2
+# BST_EXT_REF: 1.93.4
+# before_script:
+# - |
+# mkdir -p "${HOME}/.config"
+# cat <<EOF >"${HOME}/.config/buildstream.conf"
+# scheduler:
+# fetchers: 2
+# EOF
+# - dnf install -y ostree
+# # Install pinned BuildStream dependencies, BuildStream from the local repository
+# # and bst-plugins-expeirmental from its repository
+# - |
+# pip3 install \
+# -r requirements/requirements.txt . \
+# git+https://gitlab.com/buildstream/bst-plugins-experimental.git@${BST_EXT_REF}#egg=bst_plugins_experimental[cargo] \
+# .
+# - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git
+# - git -C freedesktop-sdk checkout ${FD_SDK_REF}
+# artifacts:
+# paths:
+# - "${HOME}/.cache/buildstream/logs"
+# only:
+# - schedules
+
+# overnight-tests:
+# <<: *overnight-tests-template
+# script:
+# - |
+# cat > ~/.config/buildstream.conf << EOF
+# # Get a lot of output in case of errors
+# logging:
+# error-lines: 80
+# #
+# # Artifacts
+# #
+# artifacts:
+# - url: https://bb-cache.buildstream.build:11002
+# client-cert: $OVERNIGHT_CACHE_PUSH_CERT
+# client-key: $OVERNIGHT_CACHE_PUSH_KEY
+# push: true
+# EOF
+# - make -C freedesktop-sdk
+# tags:
+# - overnight-tests
+
+# overnight-tests-no-cache:
+# <<: *overnight-tests-template
+# script:
+# - sed -i '/artifacts:/,+1 d' freedesktop-sdk/project.conf
+# - make -C freedesktop-sdk
+# tags:
+# - overnight-tests
+
+# # These tests might be a bit more flaky since they randomize the order
+# # They will also take longer to run. Hence run them only nightly
+# overnight-randomized:
+# image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
+# <<: *tests
+
+# script:
+# - mkdir -p "${INTEGRATION_CACHE}"
+# - useradd -Um buildstream
+# - chown -R buildstream:buildstream .
+
+# # Don't run tests multiprocessed here, the randomized order doesn't like that
+# - su buildstream -c "tox -e py36-randomized,py37-randomized,py38-randomized -- --color=yes --integration"
+# - su buildstream -c "tox -e py36-randomized-external,py37-randomized-external,py38-randomized-external -- --color=yes --integration"
+
+# # We need to override the exclusion from the template in order to run on schedules
+# except: []
+# only:
+# - schedules
+
+# #####################################################
+# # Post stage #
+# #####################################################
+
+# analysis:
+# stage: post
+# script:
+# - |
+# pip3 install radon
+# mkdir analysis
+
+# - |
+# echo "Calculating Maintainability Index"
+# radon mi -s -j src/buildstream > analysis/mi.json
+# radon mi -s src/buildstream
+
+# - |
+# echo "Calculating Cyclomatic Complexity"
+# radon cc -a -s -j src/buildstream > analysis/cc.json
+# radon cc -a -s src/buildstream
+
+# - |
+# echo "Calculating Raw Metrics"
+# radon raw -s -j src/buildstream > analysis/raw.json
+# radon raw -s src/buildstream
+
+# except:
+# - schedules
+# artifacts:
+# paths:
+# - analysis/
+
+# # Collate coverage reports
+# #
+# coverage:
+# stage: post
+# coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
+# script:
+# - cp -a .coverage-reports/ ./coverage-sources
+# - tox -e coverage
+# - cp -a .coverage-reports/ ./coverage-report
+# dependencies:
+# - tests-centos-7.7
+# - tests-debian-10
+# - tests-fedora-31
+# - tests-remote-execution
+# - tests-ubuntu-18.04
+# - tests-userchroot
+# except:
+# - schedules
+# artifacts:
+# paths:
+# - coverage-sources/
+# - coverage-report/
+
+# # Deploy, only for merges which land on master branch.
+# #
+# pages:
+# stage: publish
+# dependencies:
+# - coverage
+# - docs
+# variables:
+# ACME_DIR: public/.well-known/acme-challenge
+# COVERAGE_DIR: public/coverage
+# script:
+# - mkdir -p ${ACME_DIR}
+# # Required to finish the creation of the Let's Encrypt certificate,
+# # which allows using https://docs.buildstream.build/ for accessing
+# # the documentation.
+# - echo ${ACME_CHALLENGE} > ${ACME_DIR}/$(echo ${ACME_CHALLENGE} | cut -c1-43)
+# - mkdir -p ${COVERAGE_DIR}
+# - cp -a ./coverage-report/ ${COVERAGE_DIR}
+# artifacts:
+# paths:
+# - public/
+# only:
+# #
+# # FIXME:
+# #
+# # Ideally we want to publish to a different subdir of
+# # pages depending on which stable branch we are building here,
+# # not currently automatically supported but can be worked around.
+# #
+# # See https://gitlab.com/gitlab-org/gitlab-ce/issues/35141
+# #
+# - master
+# except:
+# - schedules
[buildstream] 01/03: Add buildbarn remote-execution test case
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch qinusty/add-bb-remote-exec
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 459f0a559d6eb62dd2b02f6646bc81b4ff517cac
Author: Josh Smith <jo...@codethink.co.uk>
AuthorDate: Wed Sep 2 17:51:23 2020 +0100
Add buildbarn remote-execution test case
This introduces remote execution with bb-remote-asset as the
storage index service. Ports have been standardised across the
bb-remote-cache test also.
---
.gitlab-ci.yml | 16 +++++-
.gitlab-ci/buildbarn-remote-cache.yml | 10 ++--
.gitlab-ci/buildbarn-remote-execution.yml | 83 +++++++++++++++++++++++++++++++
.gitlab-ci/config/asset.jsonnet | 2 +-
.gitlab-ci/config/common.libsonnet | 12 +++++
.gitlab-ci/config/frontend.jsonnet | 35 +++++++++++++
.gitlab-ci/config/runner.jsonnet | 9 ++++
.gitlab-ci/config/scheduler.jsonnet | 15 ++++++
.gitlab-ci/config/storage.jsonnet | 2 +-
.gitlab-ci/config/worker.jsonnet | 37 ++++++++++++++
10 files changed, 210 insertions(+), 11 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5f8949d..f6c1217 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -205,10 +205,22 @@ tests-bb-remote-cache:
variables:
<<: *docker-variables
COMPOSE_MANIFEST: .gitlab-ci/buildbarn-remote-cache.yml # < *remote-test
- ARTIFACT_INDEX_SERVICE: http://docker:7981
- ARTIFACT_STORAGE_SERVICE: http://docker:7982
+ ARTIFACT_INDEX_SERVICE: http://docker:8979
+ ARTIFACT_STORAGE_SERVICE: http://docker:8980
PYTEST_ARGS: "--color=yes --remote-cache"
+tests-bb-remote-execution:
+ <<: *tests
+ <<: *remote-test # Spin up server stack
+ variables:
+ <<: *docker-variables
+ COMPOSE_MANIFEST: .gitlab-ci/buildbarn-remote-execution.yml # < *remote-test
+ ARTIFACT_INDEX_SERVICE: http://docker:8979
+ ARTIFACT_STORAGE_SERVICE: http://docker:8980
+ REMOTE_EXECUTION_SERVICE: http://docker:8980
+ SOURCE_CACHE_SERVICE: http://docker:8980
+ PYTEST_ARGS: "--color=yes --remote-execution"
+
tests-no-usedevelop:
# Ensure that tests also pass without `--develop` flag.
image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
diff --git a/.gitlab-ci/buildbarn-remote-cache.yml b/.gitlab-ci/buildbarn-remote-cache.yml
index 0122fa8..4d22244 100644
--- a/.gitlab-ci/buildbarn-remote-cache.yml
+++ b/.gitlab-ci/buildbarn-remote-cache.yml
@@ -23,13 +23,11 @@ version: '3.4'
services:
bb-asset:
- image: qinusty/bb-asset-hub:20200814T141139Z-5e72dd1
+ image: buildbarn/bb-remote-asset:20200828T155422Z-9226c9e
command: /config/asset.jsonnet
restart: unless-stopped
- expose:
- - "7981"
ports:
- - "7981:7981"
+ - "8979:8979"
volumes:
- type: volume
source: assets
@@ -42,10 +40,8 @@ services:
image: buildbarn/bb-storage:20200810T194216Z-94a85b4
command: /config/storage.jsonnet
restart: unless-stopped
- expose:
- - "7982"
ports:
- - "7982:7982"
+ - 8980:8980
volumes:
- type: volume
source: cas
diff --git a/.gitlab-ci/buildbarn-remote-execution.yml b/.gitlab-ci/buildbarn-remote-execution.yml
new file mode 100644
index 0000000..75d0d36
--- /dev/null
+++ b/.gitlab-ci/buildbarn-remote-execution.yml
@@ -0,0 +1,83 @@
+version: '3'
+services:
+ frontend:
+ image: buildbarn/bb-storage:20200810T194216Z-94a85b4
+ command:
+ - /config/frontend.jsonnet
+ ports:
+ - 7980:7980
+ - 8980:8980
+ volumes:
+ - ./config:/config
+ - ac:/ac
+ - cas:/cas
+
+ scheduler:
+ image: buildbarn/bb-scheduler:20200816T115954Z-9fae5ec
+ command:
+ - /config/scheduler.jsonnet
+ expose:
+ - 8982
+ - 8983
+ ports:
+ - 7982:7982
+ volumes:
+ - ./config:/config
+
+ worker-ubuntu16-04:
+ image: buildbarn/bb-worker:20200816T115954Z-9fae5ec
+ command:
+ - /config/worker.jsonnet
+ ports:
+ - 7986:7986
+ volumes:
+ - ./config:/config
+ - ./worker:/worker
+ depends_on:
+ - worker-setup
+
+ runner-ubuntu16-04:
+ image: l.gcr.io/google/rbe-ubuntu16-04@sha256:b516a2d69537cb40a7c6a7d92d0008abb29fba8725243772bdaf2c83f1be2272
+ command:
+ - sh
+ - -c
+ - while ! test -f /bb/installed; do sleep 1; done; exec /bb/tini -v -g -- /bb/bb_runner /config/runner.jsonnet
+ network_mode: none
+ volumes:
+ - ./config:/config
+ - ./bb:/bb
+ - ./worker:/worker
+ depends_on:
+ - runner-installer
+ - worker-setup
+ - worker-ubuntu16-04
+
+ runner-installer:
+ image: buildbarn/bb-runner-installer:20200801T202247Z-4ea244f
+ volumes:
+ - ./bb:/bb
+
+ worker-setup:
+ image: busybox
+ volumes:
+ - ./worker:/worker
+ command:
+ - sh
+ - -c
+ - mkdir -m 0777 /worker/build && mkdir -m 0700 /worker/cache
+
+ bb-asset:
+ image: buildbarn/bb-remote-asset:20200828T155422Z-9226c9e
+ command: /config/asset.jsonnet
+ restart: unless-stopped
+ ports:
+ - 8979:8979
+ - 7981:7981
+ volumes:
+ - assets:/storage
+ - ./config/:/config
+
+volumes:
+ assets:
+ ac:
+ cas:
diff --git a/.gitlab-ci/config/asset.jsonnet b/.gitlab-ci/config/asset.jsonnet
index aad06a2..3bd5ab8 100644
--- a/.gitlab-ci/config/asset.jsonnet
+++ b/.gitlab-ci/config/asset.jsonnet
@@ -23,7 +23,7 @@
},
httpListenAddress: ':1111',
grpcServers: [{
- listenAddresses: [':7981'],
+ listenAddresses: [':8979'],
authenticationPolicy: { allow: {} },
}],
allowUpdatesForInstances: [''],
diff --git a/.gitlab-ci/config/common.libsonnet b/.gitlab-ci/config/common.libsonnet
new file mode 100644
index 0000000..75857fe
--- /dev/null
+++ b/.gitlab-ci/config/common.libsonnet
@@ -0,0 +1,12 @@
+{
+ blobstore: {
+ contentAddressableStorage: {
+ grpc: { address: 'frontend:8980' },
+ },
+ actionCache: {
+ grpc: { address: 'frontend:8980' },
+ },
+ },
+ httpListenAddress: ':80',
+ maximumMessageSizeBytes: 16 * 1024 * 1024,
+}
\ No newline at end of file
diff --git a/.gitlab-ci/config/frontend.jsonnet b/.gitlab-ci/config/frontend.jsonnet
new file mode 100644
index 0000000..486ca6d
--- /dev/null
+++ b/.gitlab-ci/config/frontend.jsonnet
@@ -0,0 +1,35 @@
+local common = import 'common.libsonnet';
+
+{
+ blobstore: {
+ contentAddressableStorage: {
+ circular: {
+ directory: '/cas',
+ offsetFileSizeBytes: 16 * 1024 * 1024,
+ offsetCacheSize: 10000,
+ dataFileSizeBytes: 10 * 1024 * 1024 * 1024,
+ dataAllocationChunkSizeBytes: 16 * 1024 * 1024,
+ },
+ },
+ actionCache: {
+ circular: {
+ directory: '/ac',
+ offsetFileSizeBytes: 1024 * 1024,
+ offsetCacheSize: 1000,
+ dataFileSizeBytes: 100 * 1024 * 1024,
+ dataAllocationChunkSizeBytes: 1048576,
+ instances: [''],
+ },
+ },
+ },
+ httpListenAddress: ':7980',
+ grpcServers: [{
+ listenAddresses: [':8980'],
+ authenticationPolicy: { allow: {} },
+ }],
+ schedulers: {
+ '': { endpoint: { address: 'scheduler:8981'} },
+ },
+ allowAcUpdatesForInstanceNamePrefixes: [''],
+ maximumMessageSizeBytes: common.maximumMessageSizeBytes,
+}
diff --git a/.gitlab-ci/config/runner.jsonnet b/.gitlab-ci/config/runner.jsonnet
new file mode 100644
index 0000000..bb09d45
--- /dev/null
+++ b/.gitlab-ci/config/runner.jsonnet
@@ -0,0 +1,9 @@
+
+{
+ buildDirectoryPath: '/worker/build',
+ grpcServers: [{
+ listenPaths: ['/worker/runner'],
+ authenticationPolicy: { allow: {} },
+ }],
+ chrootIntoInputRoot: true,
+}
\ No newline at end of file
diff --git a/.gitlab-ci/config/scheduler.jsonnet b/.gitlab-ci/config/scheduler.jsonnet
new file mode 100644
index 0000000..f3a558c
--- /dev/null
+++ b/.gitlab-ci/config/scheduler.jsonnet
@@ -0,0 +1,15 @@
+local common = import 'common.libsonnet';
+
+{
+ httpListenAddress: ':7982',
+ clientGrpcServers: [{
+ listenAddresses: [':8981'],
+ authenticationPolicy: { allow: {} },
+ }],
+ workerGrpcServers: [{
+ listenAddresses: [':8982'],
+ authenticationPolicy: { allow: {} },
+ }],
+ contentAddressableStorage: common.blobstore.contentAddressableStorage,
+ maximumMessageSizeBytes: common.maximumMessageSizeBytes,
+}
\ No newline at end of file
diff --git a/.gitlab-ci/config/storage.jsonnet b/.gitlab-ci/config/storage.jsonnet
index 11fbdbe..40a1056 100644
--- a/.gitlab-ci/config/storage.jsonnet
+++ b/.gitlab-ci/config/storage.jsonnet
@@ -18,7 +18,7 @@
},
httpListenAddress: ':6981',
grpcServers: [{
- listenAddresses: [':7982'],
+ listenAddresses: [':8980'],
authenticationPolicy: { allow: {} },
}],
allowAcUpdatesForInstanceNamePrefixes: [''],
diff --git a/.gitlab-ci/config/worker.jsonnet b/.gitlab-ci/config/worker.jsonnet
new file mode 100644
index 0000000..75ee4e3
--- /dev/null
+++ b/.gitlab-ci/config/worker.jsonnet
@@ -0,0 +1,37 @@
+local common = import 'common.libsonnet';
+
+{
+ blobstore: {
+ contentAddressableStorage: {
+ grpc: { address: 'frontend:8980' },
+ },
+ actionCache: {
+ grpc: { address: 'frontend:8980' },
+ }
+ },
+ maximumMessageSizeBytes: common.maximumMessageSizeBytes,
+ scheduler: { address: 'scheduler:8982' },
+ httpListenAddress: ':7986',
+ maximumMemoryCachedDirectories: 1000,
+ instanceName: '',
+ buildDirectories: [{
+ native: {
+ buildDirectoryPath: '/worker/build',
+ cacheDirectoryPath: '/worker/cache',
+ maximumCacheFileCount: 10000,
+ maximumCacheSizeBytes: 5 * 1024 * 1024 * 1024,
+ cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
+ },
+ runners: [{
+ endpoint: { address: 'unix:///worker/runner' },
+ concurrency: 8,
+ platform: {
+ properties: [{ name: "ISA" , value:"x86-64"}, { name:"OSFamily", value:"linux"}],
+ },
+ defaultExecutionTimeout: '1800s',
+ maximumExecutionTimeout: '3600s',
+ workerId: {
+ },
+ }],
+ }],
+}
\ No newline at end of file