You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ju...@apache.org on 2021/10/25 13:49:12 UTC
[buildstream] branch juerg/integration-cache updated (224ed10 ->
1fb916f)
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a change to branch juerg/integration-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
discard 224ed10 wip
discard f6d5644 test
discard bdf9c72 testing/integration.py: Make integration cache cleanup more robust
add 657d3b0 tests/artifactcache/config.py: Fix parallel test bug
add a3dcb04 Merge pull request #1478 from apache/juerg/parallel-test
add 58cad0e _artifactelement.py: Remove unused Artifact import
add ea17f54 _assetcache.py: Remove unused MappingNode import
add 88e29ed _cas/casserver.py: Fix --log-level default value with Click 8.0
add 4b42c15 .github: Work around bst-artifact-server bug in Docker image
add 04f14f9 .github: Rename remote service tests in CI
add e270e13 Merge pull request #1480 from apache/juerg/ci
add a99b414 .github/compose/ci.bst-artifact-server.yml: Use nightly Docker image
add bbc6d74 .github/compose/ci.bst-artifact-server.yml: Drop workaround
add c388631 Merge pull request #1481 from apache/juerg/ci
add dbbd2f4 _sandboxbuildboxrun.py: Drop --use-localcas
add eb12d1c Merge pull request #1482 from apache/juerg/buildbox-use-localcas
add 9ff82f7 element.py: Move get_logs() to internal API
add da9ebb2 Drop pull-artifact-files config option for remote execution
add 5995975 Always require artifact files
add a192d38 _sandboxremote.py: Drop unused output_files_required parameter
add 597f2e3 _cas: Add support for remote cache
add f125072 _context.py: Add support for storage-service in cache configuration
add 541c341 _sandboxremote.py: Make storage-service optional with remote cache
add cfe4044 _stream.py: Add warning to pull and fetch
add 6ec9017 tests/frontend/remote-caches.py: Add test for cache storage-service
add d429d87 tests/remoteexecution/remotecache.py: Add test for cache storage-service
add 1b9322d doc: Document storage-service in cache configuration
add fc24591 _scheduler: Add CacheQueryQueue to support cache query in job threads
add 9b6e676 _stream.py: Parallelize cache queries if a remote cache is configured
add 507ed31 Merge pull request #1431 from apache/juerg/remote-cache
add e9b68be _context.py: Fixed linting warning about unused import
add 58bad2c tests/cachekey/project/project.conf: Fix cache key test
add 0ef51a3 types.py: Add a CoreWarning for unaliased sources
add 288e2d2 source.py: Raise a fatal-able warning when using an unaliased source.
add 3fd5986 source.py: Raise an error at load time when encountering unresolved source aliases
add 2a1ee7d tests/frontend/show.py: Test (fatal) warnings for unaliased URLs in sources
add d8e9a25 tests/frontend/show.py: Added test for usage of invalid source aliases
add a096e53 Merge pull request #1491 from apache/tristan/unaliased-sources
add bbfb492 requirements/requirements.in: It depends on grpc >= 1.34
add 14868a0 Merge pull request #1490 from jjardon/jjardon/grpcio_1.34_master
add ab71290 _frontend/complete.py: Workaround API break in click.Path()
add af133e2 requirements: Refresh the static required versions used in tox environments.
add e3333aa tests/frontend/pull.py: Updated formatting to make black happier
add 1a3166d .pylintrc: Disable unused-private-member error
add 1a574de Address pylint errors
add e0a241c .github/run-ci.sh: Added script to run the github CI with docker locally.
add ca4bfc3 .github/workflows/ci.yml: Route the github CI through the new script
add be318bc tests/plugins/loading.py: Refactor test to not depend on yaml.dump() formatting
add b66e85b tests/internals/yaml.py: Mark test_roundtrip_dump() as failing
add da09edd .github/: Remove the 'no-usedevelop' CI
add 44e2207 .github/compose/ci.docker-compose.yml: Wholesale disable test parallelism
add 187d801 .github/compose/ci.docker-compose.yml: Reinstate format checking with black
add ed48998 Merge pull request #1494 from apache/tristan/refresh-deps
add 0b5326d .github/workflows/ci.yml: Use github concurrency feature instead of branch name globbing
add de5bf1d Merge pull request #1510 from apache/tristan/fix-external-fork-ci
add 8bf8a09 doc/source/developing/workspaces.rst: Rewriting documentation surrounding `bst workspace reset`
add be54a5e Removing flatpak-autotools example
add 807f0d2 data/projectconfig.yaml: Remove %{fix-pyc-timestamps}
add dda8310 doc/source/format_project.rst: Link to command docs in options docs
add bcb4d07 Merge pull request #1509 from apache/tristan/license-freedom
add 71736af source.py: Remove ambiguous kwargs from public API signature
add 6523ae7 Merge pull request #1517 from apache/tristan/source-api-kwargs
add 1bce9c4 source.py: Document Source.set_ref() properly
add 0bf1ea7 Merge pull request #1516 from apache/tristan/source-set-ref
add 9c75683 _workspaces.py: Use bst2 specific data directory for private project state
add f8f00f0 Merge pull request #1515 from apache/tristan/workspace-data-namespacing
add 24abe42 element.py: Make Element.configure_dependencies() take an Iterable
add 32a7d3f element.py: Add link to SandboxFlags in docs for Element.batch_prepare_assemble()
add 18e9320 plugin.py: Harden the Plugin.call() and Plugin.check_output() APIs
add dcb228d Merge pull request #1518 from apache/tristan/type-annotations
add 742c783 _project.py: Add support for `project-root` and `toplevel-root` variables
add 9811b51 tests/frontend/fetch.py: Use project-root variable instead of generating a project.conf
add 1d71d98 tests/frontend/mirror.py: Test project-root and toplevel-root expansion
add c2874f1 doc/source/format_declaring.rst: Document the dynamic `project-root` and `toplevel-root` variables
add 16a0bed Merge pull request #1519 from apache/tristan/project-path-variable
add ef60a6f tests/frontend/show.py: Remove test_planned_order()
add 64f8a45 tests/format/dependencies.py: Remove test_no_recurse()
add 0a7ff9e _scheduler/queues/queue.py: Prioritize enqueue order
add 850a3a5 Remove PipelineSelection.PLAN and any related frontend options
add 48be7f3 doc/source/using_config.rst: Amending docs for default build deps
add 06a02fb NEWS: Added notice of breaking change and removal of `--deps plan`
add 89100e1 Merge pull request #1520 from apache/tristan/remove-plan-selection-mode
add 971731e tests/frontend/buildcheckout.py: Remove element name tests from here
add b62dbec Strict element naming format.
add 8378283 tests/format/elementnames.py: Added format test for element names
add eb0d9f5 NEWS: Document breaking change of requiring valid element names
add 0b8c2e1 Merge pull request #1523 from apache/tristan/strict-element-names
add 8519dc3 Remove `--pull` option from `bst shell`, and pull dependencies by default
add 3624735 Remove `--pull` option from `bst artifact checkout`, and pull dependencies by default
add 147ca3d NEWS: Document breaking change of removing `--pull` options from some commands
add 1ab661b Merge pull request #1522 from apache/tristan/shell-checkout-always-pull
add e97d16b Revert to expecting '.bst' on the command line to disambiguate elements and artifacts
add 43a8b3e _frontend/cli.py: Enhanced the `bst artifact --help` output
add 556cb91 NEWS: Document new/restored behavior of `bst artifact` commands
add 15cd166 Merge pull request #1524 from apache/tristan/disambiguate-artifact-globs
add 7e7e140 .github: Remove testing of `bst-artifact-server` service tests
add 5138b50 Remove `bst-artifact-server` entrypoint
add 7a35637 doc: Remove `bst-artifact-server` docs, refer to Buildbarn
add f6181e7 Merge pull request #1526 from apache/tristan/remove-artifact-server
new 1fb916f testing: Use integration cache only for sources
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (224ed10)
\
N -- N -- N refs/heads/juerg/integration-cache (1fb916f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
...buildbarn-remote-cache.yml => ci.buildbarn.yml} | 4 +-
.../{ci.remote-execution.yml => ci.buildgrid.yml} | 6 +-
.github/compose/ci.buildstream-remote-cache.yml | 36 -
.github/compose/ci.docker-compose.yml | 26 +-
.github/run-ci.sh | 123 +++
.github/workflows/ci.yml | 63 +-
.pylintrc | 5 +
NEWS | 17 +
doc/examples/flatpak-autotools/elements/base.bst | 6 -
.../flatpak-autotools/elements/base/sdk.bst | 16 -
.../flatpak-autotools/elements/base/usrmerge.bst | 11 -
doc/examples/flatpak-autotools/elements/hello.bst | 9 -
doc/examples/flatpak-autotools/files/links/bin | 1 -
doc/examples/flatpak-autotools/files/links/etc | 1 -
doc/examples/flatpak-autotools/files/links/lib | 1 -
.../flatpak-autotools/files/src/Makefile.am | 6 -
doc/examples/flatpak-autotools/files/src/README | 2 -
.../flatpak-autotools/files/src/aclocal.m4 | 1152 --------------------
doc/examples/flatpak-autotools/files/src/compile | 347 ------
.../flatpak-autotools/files/src/configure.ac | 13 -
doc/examples/flatpak-autotools/files/src/depcomp | 791 --------------
.../flatpak-autotools/files/src/install-sh | 501 ---------
doc/examples/flatpak-autotools/files/src/missing | 215 ----
.../flatpak-autotools/files/src/src/Makefile.am | 6 -
.../flatpak-autotools/files/src/src/main.c | 14 -
doc/examples/flatpak-autotools/keys/flathub.gpg | Bin 2844 -> 0 bytes
doc/examples/flatpak-autotools/project.conf | 22 -
doc/sessions/flatpak-autotools.run | 21 -
doc/source/arch_caches.rst | 19 +-
doc/source/developing/workspaces.rst | 44 +-
doc/source/examples/flatpak-autotools.rst | 148 ---
doc/source/format_declaring.rst | 55 +
doc/source/format_project.rst | 6 +-
.../sessions-stored/flatpak-autotools-build.html | 145 ---
.../sessions-stored/flatpak-autotools-shell.html | 23 -
doc/source/using_config.rst | 71 +-
doc/source/using_configuring_cache_server.rst | 238 +---
doc/source/using_examples.rst | 1 -
requirements/cov-requirements.txt | 10 +-
requirements/dev-requirements.txt | 25 +-
requirements/requirements.in | 2 +-
requirements/requirements.txt | 28 +-
setup.py | 25 +-
src/buildstream/_artifact.py | 13 +-
src/buildstream/_artifactelement.py | 1 -
src/buildstream/_assetcache.py | 1 -
src/buildstream/_cas/cascache.py | 90 +-
src/buildstream/_cas/casdprocessmanager.py | 19 +-
src/buildstream/_cas/casremote.py | 5 +
src/buildstream/_cas/casserver.py | 32 +-
src/buildstream/_context.py | 55 +-
src/buildstream/_elementproxy.py | 3 -
src/buildstream/_frontend/app.py | 4 +-
src/buildstream/_frontend/cli.py | 82 +-
src/buildstream/_frontend/complete.py | 14 +-
src/buildstream/_frontend/widget.py | 20 +-
src/buildstream/_gitsourcebase.py | 4 +-
src/buildstream/_loader/loader.py | 77 +-
src/buildstream/_messenger.py | 2 +-
src/buildstream/_pipeline.py | 46 +-
src/buildstream/_profile.py | 2 +-
src/buildstream/_project.py | 25 +
src/buildstream/_remote.py | 9 +-
src/buildstream/_remotespec.py | 27 +-
src/buildstream/_scheduler/__init__.py | 1 +
.../_scheduler/queues/cachequeryqueue.py | 66 ++
src/buildstream/_scheduler/queues/queue.py | 11 +-
src/buildstream/_stream.py | 315 +++---
src/buildstream/_workspaces.py | 2 +-
src/buildstream/data/projectconfig.yaml | 6 -
src/buildstream/data/userconfig.yaml | 2 +-
src/buildstream/downloadablefilesource.py | 4 +-
src/buildstream/element.py | 57 +-
src/buildstream/exceptions.py | 12 +
src/buildstream/plugin.py | 161 ++-
src/buildstream/sandbox/_sandboxbuildboxrun.py | 3 +-
src/buildstream/sandbox/_sandboxremote.py | 111 +-
src/buildstream/source.py | 51 +-
src/buildstream/storage/_filebaseddirectory.py | 2 +-
src/buildstream/testing/_cachekeys.py | 2 +-
.../testing/_sourcetests/source_determinism.py | 4 +-
.../testing/_sourcetests/track_cross_junction.py | 2 +-
src/buildstream/testing/_update_cachekeys.py | 2 +-
src/buildstream/testing/_utils/site.py | 2 +-
src/buildstream/testing/integration.py | 18 +-
src/buildstream/testing/runcli.py | 26 +-
src/buildstream/types.py | 17 +-
src/buildstream/utils.py | 13 +-
tests/artifactcache/config.py | 8 +-
tests/cachekey/project/project.conf | 3 +
tests/examples/developing.py | 2 +-
tests/examples/flatpak-autotools.py | 90 --
tests/format/dependencies.py | 21 -
tests/format/elementnames.py | 43 +
.../elementnames/bad-chars-dep.bst} | 3 +-
.../elementnames/bad-suffix-dep.bst} | 3 +-
.../{dependencies2 => elementnames}/project.conf | 2 +-
tests/format/project.py | 2 +-
tests/frontend/artifact_checkout.py | 3 +-
tests/frontend/artifact_log.py | 4 +-
tests/frontend/artifact_show.py | 24 -
tests/frontend/buildcheckout.py | 86 +-
tests/frontend/completions.py | 4 +-
tests/frontend/cross_junction_workspace.py | 2 +-
tests/frontend/fetch.py | 6 +-
tests/frontend/init.py | 4 +-
tests/frontend/large_directory.py | 2 +-
tests/frontend/logging.py | 2 +-
tests/frontend/mirror.py | 53 +-
tests/frontend/order.py | 20 +-
tests/frontend/progress.py | 2 +-
tests/frontend/project/elements/invalid-alias.bst | 5 +
.../project/elements/invalid-chars-in-dep.bst | 8 -
tests/frontend/project/elements/target2.bst | 7 -
tests/frontend/project/elements/unaliased-tar.bst | 5 +
tests/frontend/project/project.conf | 4 -
tests/frontend/pull.py | 12 +-
tests/frontend/push.py | 4 +-
tests/frontend/rebuild.py | 4 +-
tests/frontend/remote-caches.py | 30 +
tests/frontend/show.py | 77 +-
tests/frontend/source-fetch/project.conf | 5 +
tests/frontend/source_checkout.py | 4 +-
tests/frontend/track.py | 6 +-
tests/frontend/workspace.py | 32 +-
tests/integration/artifact.py | 2 +-
tests/integration/manual.py | 12 +-
tests/integration/script.py | 16 +-
tests/integration/shell.py | 14 +-
tests/integration/shellbuildtrees.py | 57 +-
tests/integration/source-determinism.py | 4 +-
tests/integration/stack.py | 4 +-
tests/integration/workspace.py | 16 +-
tests/internals/loader.py | 3 +-
tests/internals/storage_vdir_import.py | 12 +-
tests/internals/utils_save_atomic.py | 10 +-
tests/internals/yaml.py | 7 +-
tests/plugins/loading.py | 21 +-
.../{project.conf => not-found-elements.conf} | 7 +
.../{project.conf => not-found-sources.conf} | 7 +
tests/remoteexecution/junction.py | 4 +-
tests/remoteexecution/partial.py | 78 --
.../remoteexecution/{simple.py => remotecache.py} | 40 +-
tests/remoteexecution/workspace.py | 10 +-
tests/sourcecache/cache.py | 2 +-
tests/sources/bzr.py | 2 +-
tests/sources/git.py | 34 +-
tests/sources/local.py | 6 +-
tests/sources/patch.py | 12 +-
tests/sources/previous_source_access.py | 4 +-
tests/testutils/filetypegenerator.py | 2 +-
tests/testutils/python_repo.py | 6 +-
152 files changed, 1614 insertions(+), 5046 deletions(-)
rename .github/compose/{ci.buildbarn-remote-cache.yml => ci.buildbarn.yml} (90%)
rename .github/compose/{ci.remote-execution.yml => ci.buildgrid.yml} (88%)
delete mode 100644 .github/compose/ci.buildstream-remote-cache.yml
create mode 100755 .github/run-ci.sh
delete mode 100644 doc/examples/flatpak-autotools/elements/base.bst
delete mode 100644 doc/examples/flatpak-autotools/elements/base/sdk.bst
delete mode 100644 doc/examples/flatpak-autotools/elements/base/usrmerge.bst
delete mode 100644 doc/examples/flatpak-autotools/elements/hello.bst
delete mode 120000 doc/examples/flatpak-autotools/files/links/bin
delete mode 120000 doc/examples/flatpak-autotools/files/links/etc
delete mode 120000 doc/examples/flatpak-autotools/files/links/lib
delete mode 100644 doc/examples/flatpak-autotools/files/src/Makefile.am
delete mode 100644 doc/examples/flatpak-autotools/files/src/README
delete mode 100644 doc/examples/flatpak-autotools/files/src/aclocal.m4
delete mode 100755 doc/examples/flatpak-autotools/files/src/compile
delete mode 100644 doc/examples/flatpak-autotools/files/src/configure.ac
delete mode 100755 doc/examples/flatpak-autotools/files/src/depcomp
delete mode 100755 doc/examples/flatpak-autotools/files/src/install-sh
delete mode 100755 doc/examples/flatpak-autotools/files/src/missing
delete mode 100644 doc/examples/flatpak-autotools/files/src/src/Makefile.am
delete mode 100644 doc/examples/flatpak-autotools/files/src/src/main.c
delete mode 100644 doc/examples/flatpak-autotools/keys/flathub.gpg
delete mode 100644 doc/examples/flatpak-autotools/project.conf
delete mode 100644 doc/sessions/flatpak-autotools.run
delete mode 100644 doc/source/examples/flatpak-autotools.rst
delete mode 100644 doc/source/sessions-stored/flatpak-autotools-build.html
delete mode 100644 doc/source/sessions-stored/flatpak-autotools-shell.html
create mode 100644 src/buildstream/_scheduler/queues/cachequeryqueue.py
delete mode 100644 tests/examples/flatpak-autotools.py
create mode 100644 tests/format/elementnames.py
copy tests/{frontend/workspaced-runtime-dep/elements/stack.bst => format/elementnames/bad-chars-dep.bst} (56%)
copy tests/{frontend/workspaced-runtime-dep/elements/stack.bst => format/elementnames/bad-suffix-dep.bst} (58%)
copy tests/format/{dependencies2 => elementnames}/project.conf (87%)
create mode 100644 tests/frontend/project/elements/invalid-alias.bst
delete mode 100644 tests/frontend/project/elements/invalid-chars-in-dep.bst
delete mode 100644 tests/frontend/project/elements/target2.bst
create mode 100644 tests/frontend/project/elements/unaliased-tar.bst
create mode 100644 tests/frontend/source-fetch/project.conf
copy tests/plugins/loading/{project.conf => not-found-elements.conf} (53%)
copy tests/plugins/loading/{project.conf => not-found-sources.conf} (54%)
delete mode 100644 tests/remoteexecution/partial.py
copy tests/remoteexecution/{simple.py => remotecache.py} (53%)
[buildstream] 01/01: testing: Use integration cache only for sources
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
juergbi pushed a commit to branch juerg/integration-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1fb916fe64eb609d62c290933fb15269709ca4c9
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Mon Oct 25 08:03:01 2021 +0200
testing: Use integration cache only for sources
The main purpose of the integration cache is to avoid redownloading
sources. Keeping the complete CAS cache across the whole test session
can result in excessive disk usage, breaking CI on shared runners.
FAILURE Cache too full
This restricts the integration cache to sources and uses the per-test
temp directory for the CAS cache, same as for non-integration tests.
---
src/buildstream/testing/integration.py | 22 +---------------------
src/buildstream/testing/runcli.py | 14 ++------------
2 files changed, 3 insertions(+), 33 deletions(-)
diff --git a/src/buildstream/testing/integration.py b/src/buildstream/testing/integration.py
index 269b3d3..fbef08f 100644
--- a/src/buildstream/testing/integration.py
+++ b/src/buildstream/testing/integration.py
@@ -78,15 +78,6 @@ class IntegrationCache:
# Use the same sources every time
self.sources = os.path.join(self.root, "sources")
- # Create a temp directory for the duration of the test for
- # the artifacts directory
- try:
- self.cachedir = tempfile.mkdtemp(dir=self.root, prefix="cache-")
- # Apply mode allowed by umask
- os.chmod(self.cachedir, 0o777 & ~utils.get_umask())
- except OSError as e:
- raise AssertionError("Unable to create test directory !") from e
-
@pytest.fixture(scope="session")
def integration_cache(request):
@@ -99,15 +90,4 @@ def integration_cache(request):
cache = IntegrationCache(cache_dir)
- yield cache
-
- # Clean up the artifacts after each test session - we only want to
- # cache sources between tests
- try:
- shutil.rmtree(cache.cachedir)
- except FileNotFoundError:
- pass
- try:
- shutil.rmtree(os.path.join(cache.root, "cas"))
- except FileNotFoundError:
- pass
+ return cache
diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py
index 088c2a8..9dc2b4c 100644
--- a/src/buildstream/testing/runcli.py
+++ b/src/buildstream/testing/runcli.py
@@ -753,20 +753,10 @@ def cli_integration(tmpdir, integration_cache):
# We want to cache sources for integration tests more permanently,
# to avoid downloading the huge base-sdk repeatedly
fixture.configure(
- {"cachedir": integration_cache.cachedir, "sourcedir": integration_cache.sources,}
+ {"sourcedir": integration_cache.sources,}
)
- yield fixture
-
- # remove following folders if necessary
- try:
- shutil.rmtree(os.path.join(integration_cache.cachedir, "build"))
- except FileNotFoundError:
- pass
- try:
- shutil.rmtree(os.path.join(integration_cache.cachedir, "tmp"))
- except FileNotFoundError:
- pass
+ return fixture
# A variant of the main fixture that is configured for remote-execution.