You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:07:15 UTC
[buildstream] branch abderrahim/source-cache created (now 6df2fa8)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch abderrahim/source-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 6df2fa8 sourcecache: keep sources in tmpdir for subsequent sources than need them
This branch includes the following new commits:
new 17825d7 tests: don't use sourcecache.export()
new 829e30b move caching sources from Element to SourceCache
new 6df2fa8 sourcecache: keep sources in tmpdir for subsequent sources than need them
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/03: move caching sources from Element to
SourceCache
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch abderrahim/source-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 829e30b1f50ff70bdd7ffa24fb73549e31c6a460
Author: Abderrahim Kitouni <ak...@gnome.org>
AuthorDate: Sun Jun 28 09:20:39 2020 +0100
move caching sources from Element to SourceCache
---
src/buildstream/_sourcecache.py | 34 ++++++++++++++++++++++++++-----
src/buildstream/element.py | 45 ++---------------------------------------
2 files changed, 31 insertions(+), 48 deletions(-)
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index dcde0b4..41efa5d 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -161,15 +161,24 @@ class SourceCache(BaseCache):
# dependent on previous sources, such as the patch source.
#
# Args:
- # source: last source
- # previous_sources: rest of the sources.
- def commit(self, source, previous_sources):
+ # sources: list of the sources of an element
+ def commit(self, sources):
+ last_requires_previous = 0
+ # commit all other sources by themselves
+ for ix, source in enumerate(sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
+ self._commit_one(source, sources[last_requires_previous:ix])
+ last_requires_previous = ix
+ else:
+ self._commit_one(source, [])
+
+ def _commit_one(self, source, previous_sources):
ref = source._get_source_name()
# Use tmpdir for now
vdir = CasBasedDirectory(self.cas)
for previous_source in previous_sources:
- vdir.import_files(self.export(previous_source))
+ vdir.import_files(self._export_one(previous_source))
if not source.BST_STAGE_VIRTUAL_DIRECTORY:
with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir:
@@ -191,7 +200,22 @@ class SourceCache(BaseCache):
#
# Returns:
# CASBasedDirectory
- def export(self, source):
+ def export(self, sources):
+ # find last required source
+ last_requires_previous_ix = 0
+ for ix, source in enumerate(sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
+ last_requires_previous_ix = ix
+
+ import_dir = CasBasedDirectory(self.cas)
+
+ for source in sources[last_requires_previous_ix:]:
+ source_dir = self._export_one(source)
+ import_dir.import_files(source_dir)
+
+ return import_dir
+
+ def _export_one(self, source):
ref = source._get_source_name()
source = self._get_source(ref)
return CasBasedDirectory(self.cas, digest=source.files)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 6a0fa5f..0a3f41c 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -106,7 +106,6 @@ from ._artifact import Artifact
from .storage.directory import Directory
from .storage._filebaseddirectory import FileBasedDirectory
-from .storage._casbaseddirectory import CasBasedDirectory
from .storage.directory import VirtualDirectoryError
if TYPE_CHECKING:
@@ -1352,16 +1351,8 @@ class Element(Plugin):
if self.__sources:
- sourcecache = context.sourcecache
- # find last required source
- last_required_previous_ix = self.__last_source_requires_previous()
- import_dir = CasBasedDirectory(context.get_cascache())
-
try:
- for source in self.__sources[last_required_previous_ix:]:
- source_dir = sourcecache.export(source)
- import_dir.import_files(source_dir)
-
+ import_dir = context.sourcecache.export(self.__sources)
except SourceCacheError as e:
raise ElementError("Error trying to export source for {}: {}".format(self.name, e))
except VirtualDirectoryError as e:
@@ -2056,7 +2047,7 @@ class Element(Plugin):
source._fetch(previous_sources)
previous_sources.append(source)
- self.__cache_sources()
+ self.__sourcecache.commit(self.__sources)
# _calculate_cache_key():
#
@@ -2889,38 +2880,6 @@ class Element(Plugin):
return True
- # __cache_sources():
- #
- # Caches the sources into the local CAS
- #
- def __cache_sources(self):
- if self.__sources and not self._has_all_sources_in_source_cache():
- last_requires_previous = 0
- # commit all other sources by themselves
- for ix, source in enumerate(self.__sources):
- if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
- self.__sourcecache.commit(source, self.__sources[last_requires_previous:ix])
- last_requires_previous = ix
- else:
- self.__sourcecache.commit(source, [])
-
- # __last_source_requires_previous
- #
- # This is the last source that requires previous sources to be cached.
- # Sources listed after this will be cached separately.
- #
- # Returns:
- # (int): index of last source that requires previous sources
- #
- def __last_source_requires_previous(self):
- if self.__last_source_requires_previous_ix is None:
- last_requires_previous = 0
- for ix, source in enumerate(self.__sources):
- if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
- last_requires_previous = ix
- self.__last_source_requires_previous_ix = last_requires_previous
- return self.__last_source_requires_previous_ix
-
# __update_cache_keys()
#
# Updates weak and strict cache keys
[buildstream] 01/03: tests: don't use sourcecache.export()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch abderrahim/source-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 17825d790cffdbf236eae0801ffcc20fc04385e9
Author: Abderrahim Kitouni <ak...@gnome.org>
AuthorDate: Sun Jun 28 20:14:53 2020 +0100
tests: don't use sourcecache.export()
---
tests/sourcecache/fetch.py | 4 ++--
tests/sourcecache/push.py | 4 ++--
tests/sourcecache/staging.py | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index 889de62..78185e5 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -96,7 +96,7 @@ def test_source_fetch(cli, tmpdir, datafiles):
# get root digest of source
sourcecache = context.sourcecache
- digest = sourcecache.export(source)._get_digest()
+ digest = sourcecache._get_source(source._get_source_name()).files
move_local_cas_to_remote_source_share(str(cache_dir), share.directory)
@@ -204,7 +204,7 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
# get root digest of source
sourcecache = context.sourcecache
- digest = sourcecache.export(source)._get_digest()
+ digest = sourcecache._get_source(source._get_source_name()).files
move_local_cas_to_remote_source_share(str(cache_dir), share.directory)
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 9f8663a..e4a12f8 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -101,7 +101,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
assert sourcecache.contains(source)
# check that the remote CAS now has it
- digest = sourcecache.export(source)._get_digest()
+ digest = sourcecache._get_source(source._get_source_name()).files
assert storage.has_object(digest)
@@ -150,7 +150,7 @@ def test_source_push(cli, tmpdir, datafiles):
assert sourcecache.contains(source)
# check that the remote CAS now has it
- digest = sourcecache.export(source)._get_digest()
+ digest = sourcecache._get_source(source._get_source_name()).files
assert share.has_object(digest)
diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py
index dbfc028..6ddef48 100644
--- a/tests/sourcecache/staging.py
+++ b/tests/sourcecache/staging.py
@@ -69,7 +69,7 @@ def test_source_staged(tmpdir, cli, datafiles):
assert sourcecache.contains(source)
# Extract the file and check it's the same as the one we imported
- digest = sourcecache.export(source)._get_digest()
+ digest = sourcecache._get_source(source._get_source_name()).files
extractdir = os.path.join(str(tmpdir), "extract")
cas.checkout(extractdir, digest)
dir1 = extractdir
@@ -102,7 +102,7 @@ def test_source_fetch(tmpdir, cli, datafiles):
assert element._has_all_sources_in_source_cache()
# check that the directory structures are identical
- digest = sourcecache.export(source)._get_digest()
+ digest = sourcecache._get_source(source._get_source_name()).files
extractdir = os.path.join(str(tmpdir), "extract")
cas.checkout(extractdir, digest)
dir1 = extractdir
[buildstream] 03/03: sourcecache: keep sources in tmpdir for
subsequent sources than need them
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch abderrahim/source-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6df2fa8be8773f7156763f6bf933b0535bc2246b
Author: Abderrahim Kitouni <ak...@gnome.org>
AuthorDate: Sun Jun 28 20:51:31 2020 +0100
sourcecache: keep sources in tmpdir for subsequent sources than need them
This should speed up caching of elements that have e.g. multiple patches
---
src/buildstream/_sourcecache.py | 55 ++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 22 deletions(-)
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index 41efa5d..2af0aa0 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -23,6 +23,7 @@ import grpc
from ._remote import BaseRemote
from ._cas.casremote import BlobNotFound
from .storage._casbaseddirectory import CasBasedDirectory
+from .storage._filebaseddirectory import FileBasedDirectory
from ._basecache import BaseCache
from ._exceptions import CASError, CASRemoteError, SourceCacheError, RemoteError
from . import utils
@@ -167,30 +168,43 @@ class SourceCache(BaseCache):
# commit all other sources by themselves
for ix, source in enumerate(sources):
if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
- self._commit_one(source, sources[last_requires_previous:ix])
last_requires_previous = ix
- else:
- self._commit_one(source, [])
- def _commit_one(self, source, previous_sources):
- ref = source._get_source_name()
+ with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as all_sources_tmpdir:
+ all_sources_vdir = FileBasedDirectory(all_sources_tmpdir)
+
+ for i, source in enumerate(sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
+ self._commit_one(source, all_sources_vdir)
+ else:
+ source_vdir = self._commit_one(source)
+ if i < last_requires_previous:
+ all_sources_vdir.import_files(source_vdir)
- # Use tmpdir for now
+ def _commit_one(self, source, previous_sources_vdir=None):
+ ref = source._get_source_name()
vdir = CasBasedDirectory(self.cas)
- for previous_source in previous_sources:
- vdir.import_files(self._export_one(previous_source))
-
- if not source.BST_STAGE_VIRTUAL_DIRECTORY:
- with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir:
- if not vdir.is_empty():
- vdir.export_files(tmpdir)
- source._stage(tmpdir)
- vdir.import_files(tmpdir, can_link=True)
+
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
+ if source.BST_STAGE_VIRTUAL_DIRECTORY:
+ source._stage(previous_sources_vdir)
+ else:
+ source._stage(previous_sources_vdir._get_underlying_directory())
+ previous_sources_vdir._mark_changed()
+
+ vdir.import_files(previous_sources_vdir)
else:
- source._stage(vdir)
+ if source.BST_STAGE_VIRTUAL_DIRECTORY:
+ source._stage(vdir)
+ else:
+ with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir:
+ source._stage(tmpdir)
+ vdir.import_files(tmpdir, can_link=True)
self._store_source(ref, vdir._get_digest())
+ return vdir
+
# export()
#
# Exports a source in the CAS to a virtual directory
@@ -210,16 +224,13 @@ class SourceCache(BaseCache):
import_dir = CasBasedDirectory(self.cas)
for source in sources[last_requires_previous_ix:]:
- source_dir = self._export_one(source)
+ ref = source._get_source_name()
+ source_proto = self._get_source(ref)
+ source_dir = CasBasedDirectory(self.cas, digest=source_proto.files)
import_dir.import_files(source_dir)
return import_dir
- def _export_one(self, source):
- ref = source._get_source_name()
- source = self._get_source(ref)
- return CasBasedDirectory(self.cas, digest=source.files)
-
# pull()
#
# Attempts to pull sources from configure remote source caches.