[buildstream] 02/03: move caching sources from Element to SourceCache

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch abderrahim/source-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 829e30b1f50ff70bdd7ffa24fb73549e31c6a460
Author: Abderrahim Kitouni <ak...@gnome.org>
AuthorDate: Sun Jun 28 09:20:39 2020 +0100

    move caching sources from Element to SourceCache
---
 src/buildstream/_sourcecache.py | 34 ++++++++++++++++++++++++++-----
 src/buildstream/element.py      | 45 ++---------------------------------------
 2 files changed, 31 insertions(+), 48 deletions(-)

diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index dcde0b4..41efa5d 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -161,15 +161,24 @@ class SourceCache(BaseCache):
     # dependent on previous sources, such as the patch source.
     #
     # Args:
-    #    source: last source
-    #    previous_sources: rest of the sources.
-    def commit(self, source, previous_sources):
+    #    sources: list of the sources of an element
+    def commit(self, sources):
+        last_requires_previous = 0
+        # commit all other sources by themselves
+        for ix, source in enumerate(sources):
+            if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
+                self._commit_one(source, sources[last_requires_previous:ix])
+                last_requires_previous = ix
+            else:
+                self._commit_one(source, [])
+
+    def _commit_one(self, source, previous_sources):
         ref = source._get_source_name()
 
         # Use tmpdir for now
         vdir = CasBasedDirectory(self.cas)
         for previous_source in previous_sources:
-            vdir.import_files(self.export(previous_source))
+            vdir.import_files(self._export_one(previous_source))
 
         if not source.BST_STAGE_VIRTUAL_DIRECTORY:
             with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir:
@@ -191,7 +200,22 @@ class SourceCache(BaseCache):
     #
     # Returns:
     #    CASBasedDirectory
-    def export(self, source):
+    def export(self, sources):
+        # find last required source
+        last_requires_previous_ix = 0
+        for ix, source in enumerate(sources):
+            if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
+                last_requires_previous_ix = ix
+
+        import_dir = CasBasedDirectory(self.cas)
+
+        for source in sources[last_requires_previous_ix:]:
+            source_dir = self._export_one(source)
+            import_dir.import_files(source_dir)
+
+        return import_dir
+
+    def _export_one(self, source):
         ref = source._get_source_name()
         source = self._get_source(ref)
         return CasBasedDirectory(self.cas, digest=source.files)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 6a0fa5f..0a3f41c 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -106,7 +106,6 @@ from ._artifact import Artifact
 
 from .storage.directory import Directory
 from .storage._filebaseddirectory import FileBasedDirectory
-from .storage._casbaseddirectory import CasBasedDirectory
 from .storage.directory import VirtualDirectoryError
 
 if TYPE_CHECKING:
@@ -1352,16 +1351,8 @@ class Element(Plugin):
 
                 if self.__sources:
 
-                    sourcecache = context.sourcecache
-                    # find last required source
-                    last_required_previous_ix = self.__last_source_requires_previous()
-                    import_dir = CasBasedDirectory(context.get_cascache())
-
                     try:
-                        for source in self.__sources[last_required_previous_ix:]:
-                            source_dir = sourcecache.export(source)
-                            import_dir.import_files(source_dir)
-
+                        import_dir = context.sourcecache.export(self.__sources)
                     except SourceCacheError as e:
                         raise ElementError("Error trying to export source for {}: {}".format(self.name, e))
                     except VirtualDirectoryError as e:
@@ -2056,7 +2047,7 @@ class Element(Plugin):
                     source._fetch(previous_sources)
                 previous_sources.append(source)
 
-            self.__cache_sources()
+            self.__sourcecache.commit(self.__sources)
 
     # _calculate_cache_key():
     #
@@ -2889,38 +2880,6 @@ class Element(Plugin):
 
         return True
 
-    # __cache_sources():
-    #
-    # Caches the sources into the local CAS
-    #
-    def __cache_sources(self):
-        if self.__sources and not self._has_all_sources_in_source_cache():
-            last_requires_previous = 0
-            # commit all other sources by themselves
-            for ix, source in enumerate(self.__sources):
-                if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
-                    self.__sourcecache.commit(source, self.__sources[last_requires_previous:ix])
-                    last_requires_previous = ix
-                else:
-                    self.__sourcecache.commit(source, [])
-
-    # __last_source_requires_previous
-    #
-    # This is the last source that requires previous sources to be cached.
-    # Sources listed after this will be cached separately.
-    #
-    # Returns:
-    #    (int): index of last source that requires previous sources
-    #
-    def __last_source_requires_previous(self):
-        if self.__last_source_requires_previous_ix is None:
-            last_requires_previous = 0
-            for ix, source in enumerate(self.__sources):
-                if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE:
-                    last_requires_previous = ix
-            self.__last_source_requires_previous_ix = last_requires_previous
-        return self.__last_source_requires_previous_ix
-
     # __update_cache_keys()
     #
     # Updates weak and strict cache keys