[buildstream] 02/03: artifactcache: Read and write cache size to disk

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch jonathan/cache-cache-size
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit b31395f7850699492698d46f498aacd69e5e8b61
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Fri Aug 17 15:36:25 2018 +0100

    artifactcache: Read and write cache size to disk
---
 buildstream/_artifactcache/__init__.py      |  2 +-
 buildstream/_artifactcache/artifactcache.py | 55 +++++++++++++++++++++++++++--
 2 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/buildstream/_artifactcache/__init__.py b/buildstream/_artifactcache/__init__.py
index 07ed52b..fad483a 100644
--- a/buildstream/_artifactcache/__init__.py
+++ b/buildstream/_artifactcache/__init__.py
@@ -17,4 +17,4 @@
 #  Authors:
 #        Tristan Van Berkom <tr...@codethink.co.uk>
 
-from .artifactcache import ArtifactCache, ArtifactCacheSpec
+from .artifactcache import ArtifactCache, ArtifactCacheSpec, CACHE_SIZE_FILE
diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py
index 95dbc08..f28fe39 100644
--- a/buildstream/_artifactcache/artifactcache.py
+++ b/buildstream/_artifactcache/artifactcache.py
@@ -28,6 +28,9 @@ from .. import utils
 from .. import _yaml
 
 
+CACHE_SIZE_FILE = "cache_size"
+
+
 # An ArtifactCacheSpec holds the user configuration for a single remote
 # artifact cache.
 #
@@ -283,7 +286,11 @@ class ArtifactCache():
         # If we don't currently have an estimate, figure out the real
         # cache size.
         if self.estimated_size is None:
-            self.estimated_size = self.calculate_cache_size()
+            stored_size = self._read_cache_size()
+            if stored_size is not None:
+                self.estimated_size = stored_size
+            else:
+                self.estimated_size = self.calculate_cache_size()
 
         return self.estimated_size
 
@@ -542,6 +549,7 @@ class ArtifactCache():
             self.estimated_size = self.calculate_cache_size()
 
         self.estimated_size += artifact_size
+        self._write_cache_size(self.estimated_size)
 
     # _set_cache_size()
     #
@@ -552,6 +560,49 @@ class ArtifactCache():
     def _set_cache_size(self, cache_size):
         self.estimated_size = cache_size
 
+        # set_cache_size is called in cleanup, where it may set the cache to None
+        if self.estimated_size is not None:
+            self._write_cache_size(self.estimated_size)
+
+    # _write_cache_size()
+    #
+    # Writes the given size of the artifact to the cache's size file
+    #
+    def _write_cache_size(self, size):
+        assert isinstance(size, int)
+        size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
+        with open(size_file_path, "w") as f:
+            f.write(str(size))
+
+    # _read_cache_size()
+    #
+    # Reads and returns the size of the artifact cache that's stored in the
+    # cache's size file
+    #
+    def _read_cache_size(self):
+        size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
+
+        if not os.path.exists(size_file_path):
+            return None
+
+        with open(size_file_path, "r") as f:
+            size = f.read()
+
+        try:
+            num_size = int(size)
+        except ValueError as e:
+            raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
+                size, size_file_path)) from e
+
+        return num_size
+
+    # _calculate_cache_quota()
+    #
+    # Calculates and sets the cache quota and lower threshold based on the
+    # quota set in Context.
+    # It checks that the quota is both a valid expression, and that there is
+    # enough disk space to satisfy that quota
+    #
     def _calculate_cache_quota(self):
         # Headroom intended to give BuildStream a bit of leeway.
         # This acts as the minimum size of cache_quota and also
@@ -577,7 +628,7 @@ class ArtifactCache():
         stat = os.statvfs(artifactdir_volume)
         available_space = (stat.f_bsize * stat.f_bavail)
 
-        cache_size = self.calculate_cache_size()
+        cache_size = self.get_approximate_cache_size()
 
         # Ensure system has enough storage for the cache_quota
         #