You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:07:08 UTC
[buildstream] branch jennis/cache_quota created (now 7a25600)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch jennis/cache_quota
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 7a25600 cascache.py: Clearly separate CASCache's public and local private methods
This branch includes the following new commits:
new 52cddcb cascache.py: Pretty print the cache quota
new 3479862 CASQuota: Don't calculate the quota when we don't need too
new 7fa7b5c cascache.py: Make calculate_cache_quota public
new 7a25600 cascache.py: Clearly separate CASCache's public and local private methods
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/04: CASQuota: Don't calculate the quota when we
don't need too
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jennis/cache_quota
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3479862abfa2f42ea22b6b26de84ebbe3a6b63f0
Author: James Ennis <ja...@codethink.co.uk>
AuthorDate: Thu Apr 18 15:54:16 2019 +0100
CASQuota: Don't calculate the quota when we don't need too
---
buildstream/_basecache.py | 1 -
buildstream/_cas/cascache.py | 6 ++++--
tests/artifactcache/cache_size.py | 3 ++-
tests/artifactcache/expiry.py | 4 ++--
4 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/buildstream/_basecache.py b/buildstream/_basecache.py
index af3fe9b..d47199b 100644
--- a/buildstream/_basecache.py
+++ b/buildstream/_basecache.py
@@ -40,7 +40,6 @@ class BaseCache():
self.context = context
self.cas = context.get_cascache()
self.casquota = context.get_casquota()
- self.casquota._calculate_cache_quota()
self._remotes_setup = False # Check to prevent double-setup of remotes
# Per-project list of _CASRemote instances.
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 8d8f3f0..647ece5 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -58,6 +58,8 @@ CACHE_SIZE_FILE = "cache_size"
class CASCacheUsage():
def __init__(self, casquota):
+ if not casquota._cache_quota_original:
+ casquota._calculate_cache_quota()
self.quota_config = casquota._config_cache_quota # Configured quota
self.quota_size = casquota._cache_quota_original # Resolved cache quota in bytes
self.used_size = casquota.get_cache_size() # Size used by artifacts in bytes
@@ -1151,8 +1153,6 @@ class CASQuota:
self._ref_callbacks = [] # Call backs to get required refs
self._remove_callbacks = [] # Call backs to remove refs
- self._calculate_cache_quota()
-
# compute_cache_size()
#
# Computes the real artifact cache size.
@@ -1218,6 +1218,8 @@ class CASQuota:
#
def full(self):
+ if not self._cache_quota:
+ self._calculate_cache_quota()
if self.get_cache_size() > self._cache_quota:
return True
diff --git a/tests/artifactcache/cache_size.py b/tests/artifactcache/cache_size.py
index 09f0f25..a7db496 100644
--- a/tests/artifactcache/cache_size.py
+++ b/tests/artifactcache/cache_size.py
@@ -75,6 +75,7 @@ def test_quota_over_1024T(cli, tmpdir):
project = tmpdir.join("main")
os.makedirs(str(project))
_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
+ create_element_size("file.bst", project, ".", [], 1024)
volume_space_patch = mock.patch(
"buildstream._cas.CASQuota._get_cache_volume_size",
@@ -83,5 +84,5 @@ def test_quota_over_1024T(cli, tmpdir):
)
with volume_space_patch:
- result = cli.run(project, args=["build", "file.bst"])
+ result = cli.run(project, args=["build", "file.bst"], project=project)
result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')
diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py
index 6aa518f..46879cf 100644
--- a/tests/artifactcache/expiry.py
+++ b/tests/artifactcache/expiry.py
@@ -348,7 +348,7 @@ def test_never_delete_required_track(cli, datafiles):
def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason):
project = str(datafiles)
os.makedirs(os.path.join(project, 'elements'))
-
+ create_element_size("file.bst", project, 'elements', [], 1024)
cli.configure({
'cache': {
'quota': quota,
@@ -382,7 +382,7 @@ def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason):
)
with volume_space_patch, cache_size_patch:
- res = cli.run(project=project, args=['workspace', 'list'])
+ res = cli.run(project=project, args=['build', 'file.bst'])
if err_domain == 'success':
res.assert_success()
[buildstream] 03/04: cascache.py: Make calculate_cache_quota public
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jennis/cache_quota
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7fa7b5c6ed9a228e5b5a6c01ff9790b13b5da30a
Author: James Ennis <ja...@codethink.co.uk>
AuthorDate: Thu Apr 25 12:59:30 2019 +0100
cascache.py: Make calculate_cache_quota public
---
buildstream/_cas/cascache.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 647ece5..7933fa6 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -59,7 +59,7 @@ class CASCacheUsage():
def __init__(self, casquota):
if not casquota._cache_quota_original:
- casquota._calculate_cache_quota()
+ casquota.calculate_cache_quota()
self.quota_config = casquota._config_cache_quota # Configured quota
self.quota_size = casquota._cache_quota_original # Resolved cache quota in bytes
self.used_size = casquota.get_cache_size() # Size used by artifacts in bytes
@@ -1219,7 +1219,7 @@ class CASQuota:
def full(self):
if not self._cache_quota:
- self._calculate_cache_quota()
+ self.calculate_cache_quota()
if self.get_cache_size() > self._cache_quota:
return True
@@ -1287,14 +1287,14 @@ class CASQuota:
def _get_cache_volume_size(self):
return utils._get_volume_size(self.casdir)
- # _calculate_cache_quota()
+ # calculate_cache_quota()
#
# Calculates and sets the cache quota and lower threshold based on the
# quota set in Context.
# It checks that the quota is both a valid expression, and that there is
# enough disk space to satisfy that quota
#
- def _calculate_cache_quota(self):
+ def calculate_cache_quota(self):
# Headroom intended to give BuildStream a bit of leeway.
# This acts as the minimum size of cache_quota and also
# is taken from the user requested cache_quota.
[buildstream] 01/04: cascache.py: Pretty print the cache quota
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jennis/cache_quota
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 52cddcb34642cccb022d5a2d5195eaf03417d573
Author: James Ennis <ja...@codethink.co.uk>
AuthorDate: Thu Apr 18 15:24:07 2019 +0100
cascache.py: Pretty print the cache quota
---
buildstream/_cas/cascache.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 5f67dc0..8d8f3f0 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -70,7 +70,7 @@ class CASCacheUsage():
def __str__(self):
return "{} / {} ({}%)" \
.format(utils._pretty_size(self.used_size, dec_places=1),
- self.quota_config,
+ utils._pretty_size(self.quota_size, dec_places=1),
self.used_percent)
[buildstream] 04/04: cascache.py: Clearly separate CASCache's
public and local private methods
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch jennis/cache_quota
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7a2560034ad1178f11c09b34d4bb0acccd8ebe51
Author: James Ennis <ja...@codethink.co.uk>
AuthorDate: Thu Apr 25 13:00:50 2019 +0100
cascache.py: Clearly separate CASCache's public and local private methods
---
buildstream/_cas/cascache.py | 116 +++++++++++++++++++++----------------------
1 file changed, 58 insertions(+), 58 deletions(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 7933fa6..499c091 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -1229,64 +1229,6 @@ class CASQuota:
return False
- ################################################
- # Local Private Methods #
- ################################################
-
- # _read_cache_size()
- #
- # Reads and returns the size of the artifact cache that's stored in the
- # cache's size file
- #
- # Returns:
- # (int): The size of the artifact cache, as recorded in the file
- #
- def _read_cache_size(self):
- size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
-
- if not os.path.exists(size_file_path):
- return None
-
- with open(size_file_path, "r") as f:
- size = f.read()
-
- try:
- num_size = int(size)
- except ValueError as e:
- raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
- size, size_file_path)) from e
-
- return num_size
-
- # _write_cache_size()
- #
- # Writes the given size of the artifact to the cache's size file
- #
- # Args:
- # size (int): The size of the artifact cache to record
- #
- def _write_cache_size(self, size):
- assert isinstance(size, int)
- size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
- with utils.save_file_atomic(size_file_path, "w", tempdir=self.cas.tmpdir) as f:
- f.write(str(size))
-
- # _get_cache_volume_size()
- #
- # Get the available space and total space for the volume on
- # which the artifact cache is located.
- #
- # Returns:
- # (int): The total number of bytes on the volume
- # (int): The number of available bytes on the volume
- #
- # NOTE: We use this stub to allow the test cases
- # to override what an artifact cache thinks
- # about it's disk size and available bytes.
- #
- def _get_cache_volume_size(self):
- return utils._get_volume_size(self.casdir)
-
# calculate_cache_quota()
#
# Calculates and sets the cache quota and lower threshold based on the
@@ -1503,6 +1445,64 @@ class CASQuota:
def add_remove_callbacks(self, callback):
self._remove_callbacks.append(callback)
+ ################################################
+ # Local Private Methods #
+ ################################################
+
+ # _read_cache_size()
+ #
+ # Reads and returns the size of the artifact cache that's stored in the
+ # cache's size file
+ #
+ # Returns:
+ # (int): The size of the artifact cache, as recorded in the file
+ #
+ def _read_cache_size(self):
+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
+
+ if not os.path.exists(size_file_path):
+ return None
+
+ with open(size_file_path, "r") as f:
+ size = f.read()
+
+ try:
+ num_size = int(size)
+ except ValueError as e:
+ raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
+ size, size_file_path)) from e
+
+ return num_size
+
+ # _write_cache_size()
+ #
+ # Writes the given size of the artifact to the cache's size file
+ #
+ # Args:
+ # size (int): The size of the artifact cache to record
+ #
+ def _write_cache_size(self, size):
+ assert isinstance(size, int)
+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
+ with utils.save_file_atomic(size_file_path, "w", tempdir=self.cas.tmpdir) as f:
+ f.write(str(size))
+
+ # _get_cache_volume_size()
+ #
+ # Get the available space and total space for the volume on
+ # which the artifact cache is located.
+ #
+ # Returns:
+ # (int): The total number of bytes on the volume
+ # (int): The number of available bytes on the volume
+ #
+ # NOTE: We use this stub to allow the test cases
+ # to override what an artifact cache thinks
+ # about it's disk size and available bytes.
+ #
+ def _get_cache_volume_size(self):
+ return utils._get_volume_size(self.casdir)
+
def _grouper(iterable, n):
while True: