[buildstream] 08/15: Allow cache quota to be set with human friendly input e.g. 2K

[no...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

not-in-ldap pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 5c38fa0ae26283333a9cd23a5a6192f6ccdd8265
Author: James Ennis <ja...@codethink.com>
AuthorDate: Thu Apr 26 16:07:37 2018 +0100

    Allow cache quota to be set with human friendly input e.g. 2K
    
    utils.py: Added _parse_size() function
    pushreceive.py: Altered receive_main() so that it now accepts
    human friendly input.
---
 buildstream/_artifactcache/pushreceive.py | 14 ++++++----
 buildstream/utils.py                      | 44 +++++++++++++++++++++++++++++++
 doc/source/artifacts.rst                  |  6 +++--
 3 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index d276c71..bcd2e4a 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -33,6 +33,7 @@ from urllib.parse import urlparse
 import click
 import gi
 
+from ..utils import _parse_size
 from .. import _signals  # nopep8
 from .._profile import Topics, profile_start, profile_end
 
@@ -573,7 +574,11 @@ class OSTreeReceiver(object):
     def __init__(self, repopath, pull_url, cache_quota):
         self.repopath = repopath
         self.pull_url = pull_url
-        self.cache_quota = cache_quota
+        if cache_quota:
+            # Parse the string and find the quota in bytes (int)
+            self.cache_quota = _parse_size(cache_quota, repopath)
+        else:
+            self.cache_quota = None
 
         if self.repopath is None:
             self.repo = OSTree.Repo.new_default()
@@ -800,8 +805,9 @@ def push(repo, remote, branches, output):
 @click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode")
 @click.option('--pull-url', type=str, required=True,
               help="Clients who try to pull over SSH will be redirected here")
-@click.option('--cache-quota', type=int,
-              help="Implement a quota on the cache (in bytes) here")
+@click.option('--cache-quota', type=str,
+              help="Implement a cache quota. Accepted suffixes are: K, M, G, T "
+              "or no suffix (for bytes).\nExample: --cache-quota 20G for 20 Gb")
 @click.argument('repo')
 def receive_main(verbose, debug, pull_url, cache_quota, repo):
     """A BuildStream sister program for receiving artifacts send to a shared artifact cache
@@ -814,7 +820,5 @@ def receive_main(verbose, debug, pull_url, cache_quota, repo):
     logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
                         level=loglevel, stream=sys.stderr)
 
-    # IDEA: have it so that cache_quota can allow human friendly input.
-
     receiver = OSTreeReceiver(repo, pull_url, cache_quota)
     return receiver.run()
diff --git a/buildstream/utils.py b/buildstream/utils.py
index e6e5989..c1aee35 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -1126,3 +1126,47 @@ def _deduplicate(iterable, key=None):
             if k not in seen:
                 seen_add(k)
                 yield element
+
+
+# _parse_size():
+#
+# Convert a string representing data size to a number of
+# bytes. E.g. "2K" -> 2048.
+#
+# This uses the same format as systemd's
+# [resource-control](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html#).
+#
+# Arguments:
+#     size (str) The string to parse
+#     volume (str) A path on the volume to consider for percentage
+#                  specifications
+#
+# Returns:
+#     (int|None) The number of bytes, or None if 'infinity' was specified.
+#
+# Raises:
+#     UtilError if the string is not a valid data size.
+#
+def _parse_size(size, volume):
+    if size == 'infinity':
+        return None
+
+    matches = re.fullmatch(r'([0-9]\.?[0-9]*)([KMGT%]?)', size)
+    if matches is None:
+        raise UtilError("{} is not a valid data size."
+                        "\nOptions are: K, M, G, T and %.".format(size))
+
+    num, unit = matches.groups()
+
+    if unit == '%':
+        num = float(num)
+        if num > 100:
+            raise UtilError("{}% is not a valid percentage value.".format(num))
+
+        stat_ = os.statvfs(volume)
+        disk_size = stat_.f_blocks * stat_.f_bsize
+
+        return disk_size * (num / 100)
+
+    units = ('', 'K', 'M', 'G', 'T')
+    return int(num) * 1024**units.index(unit)
diff --git a/doc/source/artifacts.rst b/doc/source/artifacts.rst
index f181c67..23a389f 100644
--- a/doc/source/artifacts.rst
+++ b/doc/source/artifacts.rst
@@ -160,8 +160,10 @@ For this you will want something like the following in your ``/etc/ssh/sshd_conf
 	# command must be specified here; 'artifacts' is
 	# the HOME relative path to the artifact cache.
 	# The exact pull URL must also be specified.
-	# A cache-quota is optional and should be specified in bytes
-	## FIXME: The above should have a function which converts 20G -> bytes...
+	# A cache-quota is optional and can either be specified
+	# by the exact integer, or in human friendly form,
+	# for example, a 20 GB quota is specified with "20G"
+
         ForceCommand bst-artifact-receive --pull-url https://example.com/artifacts --cache-quota quota_in_bytes --verbose artifacts