You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:11:58 UTC

[buildstream] branch jennis/136-clean-remote-cache created (now f6cab64)

This is an automated email from the ASF dual-hosted git repository.

tvb pushed a change to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git.


      at f6cab64  _ostree.py: Remove unused __list_remote_refs function

This branch includes the following new commits:

     new 49d40f8  _ostree.py: Reintroduce remove()
     new 8eb7d6a  utils.py: Add get_dir_size helper
     new bbc38f9  utils.py: Optimise get_scandir_dir_size()
     new b031c1e  utils.py: Optimise get_walk_dir_size()
     new 82c7848  _ostree.py: Introduce list_all_refs() and list_artifacts()
     new ac22a29  pushreceive.py: Ensure there is a repopath
     new f990f88  Allow the user to set a cache quota
     new 5c38fa0  Allow cache quota to be set with human friendly input e.g. 2K
     new 361efba  pushreceive.py: Remove LRP artifacts from cache, introduce clean_up_cache()
     new 4d3676c  element_generators.py: Add a utility function to create arbitrary size elements
     new 7c194fe  push.py: Add tests to test remote cache expiry
     new c33fcc2  pushreceive.py: Add comments to code and name variables more sensibly
     new 59d709d  pushreceive.py: Ensure too large artifacts are not pushed
     new 05e2395  _ostree.py: Adhere to private symbol policy
     new f6cab64  _ostree.py: Remove unused __list_remote_refs function

The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[buildstream] 08/15: Allow cache quota to be set with human friendly input e.g. 2K

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 5c38fa0ae26283333a9cd23a5a6192f6ccdd8265
Author: James Ennis <ja...@codethink.com>
AuthorDate: Thu Apr 26 16:07:37 2018 +0100

    Allow cache quota to be set with human friendly input e.g. 2K
    
    utils.py: Added _parse_size() function
    pushreceive.py: Altered receive_main() so that it now accepts
    human friendly input.
---
 buildstream/_artifactcache/pushreceive.py | 14 ++++++----
 buildstream/utils.py                      | 44 +++++++++++++++++++++++++++++++
 doc/source/artifacts.rst                  |  6 +++--
 3 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index d276c71..bcd2e4a 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -33,6 +33,7 @@ from urllib.parse import urlparse
 import click
 import gi
 
+from ..utils import _parse_size
 from .. import _signals  # nopep8
 from .._profile import Topics, profile_start, profile_end
 
@@ -573,7 +574,11 @@ class OSTreeReceiver(object):
     def __init__(self, repopath, pull_url, cache_quota):
         self.repopath = repopath
         self.pull_url = pull_url
-        self.cache_quota = cache_quota
+        if cache_quota:
+            # Parse the string and find the quota in bytes (int)
+            self.cache_quota = _parse_size(cache_quota, repopath)
+        else:
+            self.cache_quota = None
 
         if self.repopath is None:
             self.repo = OSTree.Repo.new_default()
@@ -800,8 +805,9 @@ def push(repo, remote, branches, output):
 @click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode")
 @click.option('--pull-url', type=str, required=True,
               help="Clients who try to pull over SSH will be redirected here")
-@click.option('--cache-quota', type=int,
-              help="Implement a quota on the cache (in bytes) here")
+@click.option('--cache-quota', type=str,
+              help="Implement a cache quota. Accepted suffixes are: K, M, G, T "
+              "or no suffix (for bytes).\nExample: --cache-quota 20G for 20 Gb")
 @click.argument('repo')
 def receive_main(verbose, debug, pull_url, cache_quota, repo):
     """A BuildStream sister program for receiving artifacts send to a shared artifact cache
@@ -814,7 +820,5 @@ def receive_main(verbose, debug, pull_url, cache_quota, repo):
     logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
                         level=loglevel, stream=sys.stderr)
 
-    # IDEA: have it so that cache_quota can allow human friendly input.
-
     receiver = OSTreeReceiver(repo, pull_url, cache_quota)
     return receiver.run()
diff --git a/buildstream/utils.py b/buildstream/utils.py
index e6e5989..c1aee35 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -1126,3 +1126,47 @@ def _deduplicate(iterable, key=None):
             if k not in seen:
                 seen_add(k)
                 yield element
+
+
+# _parse_size():
+#
+# Convert a string representing data size to a number of
+# bytes. E.g. "2K" -> 2048.
+#
+# This uses the same format as systemd's
+# [resource-control](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html#).
+#
+# Arguments:
+#     size (str) The string to parse
+#     volume (str) A path on the volume to consider for percentage
+#                  specifications
+#
+# Returns:
+#     (int|None) The number of bytes, or None if 'infinity' was specified.
+#
+# Raises:
+#     UtilError if the string is not a valid data size.
+#
+def _parse_size(size, volume):
+    if size == 'infinity':
+        return None
+
+    matches = re.fullmatch(r'([0-9]\.?[0-9]*)([KMGT%]?)', size)
+    if matches is None:
+        raise UtilError("{} is not a valid data size."
+                        "\nOptions are: K, M, G, T and %.".format(size))
+
+    num, unit = matches.groups()
+
+    if unit == '%':
+        num = float(num)
+        if num > 100:
+            raise UtilError("{}% is not a valid percentage value.".format(num))
+
+        stat_ = os.statvfs(volume)
+        disk_size = stat_.f_blocks * stat_.f_bsize
+
+        return disk_size * (num / 100)
+
+    units = ('', 'K', 'M', 'G', 'T')
+    return int(num) * 1024**units.index(unit)
diff --git a/doc/source/artifacts.rst b/doc/source/artifacts.rst
index f181c67..23a389f 100644
--- a/doc/source/artifacts.rst
+++ b/doc/source/artifacts.rst
@@ -160,8 +160,10 @@ For this you will want something like the following in your ``/etc/ssh/sshd_conf
 	# command must be specified here; 'artifacts' is
 	# the HOME relative path to the artifact cache.
 	# The exact pull URL must also be specified.
-	# A cache-quota is optional and should be specified in bytes
-	## FIXME: The above should have a function which converts 20G -> bytes...
+	# A cache-quota is optional and can either be specified
+	# by the exact integer, or in human friendly form,
+	# for example, a 20 GB quota is specified with "20G"
+
         ForceCommand bst-artifact-receive --pull-url https://example.com/artifacts --cache-quota quota_in_bytes --verbose artifacts
 
 


[buildstream] 15/15: _ostree.py: Remove unused __list_remote_refs function

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit f6cab6410e52f6e18a9e72bc4cf16c7d9185ac2b
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 10:41:15 2018 +0100

    _ostree.py: Remove unused __list_remote_refs function
---
 buildstream/_ostree.py | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index 93fad13..72685a8 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -596,23 +596,3 @@ def __list_all_refs(repo):
         return refs
     except GLib.GError as e:
         raise OSTreeError(message=e.message) from e
-
-
-# __list_remote_refs():
-#
-# Fetch list of refs from a remote.
-#
-# Args:
-#    repo (OSTree.Repo): The repo
-#    remote (str): An optional remote name, defaults to 'origin'
-#
-# Returns:
-#    (dict): A dict of refs to checksums.
-#
-def __list_remote_refs(repo, remote="origin"):
-    try:
-        _, refs = repo.remote_list_refs(remote)
-        return refs
-    except GLib.GError as e:
-        (_, remote_url) = repo.remote_get_url(remote)
-        raise OSTreeError(message="{} when attempting to fetch from {}".format(e.message, remote_url)) from e


[buildstream] 03/15: utils.py: Optimise get_scandir_dir_size()

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit bbc38f92fab223a7442829871351c554d835f36e
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 16:35:50 2018 +0100

    utils.py: Optimise get_scandir_dir_size()
---
 buildstream/utils.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/buildstream/utils.py b/buildstream/utils.py
index 60761dc..f22b57f 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -541,7 +541,7 @@ def save_file_atomic(filename, mode='w', *, buffering=-1, encoding=None,
 #     (str) The path whose size to check.
 #
 # Returns:
-#     (int) The size on disk.
+#     (int) The size on disk in bytes.
 #
 def _get_dir_size(path):
     scandir = None
@@ -564,11 +564,10 @@ def _get_dir_size(path):
         total = 0
 
         for f in scandir(path):
+            total += f.stat(follow_symlinks=False).st_size
+
             if f.is_dir(follow_symlinks=False):
-                total += f.stat(follow_symlinks=False).st_size
                 total += get_scandir_dir_size(f.path)
-            elif f.is_file(follow_symlinks=False):
-                total += f.stat(follow_symlinks=False).st_size
 
         return total
 


[buildstream] 04/15: utils.py: Optimise get_walk_dir_size()

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit b031c1eaf2acadaa326eb26a30aba2286b3a293e
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 16:36:54 2018 +0100

    utils.py: Optimise get_walk_dir_size()
---
 buildstream/utils.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/buildstream/utils.py b/buildstream/utils.py
index f22b57f..e6e5989 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -578,13 +578,17 @@ def _get_dir_size(path):
         for dirpath, dirs, files in os.walk(path):
             for d in dirs:
                 path = os.path.join(dirpath, d)
-                if os.path.exists(path):
-                    total += os.stat(path, follow_symlinks=False).st_size
+                try:
+                    total += os.lstat(path).st_size
+                except FileNotFoundError:
+                    pass
 
             for f in files:
                 path = os.path.join(dirpath, f)
-                if os.path.exists(path):
-                    total += os.stat(path, follow_symlinks=False).st_size
+                try:
+                    total += os.lstat(path).st_size
+                except FileNotFoundError:
+                    pass
 
         return total
 


[buildstream] 07/15: Allow the user to set a cache quota

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit f990f8888e06fea64ae5fea9f491046550a00d3c
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 18 15:51:32 2018 +0100

    Allow the user to set a cache quota
---
 buildstream/_artifactcache/pushreceive.py | 11 ++++++++---
 doc/source/artifacts.rst                  |  4 +++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index 75a65e1..d276c71 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -570,9 +570,10 @@ class OSTreePusher(object):
 #     pull_url (str): Redirection for clients who want to pull, not push.
 #
 class OSTreeReceiver(object):
-    def __init__(self, repopath, pull_url):
+    def __init__(self, repopath, pull_url, cache_quota):
         self.repopath = repopath
         self.pull_url = pull_url
+        self.cache_quota = cache_quota
 
         if self.repopath is None:
             self.repo = OSTree.Repo.new_default()
@@ -799,8 +800,10 @@ def push(repo, remote, branches, output):
 @click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode")
 @click.option('--pull-url', type=str, required=True,
               help="Clients who try to pull over SSH will be redirected here")
+@click.option('--cache-quota', type=int,
+              help="Implement a quota on the cache (in bytes) here")
 @click.argument('repo')
-def receive_main(verbose, debug, pull_url, repo):
+def receive_main(verbose, debug, pull_url, cache_quota, repo):
     """A BuildStream sister program for receiving artifacts send to a shared artifact cache
     """
     loglevel = logging.WARNING
@@ -811,5 +814,7 @@ def receive_main(verbose, debug, pull_url, repo):
     logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
                         level=loglevel, stream=sys.stderr)
 
-    receiver = OSTreeReceiver(repo, pull_url)
+    # IDEA: have it so that cache_quota can allow human friendly input.
+
+    receiver = OSTreeReceiver(repo, pull_url, cache_quota)
     return receiver.run()
diff --git a/doc/source/artifacts.rst b/doc/source/artifacts.rst
index e0808e8..f181c67 100644
--- a/doc/source/artifacts.rst
+++ b/doc/source/artifacts.rst
@@ -160,7 +160,9 @@ For this you will want something like the following in your ``/etc/ssh/sshd_conf
 	# command must be specified here; 'artifacts' is
 	# the HOME relative path to the artifact cache.
 	# The exact pull URL must also be specified.
-        ForceCommand bst-artifact-receive --pull-url https://example.com/artifacts --verbose artifacts
+	# A cache-quota is optional and should be specified in bytes
+	## FIXME: The above should have a function which converts 20G -> bytes...
+        ForceCommand bst-artifact-receive --pull-url https://example.com/artifacts --cache-quota quota_in_bytes --verbose artifacts
 
 
 User configuration


[buildstream] 09/15: pushreceive.py: Remove LRP artifacts from cache, introduce clean_up_cache()

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 361efbac89926950ebc9ae5f59944b876ca4b868
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 18 15:01:37 2018 +0100

    pushreceive.py: Remove LRP artifacts from cache, introduce clean_up_cache()
---
 buildstream/_artifactcache/pushreceive.py | 57 +++++++++++++++++++++++++++++--
 1 file changed, 54 insertions(+), 3 deletions(-)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index bcd2e4a..c60011d 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -33,7 +33,8 @@ from urllib.parse import urlparse
 import click
 import gi
 
-from ..utils import _parse_size
+from ..utils import _get_dir_size, _parse_size
+from .. import _ostree
 from .. import _signals  # nopep8
 from .._profile import Topics, profile_start, profile_end
 
@@ -278,7 +279,7 @@ class PushMessageReader(object):
         _, args = self.receive([PushCommandType.update])
         return args
 
-    def receive_putobjects(self, repo):
+    def receive_putobjects(self, repo, repo_size, cache_quota):
 
         received_objects = []
 
@@ -289,6 +290,7 @@ class PushMessageReader(object):
         #
         # This should block while tar.next() reads the next
         # tar object from the stream.
+        current_repo_size = repo_size
         while True:
             filepos = tar.fileobj.tell()
             tar_info = tar.next()
@@ -301,6 +303,11 @@ class PushMessageReader(object):
                     tar.fileobj.read(512)
                 break
 
+            # obtain size of tar object in bytes
+            artifact_size = tar_info.size
+            if cache_quota and artifact_size + current_repo_size > cache_quota:
+                current_repo_size = clean_up_cache(repo, current_repo_size, artifact_size, cache_quota)
+
             tar.extract(tar_info, self.tmpdir)
             received_objects.append(tar_info.name)
 
@@ -650,8 +657,16 @@ class OSTreeReceiver(object):
             logging.debug('Received done before any objects, exiting')
             return 0
 
+        # Determine the size (in bytes) of the artifact cache
+        # Expensive function, we should probably only work this out if there is a quota,
+        # otherwise we don't need it.
+        if self.cache_quota:
+            repo_size = _get_dir_size(self.repopath)
+        else:
+            repo_size = None
+
         # Receive the actual objects
-        received_objects = self.reader.receive_putobjects(self.repo)
+        received_objects = self.reader.receive_putobjects(self.repo, repo_size, self.cache_quota)
 
         # Ensure that pusher has sent all objects
         self.reader.receive_done()
@@ -800,6 +815,42 @@ def push(repo, remote, branches, output):
             return False
 
 
+# clean_up_cache()
+#
+# Keep removing Least Recently Pushed (LRP) artifacts in a cache until there
+# is enough space for the incoming artifact
+#
+# Args:
+#   repo: OSTree.Repo object
+#   repo_size: The size of the repo in bytes
+#   artifact_size: The size of the artifact in bytes
+#   cache_quota: quota of the cache in bytes
+#
+# Returns:
+#   int: The new repo_size
+#
+def clean_up_cache(repo, repo_size, artifact_size, cache_quota):
+    # obtain a list of LRP artifacts
+    LRP_artifacts = _ostree.list_artifacts(repo)
+
+    removed_size = 0  # in bytes
+    while artifact_size + repo_size - removed_size > cache_quota:
+        try:
+            to_remove = LRP_artifacts.pop(0)  # The first element in the list is the LRP artifact
+        except IndexError:
+            logging.info("There are no more artifacts left in the cache. Adding artifact...")
+            break
+
+        removed_size += _ostree.remove(repo, to_remove, defer_prune=False)
+
+    if removed_size > 0:
+        logging.info("Successfully removed {} bytes from the cache".format(removed_size))
+    else:
+        logging.info("No artifacts were removed from the cache.")
+
+    return artifact_size + repo_size - removed_size
+
+
 @click.command(short_help="Receive pushed artifacts over ssh")
 @click.option('--verbose', '-v', is_flag=True, default=False, help="Verbose mode")
 @click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode")


[buildstream] 10/15: element_generators.py: Add a utility function to create arbitrary size elements

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 4d3676c3062125a459e2d36efe10fb3c98bbac7a
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue Apr 24 16:38:02 2018 +0100

    element_generators.py: Add a utility function to create arbitrary size elements
    
    New file consisting of functions that allow us to generate elements
    on the fly
    __init__.py now includes create_element_size()
---
 tests/testutils/__init__.py           |  1 +
 tests/testutils/element_generators.py | 40 +++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index 7e5b792..93143b5 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -1,3 +1,4 @@
 from .runcli import cli, cli_integration
 from .repo import create_repo, ALL_REPO_KINDS
 from .artifactshare import create_artifact_share
+from .element_generators import create_element_size
diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py
new file mode 100644
index 0000000..3f6090d
--- /dev/null
+++ b/tests/testutils/element_generators.py
@@ -0,0 +1,40 @@
+import os
+
+from buildstream import _yaml
+
+
+# create_element_size()
+#
+# This will open a "<name>_data" file for writing and write
+# <size> MB of urandom (/dev/urandom) "stuff" into the file.
+# A bst import element file is then created: <name>.bst
+#
+# Args:
+#  name: (str) of the element name (e.g. target.bst)
+#  path: (str) pathway to the project/elements directory
+#  dependencies: A list of strings (can also be an empty list)
+#  size: (int) size of the element in bytes
+#
+# Returns:
+#  Nothing (creates a .bst file of specified size)
+#
+def create_element_size(name, path, dependencies, size):
+    os.makedirs(path, exist_ok=True)
+
+    # Create a file to be included in this element's artifact
+    with open(os.path.join(path, name + '_data'), 'wb+') as f:
+        f.write(os.urandom(size))
+
+    # Simplest case: We want this file (of specified size) to just
+    # be an import element.
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'local',
+                'path': os.path.join(path, name + '_data')
+            }
+        ],
+        'depends': dependencies
+    }
+    _yaml.dump(element, os.path.join(path, name))


[buildstream] 13/15: pushreceive.py: Ensure too large artifacts are not pushed

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 59d709d2196f2cdf8df847ebc51133b5134970bf
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 25 17:04:42 2018 +0100

    pushreceive.py: Ensure too large artifacts are not pushed
---
 buildstream/_artifactcache/pushreceive.py | 40 ++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index a8ba742..e13fd75 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -36,6 +36,7 @@ import gi
 from ..utils import _get_dir_size, _parse_size
 from .. import _ostree
 from .. import _signals  # nopep8
+from .._exceptions import ArtifactError
 from .._profile import Topics, profile_start, profile_end
 
 gi.require_version('OSTree', '1.0')
@@ -57,6 +58,11 @@ class PushExistsException(Exception):
     pass
 
 
+# Trying to push an artifact that is too big for the cache
+class ArtifactTooBigError(Exception):
+    pass
+
+
 class PushCommandType(Enum):
     info = 0
     update = 1
@@ -286,7 +292,6 @@ class PushMessageReader(object):
         return args
 
     def receive_putobjects(self, repo, repo_size, cache_quota):
-
         received_objects = []
 
         # Open a TarFile for reading uncompressed tar from a stream
@@ -311,11 +316,18 @@ class PushMessageReader(object):
 
             # obtain size of tar object in bytes
             artifact_size = tar_info.size
-            if cache_quota and artifact_size + current_repo_size > cache_quota:
+            if cache_quota and artifact_size < cache_quota and artifact_size + current_repo_size > cache_quota:
                 current_repo_size = clean_up_cache(repo, current_repo_size, artifact_size, cache_quota)
 
-            tar.extract(tar_info, self.tmpdir)
-            received_objects.append(tar_info.name)
+            if cache_quota and artifact_size > cache_quota:
+                # Tell the user we've done nothing with this artifact
+                # This could potentially be from accidentially setting a small cache quota...
+                raise ArtifactTooBigError("Artifact of size: {} bytes is too big for the remote cache"
+                                          " under the current quota of: {} bytes."
+                                          .format(artifact_size, cache_quota))
+            else:
+                tar.extract(tar_info, self.tmpdir)
+                received_objects.append(tar_info.name)
 
         # Finished with this stream
         tar.close()
@@ -632,8 +644,10 @@ class OSTreeReceiver(object):
     def run(self):
         try:
             exit_code = self.do_run()
-            self.close()
-            return exit_code
+        except ArtifactTooBigError:
+            logging.warning("The artifact was too big for the cache under the current quota,"
+                            " so it has not been received")
+            exit_code = 0
         except:
             # BLIND EXCEPT - Just abort if we receive any exception, this
             # can be a broken pipe, a tarfile read error when the remote
@@ -641,6 +655,9 @@ class OSTreeReceiver(object):
             self.close()
             raise
 
+        self.close()
+        return exit_code
+
     def do_run(self):
         # Receive remote info
         args = self.reader.receive_info()
@@ -822,7 +839,13 @@ def push(repo, remote, branches, output):
             return True
         except ConnectionError as e:
             # Connection attempt failed or connection was terminated unexpectedly
-            terminate_push()
+            try:
+                terminate_push()
+            except BrokenPipeError:
+                # If the artifact is too big for the cache, we force the tar stream to quit
+                # So let's just log this and return push as False
+                logging.info("Ref {} is too big for the cache under the current quota".format(branches))
+                return False
             raise PushException("Connection failed") from e
         except PushException:
             terminate_push()
@@ -858,8 +881,7 @@ def clean_up_cache(repo, repo_size, artifact_size, cache_quota):
         try:
             to_remove = LRP_artifacts.pop(0)  # The first element in the list is the LRP artifact
         except IndexError:
-            logging.info("There are no more artifacts left in the cache. Adding artifact...")
-            break
+            raise ArtifactError("There are no more artifacts in the cache.")
 
         removed_size += _ostree.remove(repo, to_remove, defer_prune=False)
 


[buildstream] 14/15: _ostree.py: Adhere to private symbol policy

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 05e239572782aa9286e5c41df0adc38867849973
Author: James Ennis <ja...@codethink.com>
AuthorDate: Tue May 22 10:19:43 2018 +0100

    _ostree.py: Adhere to private symbol policy
---
 buildstream/_ostree.py | 60 +++++++++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index 080447b..93fad13 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -554,7 +554,33 @@ def configure_remote(repo, remote, url, key_url=None):
             raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e
 
 
-# list_all_refs():
+# list_artifacts():
+#
+# List cached artifacts in Least Recently Modified (LRM) order.
+#
+# Returns:
+#     (list) - A list of refs in LRM order
+#
+def list_artifacts(repo):
+    # string of: /path/to/repo/refs/heads
+    ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')
+
+    # obtain list of <project>/<element>/<key>
+    refs = __list_all_refs(repo).keys()
+
+    mtimes = []
+    for ref in refs:
+        ref_path = os.path.join(ref_heads, ref)
+        if os.path.exists(ref_path):
+            # Obtain the mtime (the time a file was last modified)
+            mtimes.append(os.path.getmtime(ref_path))
+
+    # NOTE: Sorted will sort from earliest to latest, thus the
+    # first element of this list will be the file modified earliest.
+    return [ref for _, ref in sorted(zip(mtimes, refs))]
+
+
+# __list_all_refs():
 #
 # Create a list of all refs.
 #
@@ -564,7 +590,7 @@ def configure_remote(repo, remote, url, key_url=None):
 # Returns:
 #    (dict): A dict of refs to checksums.
 #
-def list_all_refs(repo):
+def __list_all_refs(repo):
     try:
         _, refs = repo.list_refs(None)
         return refs
@@ -572,7 +598,7 @@ def list_all_refs(repo):
         raise OSTreeError(message=e.message) from e
 
 
-# list_remote_refs():
+# __list_remote_refs():
 #
 # Fetch list of refs from a remote.
 #
@@ -583,36 +609,10 @@ def list_all_refs(repo):
 # Returns:
 #    (dict): A dict of refs to checksums.
 #
-def list_remote_refs(repo, remote="origin"):
+def __list_remote_refs(repo, remote="origin"):
     try:
         _, refs = repo.remote_list_refs(remote)
         return refs
     except GLib.GError as e:
         (_, remote_url) = repo.remote_get_url(remote)
         raise OSTreeError(message="{} when attempting to fetch from {}".format(e.message, remote_url)) from e
-
-
-# list_artifacts():
-#
-# List cached artifacts in Least Recently Modified (LRM) order.
-#
-# Returns:
-#     (list) - A list of refs in LRM order
-#
-def list_artifacts(repo):
-    # string of: /path/to/repo/refs/heads
-    ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')
-
-    # obtain list of <project>/<element>/<key>
-    refs = list_all_refs(repo).keys()
-
-    mtimes = []
-    for ref in refs:
-        ref_path = os.path.join(ref_heads, ref)
-        if os.path.exists(ref_path):
-            # Obtain the mtime (the time a file was last modified)
-            mtimes.append(os.path.getmtime(ref_path))
-
-    # NOTE: Sorted will sort from earliest to latest, thus the
-    # first element of this list will be the file modified earliest.
-    return [ref for _, ref in sorted(zip(mtimes, refs))]


[buildstream] 05/15: _ostree.py: Introduce list_all_refs() and list_artifacts()

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 82c7848a41218e51765d7cf0aa379d5ae3b2194f
Author: Tristan Maat <tm...@tlater.net>
AuthorDate: Sat Mar 24 17:56:56 2018 +0000

    _ostree.py: Introduce list_all_refs() and list_artifacts()
---
 buildstream/_ostree.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index 246b54f..080447b 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -552,3 +552,67 @@ def configure_remote(repo, remote, url, key_url=None):
             repo.remote_gpg_import(remote, stream, None, 0, None)
         except GLib.GError as e:
             raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e
+
+
+# list_all_refs():
+#
+# Create a list of all refs.
+#
+# Args:
+#    repo (OSTree.Repo): The repo
+#
+# Returns:
+#    (dict): A dict of refs to checksums.
+#
+def list_all_refs(repo):
+    try:
+        _, refs = repo.list_refs(None)
+        return refs
+    except GLib.GError as e:
+        raise OSTreeError(message=e.message) from e
+
+
+# list_remote_refs():
+#
+# Fetch list of refs from a remote.
+#
+# Args:
+#    repo (OSTree.Repo): The repo
+#    remote (str): An optional remote name, defaults to 'origin'
+#
+# Returns:
+#    (dict): A dict of refs to checksums.
+#
+def list_remote_refs(repo, remote="origin"):
+    try:
+        _, refs = repo.remote_list_refs(remote)
+        return refs
+    except GLib.GError as e:
+        (_, remote_url) = repo.remote_get_url(remote)
+        raise OSTreeError(message="{} when attempting to fetch from {}".format(e.message, remote_url)) from e
+
+
+# list_artifacts():
+#
+# List cached artifacts in Least Recently Modified (LRM) order.
+#
+# Returns:
+#     (list) - A list of refs in LRM order
+#
+def list_artifacts(repo):
+    # string of: /path/to/repo/refs/heads
+    ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')
+
+    # obtain list of <project>/<element>/<key>
+    refs = list_all_refs(repo).keys()
+
+    mtimes = []
+    for ref in refs:
+        ref_path = os.path.join(ref_heads, ref)
+        if os.path.exists(ref_path):
+            # Obtain the mtime (the time a file was last modified)
+            mtimes.append(os.path.getmtime(ref_path))
+
+    # NOTE: Sorted will sort from earliest to latest, thus the
+    # first element of this list will be the file modified earliest.
+    return [ref for _, ref in sorted(zip(mtimes, refs))]


[buildstream] 01/15: _ostree.py: Reintroduce remove()

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 49d40f896536c2263925e8c52b3a1db2b3e56509
Author: Tristan Maat <tm...@tlater.net>
AuthorDate: Fri Mar 23 17:26:32 2018 +0000

    _ostree.py: Reintroduce remove()
---
 buildstream/_ostree.py | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py
index dfa7567..246b54f 100644
--- a/buildstream/_ostree.py
+++ b/buildstream/_ostree.py
@@ -225,6 +225,42 @@ def exists(repo, ref):
     return has_object
 
 
+# remove():
+#
+# Removes the given commit or symbolic ref from the repo.
+#
+# Args:
+#    repo (OSTree.Repo): The repo
+#    ref (str): A commit checksum or symbolic ref
+#    defer_prune (bool): Whether to defer pruning to the caller. NOTE:
+#                        The space won't be freed until you manually
+#                        call repo.prune.
+#
+# Returns:
+#    (int|None) The amount of space pruned from the repository in
+#               Bytes, or None if defer_prune is True
+#
+def remove(repo, ref, *, defer_prune=False):
+
+    # Get the commit checksum, this will:
+    #
+    #  o Return a commit checksum if ref is a symbolic branch
+    #  o Return the same commit checksum if ref is a valid commit checksum
+    #  o Return None if the ostree repo doesnt know this ref.
+    #
+    check = checksum(repo, ref)
+    if check is None:
+        raise OSTreeError("Could not find artifact for ref '{}'".format(ref))
+
+    repo.set_ref_immediate(None, ref, None)
+
+    if not defer_prune:
+        _, _, _, pruned = repo.prune(OSTree.RepoPruneFlags.REFS_ONLY, -1)
+        return pruned
+
+    return None
+
+
 # checksum():
 #
 # Returns the commit checksum for a given symbolic ref,


[buildstream] 02/15: utils.py: Add get_dir_size helper

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 8eb7d6aa121c5009208daa1675963b66d42206d0
Author: Tristan Maat <tr...@codethink.co.uk>
AuthorDate: Tue Mar 20 13:19:13 2018 +0000

    utils.py: Add get_dir_size helper
---
 buildstream/utils.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)

diff --git a/buildstream/utils.py b/buildstream/utils.py
index 8e72196..60761dc 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -533,6 +533,70 @@ def save_file_atomic(filename, mode='w', *, buffering=-1, encoding=None,
         raise
 
 
+# _get_dir_size():
+#
+# Get the disk usage of a given directory in bytes.
+#
+# Arguments:
+#     (str) The path whose size to check.
+#
+# Returns:
+#     (int) The size on disk.
+#
+def _get_dir_size(path):
+    scandir = None
+    artifact_size = 0
+
+    # python3.4 does not have scandir, but it is significantly
+    # faster for this type of work.
+    if hasattr(os, 'scandir'):
+        from os import scandir               # pylint: disable=no-name-in-module
+    else:
+        # If our user has scandir for python<3.5 installed, we use
+        # that, because why not?
+        try:
+            from scandir import scandir
+        except ImportError:
+            pass
+
+    def get_scandir_dir_size(path):
+        path = os.path.abspath(path)
+        total = 0
+
+        for f in scandir(path):
+            if f.is_dir(follow_symlinks=False):
+                total += f.stat(follow_symlinks=False).st_size
+                total += get_scandir_dir_size(f.path)
+            elif f.is_file(follow_symlinks=False):
+                total += f.stat(follow_symlinks=False).st_size
+
+        return total
+
+    def get_walk_dir_size(path):
+        path = os.path.abspath(path)
+        total = 0
+
+        for dirpath, dirs, files in os.walk(path):
+            for d in dirs:
+                path = os.path.join(dirpath, d)
+                if os.path.exists(path):
+                    total += os.stat(path, follow_symlinks=False).st_size
+
+            for f in files:
+                path = os.path.join(dirpath, f)
+                if os.path.exists(path):
+                    total += os.stat(path, follow_symlinks=False).st_size
+
+        return total
+
+    if scandir:
+        artifact_size = get_scandir_dir_size(path)
+    else:
+        artifact_size = get_walk_dir_size(path)
+
+    return artifact_size
+
+
 # A sentinel to be used as a default argument for functions that need
 # to distinguish between a kwarg set to None and an unset kwarg.
 _sentinel = object()


[buildstream] 06/15: pushreceive.py: Ensure there is a repopath

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit ac22a29e609da6262453f455777ea422769a647f
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 18 12:42:52 2018 +0100

    pushreceive.py: Ensure there is a repopath
---
 buildstream/_artifactcache/pushreceive.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index 777065e..75a65e1 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -576,6 +576,9 @@ class OSTreeReceiver(object):
 
         if self.repopath is None:
             self.repo = OSTree.Repo.new_default()
+            self.repopath = self.repo.get_path().get_path()
+            # NOTE: OSTree.Repo.get_path() returns Gio.File
+            # Gio.File.get_path() returns a string of the pathway
         else:
             self.repo = OSTree.Repo.new(Gio.File.new_for_path(self.repopath))
         self.repo.open(None)


[buildstream] 12/15: pushreceive.py: Add comments to code and name variables more sensibly

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit c33fcc2bd0dbb1a6b7a3c3af09aece95b6096943
Author: James Ennis <ja...@codethink.com>
AuthorDate: Fri Apr 20 17:20:35 2018 +0100

    pushreceive.py: Add comments to code and name variables more sensibly
---
 buildstream/_artifactcache/pushreceive.py | 38 +++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index 57a4b4e..a8ba742 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -114,7 +114,7 @@ class PushMessageWriter(object):
     def __init__(self, file, byteorder=sys.byteorder):
         self.file = file
         self.byteorder = byteorder
-        self.msg_byteorder = python_to_msg_byteorder(self.byteorder)
+        self.msg_byteorder = python_to_msg_byteorder(self.byteorder)  # 'l' or 'B'
 
     def encode_header(self, cmdtype, size):
         header = self.msg_byteorder.encode() + \
@@ -177,6 +177,7 @@ class PushMessageWriter(object):
     def send_update(self, refs):
         cmdtype = PushCommandType.update
         args = {}
+        # Note: Here, revs will be a tuple of (remote_rev, local_rev)
         for branch, revs in refs.items():
             args[branch] = GLib.Variant('(ss)', revs)
         command = PushCommand(cmdtype, args)
@@ -193,14 +194,19 @@ class PushMessageWriter(object):
 
         # Open a TarFile for writing uncompressed tar to a stream
         tar = tarfile.TarFile.open(mode='w|', fileobj=self.file)
+        # Note: objects is a set containing: <commit>.GLib.Enum
+        # for each object
         for obj in objects:
 
             logging.info('Sending object {}'.format(obj))
+            # obtain the path of the object
+            # Example: path/to/ostree/cache/objects/4a/COMMIT.GlibEnum
             objpath = ostree_object_path(repo, obj)
             stat = os.stat(objpath)
 
             tar_info = tarfile.TarInfo(obj)
             tar_info.mtime = stat.st_mtime
+            # obtain the size of the file in bytes
             tar_info.size = stat.st_size
             with open(objpath, 'rb') as obj_fp:
                 tar.addfile(tar_info, obj_fp)
@@ -381,6 +387,7 @@ def foo_run(func, args, stdin_fd, stdout_fd, stderr_fd):
 
 class ProcessWithPipes(object):
     def __init__(self, func, args, *, stderr=None):
+        # Create a pipe and return a pair of file descriptors (r, w)
         r0, w0 = os.pipe()
         r1, w1 = os.pipe()
         if stderr is None:
@@ -425,10 +432,12 @@ class OSTreePusher(object):
 
         # Enumerate branches to push
         if branches is None:
+            # obtain a dict of 'refs': 'checksums'
             _, self.refs = self.repo.list_refs(None, None)
         else:
             self.refs = {}
             for branch in branches:
+                # branch is a ref, now find its checksum (i.e. rev)
                 _, rev = self.repo.resolve_rev(branch, False)
                 self.refs[branch] = rev
 
@@ -447,6 +456,9 @@ class OSTreePusher(object):
         logging.info('Executing {}'.format(' '.join(ssh_cmd)))
 
         if self.remote_host:
+            # subprocess.Popen(args, bufsize=-1,...)
+            # Executes a child program in a new process which returns an open file
+            # object connected to the pipe.
             self.ssh = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE,
                                         stdout=subprocess.PIPE,
                                         stderr=self.output,
@@ -457,31 +469,33 @@ class OSTreePusher(object):
         self.writer = PushMessageWriter(self.ssh.stdin)
         self.reader = PushMessageReader(self.ssh.stdout)
 
-    def needed_commits(self, remote, local, needed):
-        parent = local
-        if remote == '0' * 64:
+    def needed_commits(self, remote_rev, local_rev, needed):
+        parent = local_rev
+        if remote_rev == '0' * 64:
             # Nonexistent remote branch, use None for convenience
-            remote = None
-        while parent != remote:
+            remote_rev = None
+        while parent != remote_rev:
             needed.add(parent)
             _, commit = self.repo.load_variant_if_exists(OSTree.ObjectType.COMMIT,
                                                          parent)
             if commit is None:
                 raise PushException('Shallow history from commit {} does '
-                                    'not contain remote commit {}'.format(local, remote))
+                                    'not contain remote commit {}'.format(local_rev, remote_rev))
             parent = OSTree.commit_get_parent(commit)
             if parent is None:
                 break
-        if remote is not None and parent != remote:
+        if remote_rev is not None and parent != remote_rev:
             self.writer.send_done()
             raise PushExistsException('Remote commit {} not descendent of '
-                                      'commit {}'.format(remote, local))
+                                      'commit {}'.format(remote_rev, local_rev))
 
     def needed_objects(self, commits):
         objects = set()
         for rev in commits:
+            # obtain a set containing all reachable objects from a specific rev
             _, reachable = self.repo.traverse_commit(rev, 0, None)
             for obj in reachable:
+                # Get the objects name. General form: <commit>.GLib.Enum
                 objname = OSTree.object_to_string(obj[0], obj[1])
                 if obj[1] == OSTree.ObjectType.FILE:
                     # Make this a filez since we're archive-z2
@@ -523,6 +537,8 @@ class OSTreePusher(object):
         for branch, rev in self.refs.items():
             remote_rev = remote_refs.get(branch, '0' * 64)
             if rev != remote_rev:
+                # if the checksums for a branch aren't equal add a tuple of
+                # the remote_rev and local rev to a new dictionary.
                 update_refs[branch] = remote_rev, rev
         if not update_refs:
             logging.info('Nothing to update')
@@ -543,9 +559,12 @@ class OSTreePusher(object):
         commits = set()
         exc_info = None
         ref_count = 0
+
+        # update the remote checksum with the local one
         for branch, revs in update_refs.items():
             logging.info('Updating {} {} to {}'.format(branch, revs[0], revs[1]))
             try:
+                # obtain a set of the commits needed to be pushed
                 self.needed_commits(revs[0], revs[1], commits)
                 ref_count += 1
             except PushExistsException:
@@ -557,6 +576,7 @@ class OSTreePusher(object):
             raise exc_info[0].with_traceback(exc_info[1], exc_info[2])
 
         logging.info('Enumerating objects to send')
+        # obtain a set of the objects which need to be pushed to the server
         objects = self.needed_objects(commits)
 
         # Send all the objects to receiver, checking status after each


[buildstream] 11/15: push.py: Add tests to test remote cache expiry

Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

tvb pushed a commit to branch jennis/136-clean-remote-cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 7c194feaf681f3896799f65b99d518318b0378ac
Author: James Ennis <ja...@codethink.com>
AuthorDate: Wed Apr 25 10:47:21 2018 +0100

    push.py: Add tests to test remote cache expiry
    
    The tests include a new environment variable: BST_CACHE_QUOTA.
    This variable is used in receive_main() in pushreceive.py.
    Test names: test_artifact_expires, test_large_artifact_is_not_cached_remotely
    test_recently_pulled_artifact_does_not_expire: marked as xfail until
    we implement LRU expiry in remote share
---
 buildstream/_artifactcache/pushreceive.py |   4 +
 tests/frontend/push.py                    | 164 +++++++++++++++++++++++++++++-
 2 files changed, 167 insertions(+), 1 deletion(-)

diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py
index c60011d..57a4b4e 100644
--- a/buildstream/_artifactcache/pushreceive.py
+++ b/buildstream/_artifactcache/pushreceive.py
@@ -871,5 +871,9 @@ def receive_main(verbose, debug, pull_url, cache_quota, repo):
     logging.basicConfig(format='%(module)s: %(levelname)s: %(message)s',
                         level=loglevel, stream=sys.stderr)
 
+    # Use environment variable for the tests
+    if not cache_quota and 'BST_TEST_CACHE_QUOTA' in os.environ:
+        cache_quota = os.getenv('BST_TEST_CACHE_QUOTA')
+
     receiver = OSTreeReceiver(repo, pull_url, cache_quota)
     return receiver.run()
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index e9c2daa..e23d134 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -2,7 +2,7 @@ import os
 import shutil
 import pytest
 from buildstream._exceptions import ErrorDomain
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import cli, create_artifact_share, create_element_size
 from tests.testutils.site import IS_LINUX
 
 # Project directory
@@ -195,3 +195,165 @@ def test_push_after_pull(cli, tmpdir, datafiles):
     result.assert_success()
     assert result.get_pulled_elements() == ['target.bst']
     assert result.get_pushed_elements() == ['target.bst']
+
+
+# Ensure that when a pushed artifact's size exceeds the artifact share's
+# quota, the least recently pushed artifact is deleted.
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_expires(cli, datafiles, tmpdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_path = os.path.join(project, 'elements')
+
+    # Create an artifact share (remote artifact cache) in the tmpdir/artifactshare
+    share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
+
+    # Set an environment variable of 12 MB for the cache_quota
+    os.environ['BST_TEST_CACHE_QUOTA'] = "12M"
+
+    # Configure bst to push to the cache
+    cli.configure({
+        'artifacts': {'url': share.repo, 'push': True},
+    })
+
+    # Create and build an element of 5 MB
+    create_element_size('element1.bst', element_path, [], 5000000)  # [] => no deps
+    result = cli.run(project=project, args=['build', 'element1.bst'])
+    result.assert_success()
+
+    # Create and build an element of 5 MB
+    create_element_size('element2.bst', element_path, [], 5000000)  # [] => no deps
+    result = cli.run(project=project, args=['build', 'element2.bst'])
+    result.assert_success()
+
+    # update the share
+    share.update_summary()
+
+    # check that element's 1 and 2 are cached both locally and remotely
+    assert cli.get_element_state(project, 'element1.bst') == 'cached'
+    assert_shared(cli, share, project, 'element1.bst')
+    assert cli.get_element_state(project, 'element2.bst') == 'cached'
+    assert_shared(cli, share, project, 'element2.bst')
+
+    # Create and build another element of 5 MB (This will exceed cache quota)
+    create_element_size('element3.bst', element_path, [], 5000000)
+    result = cli.run(project=project, args=['build', 'element3.bst'])
+    result.assert_success()
+
+    # update the share
+    share.update_summary()
+
+    # Ensure it is cached both locally and remotely
+    assert cli.get_element_state(project, 'element3.bst') == 'cached'
+    assert_shared(cli, share, project, 'element3.bst')
+
+    # Ensure element1 has been removed from the share
+    assert_not_shared(cli, share, project, 'element1.bst')
+    # Ensure that elemen2 remains
+    assert_shared(cli, share, project, 'element2.bst')
+
+
+# Test that a large artifact, whose size exceeds the quota, is not pushed
+# to the remote share
+@pytest.mark.datafiles(DATA_DIR)
+def test_huge_artifact(cli, datafiles, tmpdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_path = os.path.join(project, 'elements')
+
+    # Create an artifact share (remote cache) in tmpdir/artifactshare
+    share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
+
+    # Set an environment variable of 5 MB for the cache_quota
+    os.environ['BST_TEST_CACHE_QUOTA'] = "5M"
+
+    # Configure bst to push to the cache
+    cli.configure({
+        'artifacts': {'url': share.repo, 'push': True},
+    })
+
+    create_element_size('small_element.bst', element_path, [], 3000000)
+    result = cli.run(project=project, args=['build', 'small_element.bst'])
+    result.assert_success()
+
+    # Create and build element of 6 MB.
+    create_element_size('large_element.bst', element_path, [], 6000000)
+    result = cli.run(project=project, args=['build', 'large_element.bst'])
+    result.assert_success()
+
+    # update the cache
+    share.update_summary()
+
+    # Ensure that the small artifact is still in the share
+    assert cli.get_element_state(project, 'small_element.bst') == 'cached'
+    assert_shared(cli, share, project, 'small_element.bst')
+
+    # Ensure that the artifact is cached locally but NOT remotely
+    assert cli.get_element_state(project, 'large_element.bst') == 'cached'
+    assert_not_shared(cli, share, project, 'large_element.bst')
+
+
+# Test that when an element is pulled recently, it is not considered the LRU element.
+# NOTE: We expect this test to fail as the current implementation of remote cache
+# expiry only expiries from least recently pushed. NOT least recently used. This will
+# hopefully change when we implement as CAS cache.
+@pytest.mark.xfail
+@pytest.mark.datafiles(DATA_DIR)
+def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_path = os.path.join(project, 'elements')
+
+    # Create an artifact share (remote cache) in tmpdir/artifactshare
+    share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'))
+
+    # Set an environment variable of 12 MB for the cache_quota
+    os.environ['BST_TEST_CACHE_QUOTA'] = "12M"
+
+    # Configure bst to push to the cache
+    cli.configure({
+        'artifacts': {'url': share.repo, 'push': True},
+    })
+
+    # Create and build 2 elements, each of 5 MB.
+    create_element_size('element1.bst', element_path, [], 5000000)
+    result = cli.run(project=project, args=['build', 'element1.bst'])
+    result.assert_success()
+
+    create_element_size('element2.bst', element_path, [], 5000000)
+    result = cli.run(project=project, args=['build', 'element2.bst'])
+    result.assert_success()
+
+    share.update_summary()
+
+    # Ensure they are cached locally
+    assert cli.get_element_state(project, 'element1.bst') == 'cached'
+    assert cli.get_element_state(project, 'element2.bst') == 'cached'
+
+    # Ensure that they have  been pushed to the cache
+    assert_shared(cli, share, project, 'element1.bst')
+    assert_shared(cli, share, project, 'element2.bst')
+
+    # Remove element1 from the local cache
+    cli.remove_artifact_from_cache(project, 'element1.bst')
+    assert cli.get_element_state(project, 'element1.bst') != 'cached'
+
+    # Pull the first element from the remote (this should update its mtime)
+    result = cli.run(project=project, args=['pull', 'element1.bst', '--remote',
+                                            share.repo])
+    result.assert_success()
+
+    # Ensure element1 is cached locally
+    assert cli.get_element_state(project, 'element1.bst') == 'cached'
+
+    # Create and build the element3 (of 5 MB)
+    create_element_size('element3.bst', element_path, [], 5000000)
+    result = cli.run(project=project, args=['build', 'element3.bst'])
+    result.assert_success()
+
+    share.update_summary()
+
+    # Make sure it's cached locally and remotely
+    assert cli.get_element_state(project, 'element3.bst') == 'cached'
+    assert_shared(cli, share, project, 'element3.bst')
+
+    # Ensure that element2 was deleted from the share and element1 remains
+    assert_not_shared(cli, share, project, 'element2.bst')
+    assert_shared(cli, share, project, 'element1.bst')