You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 07:12:24 UTC
[buildstream] branch tpollard/prototemp created (now f424ad1)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at f424ad1 Move back to using artifact method for proto in push
This branch includes the following new commits:
new c65ba16 Add artifact directory
new 8046145 cascache: move list refs method to utils
new e53a64f casremote: Add artifact service stub
new 23784d0 artifactcache tests: change artifactcache variables
new 46d6dc9 ArtifactServicer: Make GetArtifact update mtime blobs
new 7570f90 Add element_ref_name method
new d19d31c cascache: Make diff_trees public
new 080edf8 Move _remove_ref method to utils module
new 97479a6 _artifact.py: Rework to use artifact proto
new d8b1ff9 Remove excluded_subdir/subdir options
new 8f8e8b8 Remove unused progress callback
new 6e8c9bc _cas/cascache.py: Remove unused list_refs() method
new f424ad1 Move back to using artifact method for proto in push
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 07/13: cascache: Make diff_trees public
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d19d31cf8600a73e2b63ece4284288e745c852fb
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Thu Apr 11 15:41:26 2019 +0100
cascache: Make diff_trees public
Part of #974
---
buildstream/_cas/cascache.py | 108 +++++++++++++++++++++----------------------
1 file changed, 54 insertions(+), 54 deletions(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index d7ab869..265ee58 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -241,7 +241,7 @@ class CASCache():
removed = []
modified = []
- self._diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified)
+ self.diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified)
return modified, removed, added
@@ -693,6 +693,59 @@ class CASCache():
if dirnode.name not in excluded_subdirs:
yield from self.required_blobs_for_directory(dirnode.digest)
+ def diff_trees(self, tree_a, tree_b, *, added, removed, modified, path=""):
+ dir_a = remote_execution_pb2.Directory()
+ dir_b = remote_execution_pb2.Directory()
+
+ if tree_a:
+ with open(self.objpath(tree_a), 'rb') as f:
+ dir_a.ParseFromString(f.read())
+ if tree_b:
+ with open(self.objpath(tree_b), 'rb') as f:
+ dir_b.ParseFromString(f.read())
+
+ a = 0
+ b = 0
+ while a < len(dir_a.files) or b < len(dir_b.files):
+ if b < len(dir_b.files) and (a >= len(dir_a.files) or
+ dir_a.files[a].name > dir_b.files[b].name):
+ added.append(os.path.join(path, dir_b.files[b].name))
+ b += 1
+ elif a < len(dir_a.files) and (b >= len(dir_b.files) or
+ dir_b.files[b].name > dir_a.files[a].name):
+ removed.append(os.path.join(path, dir_a.files[a].name))
+ a += 1
+ else:
+ # File exists in both directories
+ if dir_a.files[a].digest.hash != dir_b.files[b].digest.hash:
+ modified.append(os.path.join(path, dir_a.files[a].name))
+ a += 1
+ b += 1
+
+ a = 0
+ b = 0
+ while a < len(dir_a.directories) or b < len(dir_b.directories):
+ if b < len(dir_b.directories) and (a >= len(dir_a.directories) or
+ dir_a.directories[a].name > dir_b.directories[b].name):
+ self.diff_trees(None, dir_b.directories[b].digest,
+ added=added, removed=removed, modified=modified,
+ path=os.path.join(path, dir_b.directories[b].name))
+ b += 1
+ elif a < len(dir_a.directories) and (b >= len(dir_b.directories) or
+ dir_b.directories[b].name > dir_a.directories[a].name):
+ self.diff_trees(dir_a.directories[a].digest, None,
+ added=added, removed=removed, modified=modified,
+ path=os.path.join(path, dir_a.directories[a].name))
+ a += 1
+ else:
+ # Subdirectory exists in both directories
+ if dir_a.directories[a].digest.hash != dir_b.directories[b].digest.hash:
+ self.diff_trees(dir_a.directories[a].digest, dir_b.directories[b].digest,
+ added=added, removed=removed, modified=modified,
+ path=os.path.join(path, dir_a.directories[a].name))
+ a += 1
+ b += 1
+
################################################
# Local Private Methods #
################################################
@@ -807,59 +860,6 @@ class CASCache():
raise CASCacheError("Subdirectory {} not found".format(name))
- def _diff_trees(self, tree_a, tree_b, *, added, removed, modified, path=""):
- dir_a = remote_execution_pb2.Directory()
- dir_b = remote_execution_pb2.Directory()
-
- if tree_a:
- with open(self.objpath(tree_a), 'rb') as f:
- dir_a.ParseFromString(f.read())
- if tree_b:
- with open(self.objpath(tree_b), 'rb') as f:
- dir_b.ParseFromString(f.read())
-
- a = 0
- b = 0
- while a < len(dir_a.files) or b < len(dir_b.files):
- if b < len(dir_b.files) and (a >= len(dir_a.files) or
- dir_a.files[a].name > dir_b.files[b].name):
- added.append(os.path.join(path, dir_b.files[b].name))
- b += 1
- elif a < len(dir_a.files) and (b >= len(dir_b.files) or
- dir_b.files[b].name > dir_a.files[a].name):
- removed.append(os.path.join(path, dir_a.files[a].name))
- a += 1
- else:
- # File exists in both directories
- if dir_a.files[a].digest.hash != dir_b.files[b].digest.hash:
- modified.append(os.path.join(path, dir_a.files[a].name))
- a += 1
- b += 1
-
- a = 0
- b = 0
- while a < len(dir_a.directories) or b < len(dir_b.directories):
- if b < len(dir_b.directories) and (a >= len(dir_a.directories) or
- dir_a.directories[a].name > dir_b.directories[b].name):
- self._diff_trees(None, dir_b.directories[b].digest,
- added=added, removed=removed, modified=modified,
- path=os.path.join(path, dir_b.directories[b].name))
- b += 1
- elif a < len(dir_a.directories) and (b >= len(dir_b.directories) or
- dir_b.directories[b].name > dir_a.directories[a].name):
- self._diff_trees(dir_a.directories[a].digest, None,
- added=added, removed=removed, modified=modified,
- path=os.path.join(path, dir_a.directories[a].name))
- a += 1
- else:
- # Subdirectory exists in both directories
- if dir_a.directories[a].digest.hash != dir_b.directories[b].digest.hash:
- self._diff_trees(dir_a.directories[a].digest, dir_b.directories[b].digest,
- added=added, removed=removed, modified=modified,
- path=os.path.join(path, dir_a.directories[a].name))
- a += 1
- b += 1
-
def _reachable_refs_dir(self, reachable, tree, update_mtime=False, check_exists=False):
if tree.hash in reachable:
return
[buildstream] 06/13: Add element_ref_name method
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7570f90a68010d2909aacf54778c53ced4e0d34c
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Wed Apr 10 11:39:59 2019 +0100
Add element_ref_name method
This takes a bst path and converts it to what is used in the ref
directory.
---
tests/testutils/artifactshare.py | 11 +++--------
tests/testutils/element_name.py | 14 ++++++++++++++
2 files changed, 17 insertions(+), 8 deletions(-)
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 6c484ce..80959e9 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -1,4 +1,3 @@
-import string
import os
import shutil
import signal
@@ -12,6 +11,8 @@ from buildstream._cas.casserver import create_server
from buildstream._exceptions import CASError
from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from tests.testutils.element_name import element_ref_name
+
# ArtifactShare()
#
@@ -136,13 +137,7 @@ class ArtifactShare():
#
# Replace path separator and chop off the .bst suffix
- element_name = os.path.splitext(element_name.replace(os.sep, '-'))[0]
-
- valid_chars = string.digits + string.ascii_letters + '-._'
- element_name = ''.join([
- x if x in valid_chars else '_'
- for x in element_name
- ])
+ element_name = element_ref_name(element_name)
artifact_key = '{0}/{1}/{2}'.format(project_name, element_name, cache_key)
try:
diff --git a/tests/testutils/element_name.py b/tests/testutils/element_name.py
new file mode 100644
index 0000000..762354e
--- /dev/null
+++ b/tests/testutils/element_name.py
@@ -0,0 +1,14 @@
+import os
+import string
+
+
+def element_ref_name(element_name):
+ # Replace path separator and chop off the .bst suffix
+ element_name = os.path.splitext(element_name.replace(os.sep, '-'))[0]
+
+ # replace other sybols with '_'
+ valid_chars = string.digits + string.ascii_letters + '-._'
+ return ''.join([
+ x if x in valid_chars else '_'
+ for x in element_name
+ ])
[buildstream] 11/13: Remove unused progress callback
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8f8e8b8f208c3afa5d6b2654fdb265a23b39a698
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Mon Apr 15 11:15:29 2019 +0100
Remove unused progress callback
---
buildstream/_artifactcache.py | 3 +--
buildstream/_cas/cascache.py | 5 +----
buildstream/_sourcecache.py | 4 ++--
buildstream/element.py | 17 ++++++-----------
4 files changed, 10 insertions(+), 19 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 2fc3dcf..5df3804 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -324,13 +324,12 @@ class ArtifactCache(BaseCache):
# Args:
# element (Element): The Element whose artifact is to be fetched
# key (str): The cache key to use
- # progress (callable): The progress callback, if any
# pull_buildtrees (bool): Whether to pull buildtrees or not
#
# Returns:
# (bool): True if pull was successful, False if artifact was not available
#
- def pull(self, element, key, *, progress=None, pull_buildtrees=False):
+ def pull(self, element, key, *, pull_buildtrees=False):
project = element._get_project()
for remote in self._remotes[project]:
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 5c1642d..85f4c7e 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -250,14 +250,11 @@ class CASCache():
# Args:
# ref (str): The ref to pull
# remote (CASRemote): The remote repository to pull from
- # progress (callable): The progress callback, if any
- # subdir (str): The optional specific subdir to pull
- # excluded_subdirs (list): The optional list of subdirs to not pull
#
# Returns:
# (bool): True if pull was successful, False if ref was not available
#
- def pull(self, ref, remote, *, progress=None):
+ def pull(self, ref, remote):
try:
remote.init()
diff --git a/buildstream/_sourcecache.py b/buildstream/_sourcecache.py
index 8dfa975..0c2ae3b 100644
--- a/buildstream/_sourcecache.py
+++ b/buildstream/_sourcecache.py
@@ -185,7 +185,7 @@ class SourceCache(BaseCache):
#
# Returns:
# (bool): True if pull successful, False if not
- def pull(self, source, *, progress=None):
+ def pull(self, source):
ref = source._get_source_name()
project = source._get_project()
@@ -196,7 +196,7 @@ class SourceCache(BaseCache):
try:
source.status("Pulling source {} <- {}".format(display_key, remote.spec.url))
- if self.cas.pull(ref, remote, progress=progress):
+ if self.cas.pull(ref, remote):
source.info("Pulled source {} <- {}".format(display_key, remote.spec.url))
# no need to pull from additional remotes
return True
diff --git a/buildstream/element.py b/buildstream/element.py
index 4cf8d89..ba8e93d 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1890,18 +1890,15 @@ class Element(Plugin):
def _pull(self):
context = self._get_context()
- def progress(percent, message):
- self.status(message)
-
# Get optional specific subdir to pull and optional list to not pull
# based off of user context
pull_buildtrees = context.pull_buildtrees
# Attempt to pull artifact without knowing whether it's available
- pulled = self.__pull_strong(progress=progress, pull_buildtrees=pull_buildtrees)
+ pulled = self.__pull_strong(pull_buildtrees=pull_buildtrees)
if not pulled and not self._cached() and not context.get_strict():
- pulled = self.__pull_weak(progress=progress, pull_buildtrees=pull_buildtrees)
+ pulled = self.__pull_weak(pull_buildtrees=pull_buildtrees)
if not pulled:
return False
@@ -2840,11 +2837,10 @@ class Element(Plugin):
# Returns:
# (bool): Whether or not the pull was successful
#
- def __pull_strong(self, *, progress=None, pull_buildtrees):
+ def __pull_strong(self, *, pull_buildtrees):
weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
key = self.__strict_cache_key
- if not self.__artifacts.pull(self, key, progress=progress,
- pull_buildtrees=pull_buildtrees):
+ if not self.__artifacts.pull(self, key, pull_buildtrees=pull_buildtrees):
return False
# update weak ref by pointing it to this newly fetched artifact
@@ -2858,16 +2854,15 @@ class Element(Plugin):
# the weak cache key
#
# Args:
- # progress (callable): The progress callback, if any
# subdir (str): The optional specific subdir to pull
# excluded_subdirs (list): The optional list of subdirs to not pull
#
# Returns:
# (bool): Whether or not the pull was successful
#
- def __pull_weak(self, *, progress=None, pull_buildtrees):
+ def __pull_weak(self, *, pull_buildtrees):
weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
- if not self.__artifacts.pull(self, weak_key, progress=progress,
+ if not self.__artifacts.pull(self, weak_key,
pull_buildtrees=pull_buildtrees):
return False
[buildstream] 04/13: artifactcache tests: change artifactcache
variables
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 23784d07c44671e99ea999e7412e5c033fe5bac7
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Fri Apr 5 11:09:01 2019 +0100
artifactcache tests: change artifactcache variables
A few variables were naming artifactcache variables cas, which was
confusing. Changed this and comments to clarify this.
---
tests/artifactcache/pull.py | 26 +++++++++++++-------------
tests/artifactcache/push.py | 28 ++++++++++++++--------------
2 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index 96fdef8..40fed76 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -140,18 +140,18 @@ def _test_pull(user_config_file, project_dir, cache_dir,
project = Project(project_dir, context)
project.ensure_fully_loaded()
- # Create a local CAS cache handle
- cas = context.artifactcache
+ # Create a local artifact cache handle
+ artifactcache = context.artifactcache
# Load the target element
element = project.load_elements([element_name])[0]
# Manually setup the CAS remote
- cas.setup_remotes(use_config=True)
+ artifactcache.setup_remotes(use_config=True)
- if cas.has_push_remotes(plugin=element):
+ if artifactcache.has_push_remotes(plugin=element):
# Push the element's artifact
- if not cas.pull(element, element_key):
+ if not artifactcache.pull(element, element_key):
queue.put("Pull operation failed")
else:
queue.put(None)
@@ -203,7 +203,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
project = Project(project_dir, context)
project.ensure_fully_loaded()
artifactcache = context.artifactcache
- cas = artifactcache.cas
+ cas = context.get_cascache()
# Assert that the element's artifact is cached
element = project.load_elements(['target.bst'])[0]
@@ -278,9 +278,9 @@ def _test_push_tree(user_config_file, project_dir, artifact_digest, queue):
project = Project(project_dir, context)
project.ensure_fully_loaded()
- # Create a local CAS cache handle
+ # Create a local artifact cache and cas handle
artifactcache = context.artifactcache
- cas = artifactcache.cas
+ cas = context.get_cascache()
# Manually setup the CAS remote
artifactcache.setup_remotes(use_config=True)
@@ -313,15 +313,15 @@ def _test_pull_tree(user_config_file, project_dir, artifact_digest, queue):
project = Project(project_dir, context)
project.ensure_fully_loaded()
- # Create a local CAS cache handle
- cas = context.artifactcache
+ # Create a local artifact cache handle
+ artifactcache = context.artifactcache
# Manually setup the CAS remote
- cas.setup_remotes(use_config=True)
+ artifactcache.setup_remotes(use_config=True)
- if cas.has_push_remotes():
+ if artifactcache.has_push_remotes():
# Pull the artifact using the Tree object
- directory_digest = cas.pull_tree(project, artifact_digest)
+ directory_digest = artifactcache.pull_tree(project, artifact_digest)
queue.put((directory_digest.hash, directory_digest.size_bytes))
else:
queue.put("No remote configured")
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 7c117e3..4c1d2cd 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -112,19 +112,19 @@ def _test_push(user_config_file, project_dir, element_name, element_key, queue):
project = Project(project_dir, context)
project.ensure_fully_loaded()
- # Create a local CAS cache handle
- cas = context.artifactcache
+ # Create a local artifact cache handle
+ artifactcache = context.artifactcache
# Load the target element
element = project.load_elements([element_name])[0]
- # Manually setup the CAS remote
- cas.setup_remotes(use_config=True)
- cas.initialize_remotes()
+ # Manually setup the CAS remotes
+ artifactcache.setup_remotes(use_config=True)
+ artifactcache.initialize_remotes()
- if cas.has_push_remotes(plugin=element):
+ if artifactcache.has_push_remotes(plugin=element):
# Push the element's artifact
- if not cas.push(element, [element_key]):
+ if not artifactcache.push(element, [element_key]):
queue.put("Push operation failed")
else:
queue.put(None)
@@ -189,21 +189,21 @@ def _test_push_message(user_config_file, project_dir, queue):
project = Project(project_dir, context)
project.ensure_fully_loaded()
- # Create a local CAS cache handle
- cas = context.artifactcache
+ # Create a local artifact cache handle
+ artifactcache = context.artifactcache
- # Manually setup the CAS remote
- cas.setup_remotes(use_config=True)
- cas.initialize_remotes()
+ # Manually setup the artifact remote
+ artifactcache.setup_remotes(use_config=True)
+ artifactcache.initialize_remotes()
- if cas.has_push_remotes():
+ if artifactcache.has_push_remotes():
# Create an example message object
command = remote_execution_pb2.Command(arguments=['/usr/bin/gcc', '--help'],
working_directory='/buildstream-build',
output_directories=['/buildstream-install'])
# Push the message object
- command_digest = cas.push_message(project, command)
+ command_digest = artifactcache.push_message(project, command)
queue.put((command_digest.hash, command_digest.size_bytes))
else:
[buildstream] 01/13: Add artifact directory
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c65ba16fb630b1f184b49aa1d720865133c7fef6
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Thu Mar 28 12:59:40 2019 +0000
Add artifact directory
This will be used to store artifact protos.
Part of #974
---
buildstream/_artifactcache.py | 6 ++++++
buildstream/_context.py | 4 ++++
2 files changed, 10 insertions(+)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 7a6f2ea..215fb51 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -17,6 +17,8 @@
# Authors:
# Tristan Maat <tr...@codethink.co.uk>
+import os
+
from ._basecache import BaseCache
from .types import _KeyStrength
from ._exceptions import ArtifactError, CASCacheError, CASError
@@ -55,6 +57,10 @@ class ArtifactCache(BaseCache):
self._required_elements = set() # The elements required for this session
+ # create artifact directory
+ self.artifactdir = context.artifactdir
+ os.makedirs(self.artifactdir, exist_ok=True)
+
self.casquota.add_ref_callbacks(self.required_artifacts)
self.casquota.add_remove_callbacks((lambda x: not x.startswith('@'), self.remove))
diff --git a/buildstream/_context.py b/buildstream/_context.py
index fffeea1..93e3f62 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -75,6 +75,9 @@ class Context():
# The directory for CAS
self.casdir = None
+ # The directory for artifact protos
+ self.artifactdir = None
+
# The directory for temporary files
self.tmpdir = None
@@ -230,6 +233,7 @@ class Context():
self.tmpdir = os.path.join(self.cachedir, 'tmp')
self.casdir = os.path.join(self.cachedir, 'cas')
self.builddir = os.path.join(self.cachedir, 'build')
+ self.artifactdir = os.path.join(self.cachedir, 'artifacts')
# Move old artifact cas to cas if it exists and create symlink
old_casdir = os.path.join(self.cachedir, 'artifacts', 'cas')
[buildstream] 05/13: ArtifactServicer: Make GetArtifact update
mtime blobs
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 46d6dc9b1d329472656980c022bfcdbcca44b9db
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Mon Apr 8 17:39:24 2019 +0100
ArtifactServicer: Make GetArtifact update mtime blobs
Part of #974
---
buildstream/_cas/casserver.py | 23 +++++++++++++++++------
1 file changed, 17 insertions(+), 6 deletions(-)
diff --git a/buildstream/_cas/casserver.py b/buildstream/_cas/casserver.py
index f88db71..c08a4d5 100644
--- a/buildstream/_cas/casserver.py
+++ b/buildstream/_cas/casserver.py
@@ -428,11 +428,25 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
with open(artifact_path, 'rb') as f:
artifact.ParseFromString(f.read())
- files_digest = artifact.files
-
# Now update mtimes of files present.
try:
- self.cas.update_tree_mtime(files_digest)
+
+ if str(artifact.files):
+ self.cas.update_tree_mtime(artifact.files)
+
+ if str(artifact.buildtree):
+ # buildtrees might not be there
+ try:
+ self.cas.update_tree_mtime(artifact.buildtree)
+ except FileNotFoundError:
+ pass
+
+ if str(artifact.public_data):
+ os.utime(self.cas.objpath(artifact.public_data))
+
+ for log_file in artifact.logs:
+ os.utime(self.cas.objpath(log_file.digest))
+
except FileNotFoundError:
os.unlink(artifact_path)
context.abort(grpc.StatusCode.NOT_FOUND,
@@ -451,9 +465,6 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer):
# Unset protocol buffers don't evaluated to False but do return empty
# strings, hence str()
- if str(artifact.buildtree):
- self._check_directory("buildtree", artifact.buildtree, context)
-
if str(artifact.public_data):
self._check_file("public data", artifact.public_data, context)
[buildstream] 10/13: Remove excluded_subdir/subdir options
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d8b1ff90339e31daa16cf5c9251f11c14d22d8d6
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Fri Apr 12 12:08:35 2019 +0100
Remove excluded_subdir/subdir options
With artifact as a proto, it doesn't make sense to do it this way, bits
of code can be removed.
Part of #974
---
buildstream/_artifactcache.py | 9 +++-----
buildstream/_cas/cascache.py | 10 +++-----
buildstream/element.py | 53 +++++++++----------------------------------
3 files changed, 17 insertions(+), 55 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 7dc1aee..2fc3dcf 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -268,9 +268,8 @@ class ArtifactCache(BaseCache):
# element (Element): The element whose artifacts to compare
# key_a (str): The first artifact key
# key_b (str): The second artifact key
- # subdir (str): A subdirectory to limit the comparison to
#
- def diff(self, element, key_a, key_b, *, subdir=None):
+ def diff(self, element, key_a, key_b):
digest_a = self.get_artifact_proto(element.get_artifact_name(key_a)).files
digest_b = self.get_artifact_proto(element.get_artifact_name(key_b)).files
@@ -326,15 +325,13 @@ class ArtifactCache(BaseCache):
# element (Element): The Element whose artifact is to be fetched
# key (str): The cache key to use
# progress (callable): The progress callback, if any
- # subdir (str): The optional specific subdir to pull
- # excluded_subdirs (list): The optional list of subdirs to not pull
+ # pull_buildtrees (bool): Whether to pull buildtrees or not
#
# Returns:
# (bool): True if pull was successful, False if artifact was not available
#
- def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
+ def pull(self, element, key, *, progress=None, pull_buildtrees=False):
project = element._get_project()
- pull_buildtrees = "buildtree" not in excluded_subdirs if excluded_subdirs else True
for remote in self._remotes[project]:
remote.init()
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 0bbeedd..5c1642d 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -231,14 +231,10 @@ class CASCache():
# ref_b (str): The second ref
# subdir (str): A subdirectory to limit the comparison to
#
- def diff(self, ref_a, ref_b, *, subdir=None):
+ def diff(self, ref_a, ref_b):
tree_a = self.resolve_ref(ref_a)
tree_b = self.resolve_ref(ref_b)
- if subdir:
- tree_a = self._get_subdir(tree_a, subdir)
- tree_b = self._get_subdir(tree_b, subdir)
-
added = []
removed = []
modified = []
@@ -261,7 +257,7 @@ class CASCache():
# Returns:
# (bool): True if pull was successful, False if ref was not available
#
- def pull(self, ref, remote, *, progress=None, subdir=None, excluded_subdirs=None):
+ def pull(self, ref, remote, *, progress=None):
try:
remote.init()
@@ -275,7 +271,7 @@ class CASCache():
self._fetch_directory(remote, tree)
# Fetch files, excluded_subdirs determined in pullqueue
- required_blobs = self.required_blobs_for_directory(tree, excluded_subdirs=excluded_subdirs)
+ required_blobs = self.required_blobs_for_directory(tree)
missing_blobs = self.local_missing_blobs(required_blobs)
if missing_blobs:
self.fetch_blobs(remote, missing_blobs)
diff --git a/buildstream/element.py b/buildstream/element.py
index 7b7d2f8..4cf8d89 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1853,9 +1853,9 @@ class Element(Plugin):
# Check whether the pull has been invoked with a specific subdir requested
# in user context, as to complete a partial artifact
- subdir, _ = self.__pull_directories()
+ pull_buildtrees = self._get_context().pull_buildtrees
- if self.__strong_cached and subdir == 'buildtree':
+ if self.__strong_cached and pull_buildtrees:
# If we've specified a subdir, check if the subdir is cached locally
if self.__artifacts.contains_buildtree(self, self.__strict_cache_key):
return False
@@ -1895,13 +1895,13 @@ class Element(Plugin):
# Get optional specific subdir to pull and optional list to not pull
# based off of user context
- subdir, excluded_subdirs = self.__pull_directories()
+ pull_buildtrees = context.pull_buildtrees
# Attempt to pull artifact without knowing whether it's available
- pulled = self.__pull_strong(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
+ pulled = self.__pull_strong(progress=progress, pull_buildtrees=pull_buildtrees)
if not pulled and not self._cached() and not context.get_strict():
- pulled = self.__pull_weak(progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
+ pulled = self.__pull_weak(progress=progress, pull_buildtrees=pull_buildtrees)
if not pulled:
return False
@@ -2840,11 +2840,11 @@ class Element(Plugin):
# Returns:
# (bool): Whether or not the pull was successful
#
- def __pull_strong(self, *, progress=None, subdir=None, excluded_subdirs=None):
+ def __pull_strong(self, *, progress=None, pull_buildtrees):
weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
key = self.__strict_cache_key
- if not self.__artifacts.pull(self, key, progress=progress, subdir=subdir,
- excluded_subdirs=excluded_subdirs):
+ if not self.__artifacts.pull(self, key, progress=progress,
+ pull_buildtrees=pull_buildtrees):
return False
# update weak ref by pointing it to this newly fetched artifact
@@ -2865,10 +2865,10 @@ class Element(Plugin):
# Returns:
# (bool): Whether or not the pull was successful
#
- def __pull_weak(self, *, progress=None, subdir=None, excluded_subdirs=None):
+ def __pull_weak(self, *, progress=None, pull_buildtrees):
weak_key = self._get_cache_key(strength=_KeyStrength.WEAK)
- if not self.__artifacts.pull(self, weak_key, progress=progress, subdir=subdir,
- excluded_subdirs=excluded_subdirs):
+ if not self.__artifacts.pull(self, weak_key, progress=progress,
+ pull_buildtrees=pull_buildtrees):
return False
# extract strong cache key from this newly fetched artifact
@@ -2880,37 +2880,6 @@ class Element(Plugin):
return True
- # __pull_directories():
- #
- # Which directories to include or exclude given the current
- # context
- #
- # Returns:
- # subdir (str): The optional specific subdir to include, based
- # on user context
- # excluded_subdirs (list): The optional list of subdirs to not
- # pull, referenced against subdir value
- #
- def __pull_directories(self):
- context = self._get_context()
-
- # Current default exclusions on pull
- excluded_subdirs = ["buildtree"]
- subdir = ''
-
- # If buildtrees are to be pulled, remove the value from exclusion list
- # and set specific subdir
- if context.pull_buildtrees:
- subdir = "buildtree"
- excluded_subdirs.remove(subdir)
-
- # If file contents are not required for this element, don't pull them.
- # The directories themselves will always be pulled.
- if not context.require_artifact_files and not self._artifact_files_required():
- excluded_subdirs.append("files")
-
- return (subdir, excluded_subdirs)
-
# __cache_sources():
#
# Caches the sources into the local CAS
[buildstream] 02/13: cascache: move list refs method to utils
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 8046145bc6f21895a3e59c33348de79a69898f03
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Mon Apr 1 14:04:20 2019 +0100
cascache: move list refs method to utils
This is more generic such that it can be used for listing files in other
paths.
Part #974
---
buildstream/_cas/cascache.py | 28 ++--------------------------
buildstream/utils.py | 30 ++++++++++++++++++++++++++++++
2 files changed, 32 insertions(+), 26 deletions(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 5f67dc0..d7ab869 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -24,7 +24,6 @@ import stat
import errno
import uuid
import contextlib
-from fnmatch import fnmatch
import grpc
@@ -514,31 +513,8 @@ class CASCache():
#
def list_refs(self, *, glob=None):
# string of: /path/to/repo/refs/heads
- ref_heads = os.path.join(self.casdir, 'refs', 'heads')
- path = ref_heads
-
- if glob is not None:
- globdir = os.path.dirname(glob)
- if not any(c in "*?[" for c in globdir):
- # path prefix contains no globbing characters so
- # append the glob to optimise the os.walk()
- path = os.path.join(ref_heads, globdir)
-
- refs = []
- mtimes = []
-
- for root, _, files in os.walk(path):
- for filename in files:
- ref_path = os.path.join(root, filename)
- relative_path = os.path.relpath(ref_path, ref_heads) # Relative to refs head
- if not glob or fnmatch(relative_path, glob):
- refs.append(relative_path)
- # Obtain the mtime (the time a file was last modified)
- mtimes.append(os.path.getmtime(ref_path))
-
- # NOTE: Sorted will sort from earliest to latest, thus the
- # first ref of this list will be the file modified earliest.
- return [ref for _, ref in sorted(zip(mtimes, refs))]
+ return [ref for _, ref in sorted(list(utils._list_directory(
+ os.path.join(self.casdir, 'refs', 'heads'), glob_expr=glob)))]
# list_objects():
#
diff --git a/buildstream/utils.py b/buildstream/utils.py
index ade5937..7d6db0b 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -23,6 +23,7 @@ Utilities
import calendar
import errno
+from fnmatch import fnmatch
import hashlib
import os
import re
@@ -1291,3 +1292,32 @@ def _deterministic_umask():
yield
finally:
os.umask(old_umask)
+
+
+# _list_directory()
+#
+# List files in a directory, given a base path
+#
+# Args:
+# base_path (str): Base path to traverse over
+# glob_expr (str|None): Optional glob expression to match against files
+#
+# Returns:
+# (iter (mtime, filename)]): iterator of tuples of mtime and filenames
+#
+def _list_directory(base_path, *, glob_expr=None):
+ path = base_path
+ if glob_expr is not None:
+ globdir = os.path.dirname(glob_expr)
+ if not any(c in "*?[" for c in globdir):
+ # path prefix contains no globbing characters so
+ # append the glob to optimise the os.walk()
+ path = os.path.join(base_path, globdir)
+
+ for root, _, files in os.walk(path):
+ for filename in files:
+ ref_path = os.path.join(root, filename)
+ relative_path = os.path.relpath(ref_path, base_path) # Relative to refs head
+ if not glob_expr or fnmatch(relative_path, glob_expr):
+ # Obtain the mtime (the time a file was last modified)
+ yield (os.path.getmtime(ref_path), relative_path)
[buildstream] 12/13: _cas/cascache.py: Remove unused list_refs()
method
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6e8c9bcfe2b0afae97264358908852886fc27140
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Thu Apr 25 11:01:06 2019 +0100
_cas/cascache.py: Remove unused list_refs() method
---
buildstream/_cas/cascache.py | 15 ---------------
1 file changed, 15 deletions(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 85f4c7e..e15da1a 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -496,21 +496,6 @@ class CASCache():
except FileNotFoundError as e:
raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
- # list_refs():
- #
- # List refs in Least Recently Modified (LRM) order.
- #
- # Args:
- # glob (str) - An optional glob expression to be used to list refs satisfying the glob
- #
- # Returns:
- # (list) - A list of refs in LRM order
- #
- def list_refs(self, *, glob=None):
- # string of: /path/to/repo/refs/heads
- return [ref for _, ref in sorted(list(utils._list_directory(
- os.path.join(self.casdir, 'refs', 'heads'), glob_expr=glob)))]
-
# list_objects():
#
# List cached objects in Least Recently Modified (LRM) order.
[buildstream] 13/13: Move back to using artifact method for proto
in push
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f424ad1d857b1c1c79adb0e6823534f1786398ca
Author: Tom Pollard <to...@codethink.co.uk>
AuthorDate: Mon Apr 29 17:12:23 2019 +0100
Move back to using artifact method for proto in push
---
buildstream/_artifactcache.py | 2 +-
buildstream/element.py | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 5df3804..4d074c2 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -542,7 +542,7 @@ class ArtifactCache(BaseCache):
keys = list(keys)
if not keys:
keys = [element._get_cache_key()]
- artifacts = list(map(self.get_artifact_proto, list(map(element.get_artifact_name, keys))))
+ artifacts = list(map(element.get_artifact_proto, keys))
# check the artifacts are the same for each key
# unsure how necessary this is
artifact = artifacts[0]
diff --git a/buildstream/element.py b/buildstream/element.py
index ba8e93d..0f45606 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -455,6 +455,9 @@ class Element(Plugin):
yield from visit(self, scope, visited)
+ def get_artifact_proto(self, key):
+ return self.__artifact._get_proto()
+
def search(self, scope, name):
"""Search for a dependency by name
[buildstream] 08/13: Move _remove_ref method to utils module
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 080edf8967c85262ec713cc2e9a5ff340ca0b24f
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Thu Apr 11 17:34:03 2019 +0100
Move _remove_ref method to utils module
Part of #974
---
buildstream/_cas/cascache.py | 55 ++++----------------------------------------
buildstream/utils.py | 46 ++++++++++++++++++++++++++++++++++++
2 files changed, 50 insertions(+), 51 deletions(-)
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 265ee58..10e8b3e 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -21,7 +21,6 @@ import hashlib
import itertools
import os
import stat
-import errno
import uuid
import contextlib
@@ -568,7 +567,10 @@ class CASCache():
def remove(self, ref, *, defer_prune=False):
# Remove cache ref
- self._remove_ref(ref)
+ try:
+ utils._remove_ref(os.path.join(self.casdir, 'refs', 'heads'), ref)
+ except FileNotFoundError:
+ raise CASCacheError("Could not find ref '{}'".format(ref))
if not defer_prune:
pruned = self.prune()
@@ -753,55 +755,6 @@ class CASCache():
def _refpath(self, ref):
return os.path.join(self.casdir, 'refs', 'heads', ref)
- # _remove_ref()
- #
- # Removes a ref.
- #
- # This also takes care of pruning away directories which can
- # be removed after having removed the given ref.
- #
- # Args:
- # ref (str): The ref to remove
- #
- # Raises:
- # (CASCacheError): If the ref didnt exist, or a system error
- # occurred while removing it
- #
- def _remove_ref(self, ref):
-
- # Remove the ref itself
- refpath = self._refpath(ref)
- try:
- os.unlink(refpath)
- except FileNotFoundError as e:
- raise CASCacheError("Could not find ref '{}'".format(ref)) from e
-
- # Now remove any leading directories
- basedir = os.path.join(self.casdir, 'refs', 'heads')
- components = list(os.path.split(ref))
- while components:
- components.pop()
- refdir = os.path.join(basedir, *components)
-
- # Break out once we reach the base
- if refdir == basedir:
- break
-
- try:
- os.rmdir(refdir)
- except FileNotFoundError:
- # The parent directory did not exist, but it's
- # parent directory might still be ready to prune
- pass
- except OSError as e:
- if e.errno == errno.ENOTEMPTY:
- # The parent directory was not empty, so we
- # cannot prune directories beyond this point
- break
-
- # Something went wrong here
- raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e
-
# _commit_directory():
#
# Adds local directory to content addressable store.
diff --git a/buildstream/utils.py b/buildstream/utils.py
index 7d6db0b..f655286 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -1321,3 +1321,49 @@ def _list_directory(base_path, *, glob_expr=None):
if not glob_expr or fnmatch(relative_path, glob_expr):
# Obtain the mtime (the time a file was last modified)
yield (os.path.getmtime(ref_path), relative_path)
+
+
+# remove_ref()
+#
+# Removes a ref
+#
+# This also takes care of pruning away directories which can
+# be removed after having removed the given ref.
+#
+# Args:
+# basedir (str): Path of base directory the ref is in
+# ref (str): The ref to remove
+#
+# Raises:
+# (CASCacheError): If the ref didnt exist, or a system error
+# occurred while removing it
+#
+def _remove_ref(basedir, ref):
+ # Remove the ref itself
+ refpath = os.path.join(basedir, ref)
+ os.unlink(refpath)
+
+ # Now remove any leading directories
+ components = list(os.path.split(ref))
+ while components:
+ components.pop()
+ refdir = os.path.join(basedir, *components)
+
+ # Break out once we reach the base
+ if refdir == basedir:
+ break
+
+ try:
+ os.rmdir(refdir)
+ except FileNotFoundError:
+ # The parent directory did not exist, but it's
+ # parent directory might still be ready to prune
+ pass
+ except OSError as e:
+ if e.errno == errno.ENOTEMPTY:
+ # The parent directory was not empty, so we
+ # cannot prune directories beyond this point
+ break
+
+ # Something went wrong here
+ raise BstError("System error while removing ref '{}': {}".format(ref, e)) from e
[buildstream] 03/13: casremote: Add artifact service stub
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e53a64fb47f7b558aaaf0ebfb2d8bd63e0a53c9b
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Thu Apr 4 11:31:48 2019 +0100
casremote: Add artifact service stub
Part of #974
---
buildstream/_cas/casremote.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/buildstream/_cas/casremote.py b/buildstream/_cas/casremote.py
index aac0d28..ef6a508 100644
--- a/buildstream/_cas/casremote.py
+++ b/buildstream/_cas/casremote.py
@@ -12,6 +12,7 @@ from .. import _yaml
from .._protos.google.rpc import code_pb2
from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
+from .._protos.buildstream.v2 import artifact_pb2_grpc
from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
from .._exceptions import CASRemoteError, LoadError, LoadErrorReason
@@ -87,6 +88,7 @@ class CASRemote():
self.bytestream = None
self.cas = None
self.ref_storage = None
+ self.artifact = None
self.batch_update_supported = None
self.batch_read_supported = None
self.capabilities = None
@@ -132,6 +134,7 @@ class CASRemote():
self.cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self.channel)
self.capabilities = remote_execution_pb2_grpc.CapabilitiesStub(self.channel)
self.ref_storage = buildstream_pb2_grpc.ReferenceStorageStub(self.channel)
+ self.artifact = artifact_pb2_grpc.ArtifactServiceStub(self.channel)
self.max_batch_total_size_bytes = _MAX_PAYLOAD_BYTES
try:
[buildstream] 09/13: _artifact.py: Rework to use artifact proto
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tpollard/prototemp
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 97479a69d681710202ced5a81089c20e60e5b4da
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Thu Mar 28 12:31:50 2019 +0000
_artifact.py: Rework to use artifact proto
This will replace the previous use of a directory structure.
Quite a lot is changed here, predominantly _artifact and _artifactcache
modules.
Part of #974
---
buildstream/_artifact.py | 304 +++++++++++++++------------------
buildstream/_artifactcache.py | 320 ++++++++++++++++++++++++++++-------
buildstream/_cas/cascache.py | 199 ++++++++++++----------
buildstream/_context.py | 2 +-
buildstream/_sourcecache.py | 41 ++++-
buildstream/_stream.py | 4 +-
buildstream/element.py | 8 +-
buildstream/testing/runcli.py | 37 ++--
tests/artifactcache/junctions.py | 2 +
tests/artifactcache/pull.py | 3 +-
tests/frontend/artifact.py | 12 +-
tests/frontend/pull.py | 46 +++--
tests/frontend/push.py | 8 +
tests/frontend/remote-caches.py | 5 +-
tests/integration/artifact.py | 31 ++--
tests/integration/pullbuildtrees.py | 15 +-
tests/integration/shellbuildtrees.py | 3 +
tests/sourcecache/fetch.py | 1 +
tests/testutils/artifactshare.py | 44 ++++-
19 files changed, 692 insertions(+), 393 deletions(-)
diff --git a/buildstream/_artifact.py b/buildstream/_artifact.py
index 6cf51ee..ed1d6ce 100644
--- a/buildstream/_artifact.py
+++ b/buildstream/_artifact.py
@@ -29,11 +29,11 @@ artifact composite interaction away from Element class
"""
import os
-import shutil
+import tempfile
+from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto
from . import _yaml
from . import utils
-from ._exceptions import ArtifactError
from .types import Scope
from .storage._casbaseddirectory import CasBasedDirectory
@@ -49,12 +49,17 @@ from .storage._casbaseddirectory import CasBasedDirectory
#
class Artifact():
+ version = 0
+
def __init__(self, element, context, *, strong_key=None, weak_key=None):
self._element = element
self._context = context
self._artifacts = context.artifactcache
self._cache_key = strong_key
self._weak_cache_key = weak_key
+ self._artifactdir = context.artifactdir
+ self._cas = context.get_cascache()
+ self._tmpdir = context.tmpdir
self._metadata_keys = None # Strong and weak key tuple extracted from the artifact
self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact
@@ -69,7 +74,9 @@ class Artifact():
# (Directory): The virtual directory object
#
def get_files(self):
- return self._get_subdirectory("files")
+ files_digest = self._get_artifact_field("files")
+
+ return CasBasedDirectory(self._cas, digest=files_digest)
# get_buildtree():
#
@@ -79,7 +86,9 @@ class Artifact():
# (Directory): The virtual directory object
#
def get_buildtree(self):
- return self._get_subdirectory("buildtree")
+ buildtree_digest = self._get_artifact_field("buildtree")
+
+ return CasBasedDirectory(self._cas, digest=buildtree_digest)
# get_extract_key():
#
@@ -100,7 +109,6 @@ class Artifact():
# sandbox_build_dir (Directory): Virtual Directory object for the sandbox build-root
# collectvdir (Directory): Virtual Directoy object from within the sandbox for collection
# buildresult (tuple): bool, short desc and detailed desc of result
- # keys (list): list of keys for the artifact commit metadata
# publicdata (dict): dict of public data to commit to artifact metadata
#
# Returns:
@@ -110,80 +118,78 @@ class Artifact():
context = self._context
element = self._element
+ size = 0
- assemblevdir = CasBasedDirectory(cas_cache=self._artifacts.cas)
- logsvdir = assemblevdir.descend("logs", create=True)
- metavdir = assemblevdir.descend("meta", create=True)
+ filesvdir = None
+ buildtreevdir = None
- # Create artifact directory structure
- assembledir = os.path.join(rootdir, 'artifact')
- logsdir = os.path.join(assembledir, 'logs')
- metadir = os.path.join(assembledir, 'meta')
- os.mkdir(assembledir)
- os.mkdir(logsdir)
- os.mkdir(metadir)
+ artifact = ArtifactProto()
- if collectvdir is not None:
- filesvdir = assemblevdir.descend("files", create=True)
- filesvdir.import_files(collectvdir)
+ artifact.version = self.version
- if sandbox_build_dir:
- buildtreevdir = assemblevdir.descend("buildtree", create=True)
- buildtreevdir.import_files(sandbox_build_dir)
+ # Store result
+ artifact.build_success = buildresult[0]
+ artifact.build_error = buildresult[1]
+ artifact.build_error_details = "" if not buildresult[2] else buildresult[2]
- # Write some logs out to normal directories: logsdir and metadir
- # Copy build log
- log_filename = context.get_log_filename()
- element._build_log_path = os.path.join(logsdir, 'build.log')
- if log_filename:
- shutil.copyfile(log_filename, element._build_log_path)
+ # Store keys
+ artifact.strong_key = self._cache_key
+ artifact.weak_key = self._weak_cache_key
+
+ artifact.was_workspaced = bool(element._get_workspace())
+
+ # Store files
+ if collectvdir:
+ filesvdir = CasBasedDirectory(cas_cache=self._cas)
+ filesvdir.import_files(collectvdir)
+ artifact.files.CopyFrom(filesvdir._get_digest())
+ size += filesvdir.get_size()
# Store public data
- _yaml.dump(_yaml.node_sanitize(publicdata), os.path.join(metadir, 'public.yaml'))
+ with tempfile.NamedTemporaryFile(dir=self._tmpdir) as tmp:
+ _yaml.dump(_yaml.node_sanitize(publicdata), tmp.name)
+ public_data_digest = self._cas.add_object(path=tmp.name, link_directly=True)
+ artifact.public_data.CopyFrom(public_data_digest)
+ size += public_data_digest.size_bytes
+
+ # store build dependencies
+ for e in element.dependencies(Scope.BUILD):
+ new_build = artifact.build_deps.add()
+ new_build.element_name = e.name
+ new_build.cache_key = e._get_cache_key()
+ new_build.was_workspaced = bool(e._get_workspace())
+
+ # Store log file
+ log_filename = context.get_log_filename()
+ if log_filename:
+ digest = self._cas.add_object(path=log_filename)
+ element._build_log_path = self._cas.objpath(digest)
+ log = artifact.logs.add()
+ log.name = os.path.basename(log_filename)
+ log.digest.CopyFrom(digest)
+ size += log.digest.size_bytes
+
+ # Store build tree
+ if sandbox_build_dir:
+ buildtreevdir = CasBasedDirectory(cas_cache=self._cas)
+ buildtreevdir.import_files(sandbox_build_dir)
+ artifact.buildtree.CopyFrom(buildtreevdir._get_digest())
+ size += buildtreevdir.get_size()
- # Store result
- build_result_dict = {"success": buildresult[0], "description": buildresult[1]}
- if buildresult[2] is not None:
- build_result_dict["detail"] = buildresult[2]
- _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
-
- # Store keys.yaml
- _yaml.dump(_yaml.node_sanitize({
- 'strong': self._cache_key,
- 'weak': self._weak_cache_key,
- }), os.path.join(metadir, 'keys.yaml'))
-
- # Store dependencies.yaml
- _yaml.dump(_yaml.node_sanitize({
- e.name: e._get_cache_key() for e in element.dependencies(Scope.BUILD)
- }), os.path.join(metadir, 'dependencies.yaml'))
-
- # Store workspaced.yaml
- _yaml.dump(_yaml.node_sanitize({
- 'workspaced': bool(element._get_workspace())
- }), os.path.join(metadir, 'workspaced.yaml'))
-
- # Store workspaced-dependencies.yaml
- _yaml.dump(_yaml.node_sanitize({
- 'workspaced-dependencies': [
- e.name for e in element.dependencies(Scope.BUILD)
- if e._get_workspace()
- ]
- }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
-
- metavdir.import_files(metadir)
- logsvdir.import_files(logsdir)
-
- artifact_size = assemblevdir.get_size()
+ os.makedirs(os.path.dirname(os.path.join(
+ self._artifactdir, element.get_artifact_name())), exist_ok=True)
keys = utils._deduplicate([self._cache_key, self._weak_cache_key])
- self._artifacts.commit(element, assemblevdir, keys)
+ for key in keys:
+ path = os.path.join(self._artifactdir, element.get_artifact_name(key=key))
+ with open(path, mode='w+b') as f:
+ f.write(artifact.SerializeToString())
- return artifact_size
+ return size
# cached_buildtree()
#
# Check if artifact is cached with expected buildtree. A
- # buildtree will not be present if the res tof the partial artifact
+ # buildtree will not be present if the rest of the partial artifact
# is not cached.
#
# Returns:
@@ -193,14 +199,12 @@ class Artifact():
#
def cached_buildtree(self):
- element = self._element
-
- key = self.get_extract_key()
- if not self._artifacts.contains_subdir_artifact(element, key, 'buildtree'):
+ buildtree_digest = self._get_artifact_field("buildtree")
+ if buildtree_digest:
+ return self._cas.contains_directory(buildtree_digest, with_files=True)
+ else:
return False
- return True
-
# buildtree_exists()
#
# Check if artifact was created with a buildtree. This does not check
@@ -211,8 +215,8 @@ class Artifact():
#
def buildtree_exists(self):
- artifact_vdir = self._get_directory()
- return artifact_vdir._exists('buildtree')
+ artifact = self._get_proto()
+ return bool(str(artifact.buildtree))
# load_public_data():
#
@@ -224,8 +228,8 @@ class Artifact():
def load_public_data(self):
# Load the public data from the artifact
- meta_vdir = self._get_subdirectory('meta')
- meta_file = meta_vdir._objpath('public.yaml')
+ artifact = self._get_proto()
+ meta_file = self._cas.objpath(artifact.public_data)
data = _yaml.load(meta_file, shortname='public.yaml')
return data
@@ -241,20 +245,10 @@ class Artifact():
#
def load_build_result(self):
- meta_vdir = self._get_subdirectory('meta')
-
- meta_file = meta_vdir._objpath('build-result.yaml')
- if not os.path.exists(meta_file):
- build_result = (True, "succeeded", None)
- return build_result
-
- data = _yaml.load(meta_file, shortname='build-result.yaml')
-
- success = _yaml.node_get(data, bool, 'success')
- description = _yaml.node_get(data, str, 'description', default_value=None)
- detail = _yaml.node_get(data, str, 'detail', default_value=None)
-
- build_result = (success, description, detail)
+ artifact = self._get_proto()
+ build_result = (artifact.build_success,
+ artifact.build_error,
+ artifact.build_error_details)
return build_result
@@ -271,14 +265,11 @@ class Artifact():
if self._metadata_keys is not None:
return self._metadata_keys
- # Extract the metadata dir
- meta_vdir = self._get_subdirectory('meta')
+ # Extract proto
+ artifact = self._get_proto()
- # Parse the expensive yaml now and cache the result
- meta_file = meta_vdir._objpath('keys.yaml')
- meta = _yaml.load(meta_file, shortname='keys.yaml')
- strong_key = _yaml.node_get(meta, str, 'strong')
- weak_key = _yaml.node_get(meta, str, 'weak')
+ strong_key = artifact.strong_key
+ weak_key = artifact.weak_key
self._metadata_keys = (strong_key, weak_key)
@@ -296,14 +287,10 @@ class Artifact():
if self._metadata_dependencies is not None:
return self._metadata_dependencies
- # Extract the metadata dir
- meta_vdir = self._get_subdirectory('meta')
-
- # Parse the expensive yaml now and cache the result
- meta_file = meta_vdir._objpath('dependencies.yaml')
- meta = _yaml.load(meta_file, shortname='dependencies.yaml')
+ # Extract proto
+ artifact = self._get_proto()
- self._metadata_dependencies = meta
+ self._metadata_dependencies = {dep.element_name: dep.cache_key for dep in artifact.build_deps}
return self._metadata_dependencies
@@ -319,14 +306,10 @@ class Artifact():
if self._metadata_workspaced is not None:
return self._metadata_workspaced
- # Extract the metadata dir
- meta_vdir = self._get_subdirectory('meta')
+ # Extract proto
+ artifact = self._get_proto()
- # Parse the expensive yaml now and cache the result
- meta_file = meta_vdir._objpath('workspaced.yaml')
- meta = _yaml.load(meta_file, shortname='workspaced.yaml')
-
- self._metadata_workspaced = _yaml.node_get(meta, bool, 'workspaced')
+ self._metadata_workspaced = artifact.was_workspaced
return self._metadata_workspaced
@@ -342,15 +325,11 @@ class Artifact():
if self._metadata_workspaced_dependencies is not None:
return self._metadata_workspaced_dependencies
- # Extract the metadata dir
- meta_vdir = self._get_subdirectory('meta')
-
- # Parse the expensive yaml now and cache the result
- meta_file = meta_vdir._objpath('workspaced-dependencies.yaml')
- meta = _yaml.load(meta_file, shortname='workspaced-dependencies.yaml')
+ # Extract proto
+ artifact = self._get_proto()
- self._metadata_workspaced_dependencies = _yaml.node_sanitize(_yaml.node_get(meta, list,
- 'workspaced-dependencies'))
+ self._metadata_workspaced_dependencies = [dep.element_name for dep in artifact.build_deps
+ if dep.was_workspaced]
return self._metadata_workspaced_dependencies
@@ -369,30 +348,21 @@ class Artifact():
def cached(self):
context = self._context
- try:
- vdir = self._get_directory()
- except ArtifactError:
- # Either ref or top-level artifact directory missing
- return False
+ artifact = self._get_proto()
- # Check whether all metadata is available
- metadigest = vdir._get_child_digest('meta')
- if not self._artifacts.cas.contains_directory(metadigest, with_files=True):
+ if not artifact:
return False
- # Additional checks only relevant if artifact was created with 'files' subdirectory
- if vdir._exists('files'):
- # Determine whether directories are required
- require_directories = context.require_artifact_directories
- # Determine whether file contents are required as well
- require_files = context.require_artifact_files or self._element._artifact_files_required()
-
- filesdigest = vdir._get_child_digest('files')
+ # Determine whether directories are required
+ require_directories = context.require_artifact_directories
+ # Determine whether file contents are required as well
+ require_files = (context.require_artifact_files or
+ self._element._artifact_files_required())
- # Check whether 'files' subdirectory is available, with or without file contents
- if (require_directories and
- not self._artifacts.cas.contains_directory(filesdigest, with_files=require_files)):
- return False
+ # Check whether 'files' subdirectory is available, with or without file contents
+ if (require_directories and str(artifact.files) and
+ not self._cas.contains_directory(artifact.files, with_files=require_files)):
+ return False
return True
@@ -408,46 +378,50 @@ class Artifact():
if not self._element._cached():
return False
- log_vdir = self._get_subdirectory('logs')
+ artifact = self._get_proto()
+
+ for logfile in artifact.logs:
+ if not self._cas.contains(logfile.digest.hash):
+ return False
- logsdigest = log_vdir._get_digest()
- return self._artifacts.cas.contains_directory(logsdigest, with_files=True)
+ return True
- # _get_directory():
- #
- # Get a virtual directory for the artifact contents
+ # _get_proto()
#
# Args:
- # key (str): The key for the artifact to extract,
- # or None for the default key
+ # key (str): Key to use, or None for the default key
#
# Returns:
- # (Directory): The virtual directory object
+ # (Artifact): Artifact proto
#
- def _get_directory(self, key=None):
-
- element = self._element
-
- if key is None:
+ def _get_proto(self, key=None):
+ if not key:
key = self.get_extract_key()
- return self._artifacts.get_artifact_directory(element, key)
+ proto_path = os.path.join(self._artifactdir,
+ self._element.get_artifact_name(key=key))
+ artifact = ArtifactProto()
+ try:
+ with open(proto_path, mode='r+b') as f:
+ artifact.ParseFromString(f.read())
+ except FileNotFoundError:
+ return None
- # _get_subdirectory():
- #
- # Get a virtual directory for the artifact subdir contents
+ os.utime(proto_path)
+ return artifact
+
+ # _get_artifact_field()
#
# Args:
- # subdir (str): The specific artifact subdir
- # key (str): The key for the artifact to extract,
- # or None for the default key
+ # key (str): Key to use, or None for the default key
#
# Returns:
- # (Directory): The virtual subdirectory object
+ # (Digest): Digest of field specified
#
- def _get_subdirectory(self, subdir, key=None):
-
- artifact_vdir = self._get_directory(key)
- sub_vdir = artifact_vdir.descend(subdir)
+ def _get_artifact_field(self, field, key=None):
+ artifact_proto = self._get_proto(key=key)
+ digest = getattr(artifact_proto, field)
+ if not str(digest):
+ return None
- return sub_vdir
+ return digest
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 215fb51..7dc1aee 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -18,14 +18,16 @@
# Tristan Maat <tr...@codethink.co.uk>
import os
+import grpc
+from . import utils
from ._basecache import BaseCache
from .types import _KeyStrength
-from ._exceptions import ArtifactError, CASCacheError, CASError
+from ._exceptions import ArtifactError, CASError
+from ._protos.buildstream.v2 import artifact_pb2
from ._cas import CASRemoteSpec
from .storage._casbaseddirectory import CasBasedDirectory
-from .storage.directory import VirtualDirectoryError
# An ArtifactCacheSpec holds the user configuration for a single remote
@@ -61,8 +63,11 @@ class ArtifactCache(BaseCache):
self.artifactdir = context.artifactdir
os.makedirs(self.artifactdir, exist_ok=True)
- self.casquota.add_ref_callbacks(self.required_artifacts)
- self.casquota.add_remove_callbacks((lambda x: not x.startswith('@'), self.remove))
+ self.casquota.add_remove_callbacks(self.unrequired_artifacts, self.remove)
+ self.casquota.add_list_refs_callback(self.list_artifacts)
+
+ self.cas.add_reachable_directories_callback(self._reachable_directories)
+ self.cas.add_reachable_digests_callback(self._reachable_digests)
# mark_required_elements():
#
@@ -98,13 +103,34 @@ class ArtifactCache(BaseCache):
weak_key = element._get_cache_key(strength=_KeyStrength.WEAK)
for key in (strong_key, weak_key):
if key:
- try:
- ref = element.get_artifact_name(key)
+ ref = element.get_artifact_name(key)
- self.cas.update_mtime(ref)
- except CASError:
+ try:
+ self.update_mtime(ref)
+ except ArtifactError:
pass
+ def update_mtime(self, ref):
+ try:
+ os.utime(os.path.join(self.artifactdir, ref))
+ except FileNotFoundError as e:
+ raise ArtifactError("Couldn't find artifact: {}".format(ref)) from e
+
+ # unrequired_artifacts()
+ #
+ # Returns iterator over artifacts that are not required in the build plan
+ #
+ # Returns:
+ # (iter): Iterator over tuples of (float, str) where float is the time
+ # and str is the artifact ref
+ #
+ def unrequired_artifacts(self):
+ required_artifacts = set(map(lambda x: x.get_artifact_name(),
+ self._required_elements))
+ for (mtime, artifact) in utils._list_directory(self.artifactdir):
+ if artifact not in required_artifacts:
+ yield (mtime, artifact)
+
def required_artifacts(self):
# Build a set of the cache keys which are required
# based on the required elements at cleanup time
@@ -153,7 +179,7 @@ class ArtifactCache(BaseCache):
def contains(self, element, key):
ref = element.get_artifact_name(key)
- return self.cas.contains(ref)
+ return os.path.exists(os.path.join(self.artifactdir, ref))
# contains_subdir_artifact():
#
@@ -172,6 +198,22 @@ class ArtifactCache(BaseCache):
ref = element.get_artifact_name(key)
return self.cas.contains_subdir_artifact(ref, subdir, with_files=with_files)
+ # contains_buildtree()
+ #
+ # Args:
+ # element (Element): The Element to check
+ # key (str): The cache key to use
+ # with_files (bool): Whether to check files as well
+ #
+ # Returns: True if the subdir exists & is populated in the cache, False otherwise
+ #
+ def contains_buildtree(self, element, key, *, with_files=True):
+ artifact = self.get_artifact_proto(element.get_artifact_name(key))
+ if str(artifact.buildtree) and with_files:
+ return self.cas.contains_directory(artifact.buildtree, with_files=with_files)
+ else:
+ return False
+
# list_artifacts():
#
# List artifacts in this cache in LRU order.
@@ -183,9 +225,7 @@ class ArtifactCache(BaseCache):
# ([str]) - A list of artifact names as generated in LRU order
#
def list_artifacts(self, *, glob=None):
- return list(filter(
- lambda x: not x.startswith('@'),
- self.cas.list_refs(glob=glob)))
+ return [ref for _, ref in sorted(list(utils._list_directory(self.artifactdir, glob_expr=glob)))]
# remove():
#
@@ -202,7 +242,15 @@ class ArtifactCache(BaseCache):
# (int): The amount of space recovered in the cache, in bytes
#
def remove(self, ref, *, defer_prune=False):
- return self.cas.remove(ref, defer_prune=defer_prune)
+ try:
+ utils._remove_ref(self.artifactdir, ref)
+ except FileNotFoundError:
+ raise ArtifactError("Could not find ref '{}'".format(ref))
+
+ if not defer_prune:
+ return self.cas.prune()
+
+ return None
# prune():
#
@@ -211,47 +259,6 @@ class ArtifactCache(BaseCache):
def prune(self):
return self.cas.prune()
- # get_artifact_directory():
- #
- # Get virtual directory for cached artifact of the specified Element.
- #
- # Assumes artifact has previously been fetched or committed.
- #
- # Args:
- # element (Element): The Element to extract
- # key (str): The cache key to use
- #
- # Raises:
- # ArtifactError: In cases there was an OSError, or if the artifact
- # did not exist.
- #
- # Returns: virtual directory object
- #
- def get_artifact_directory(self, element, key):
- ref = element.get_artifact_name(key)
- try:
- digest = self.cas.resolve_ref(ref, update_mtime=True)
- return CasBasedDirectory(self.cas, digest=digest)
- except (CASCacheError, VirtualDirectoryError) as e:
- raise ArtifactError('Directory not in local cache: {}'.format(e)) from e
-
- # commit():
- #
- # Commit built artifact to cache.
- #
- # Args:
- # element (Element): The Element commit an artifact for
- # content (Directory): The element's content directory
- # keys (list): The cache keys to use
- #
- def commit(self, element, content, keys):
- refs = [element.get_artifact_name(key) for key in keys]
-
- tree = content._get_digest()
-
- for ref in refs:
- self.cas.set_ref(ref, tree)
-
# diff():
#
# Return a list of files that have been added or modified between
@@ -264,10 +271,16 @@ class ArtifactCache(BaseCache):
# subdir (str): A subdirectory to limit the comparison to
#
def diff(self, element, key_a, key_b, *, subdir=None):
- ref_a = element.get_artifact_name(key_a)
- ref_b = element.get_artifact_name(key_b)
+ digest_a = self.get_artifact_proto(element.get_artifact_name(key_a)).files
+ digest_b = self.get_artifact_proto(element.get_artifact_name(key_b)).files
- return self.cas.diff(ref_a, ref_b, subdir=subdir)
+ added = []
+ removed = []
+ modified = []
+
+ self.cas.diff_trees(digest_a, digest_b, added=added, removed=removed, modified=modified)
+
+ return modified, removed, added
# push():
#
@@ -284,8 +297,6 @@ class ArtifactCache(BaseCache):
# (ArtifactError): if there was an error
#
def push(self, element, keys):
- refs = [element.get_artifact_name(key) for key in list(keys)]
-
project = element._get_project()
push_remotes = [r for r in self._remotes[project] if r.spec.push]
@@ -297,7 +308,7 @@ class ArtifactCache(BaseCache):
display_key = element._get_brief_display_key()
element.status("Pushing artifact {} -> {}".format(display_key, remote.spec.url))
- if self.cas.push(refs, remote):
+ if self._push_artifact(element, keys, remote):
element.info("Pushed artifact {} -> {}".format(display_key, remote.spec.url))
pushed = True
else:
@@ -322,16 +333,16 @@ class ArtifactCache(BaseCache):
# (bool): True if pull was successful, False if artifact was not available
#
def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):
- ref = element.get_artifact_name(key)
-
project = element._get_project()
+ pull_buildtrees = "buildtree" not in excluded_subdirs if excluded_subdirs else True
for remote in self._remotes[project]:
+ remote.init()
try:
display_key = element._get_brief_display_key()
element.status("Pulling artifact {} <- {}".format(display_key, remote.spec.url))
- if self.cas.pull(ref, remote, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
+ if self._pull_artifact(element, key, remote, pull_buildtrees=pull_buildtrees):
element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
# no need to pull from additional remotes
return True
@@ -405,7 +416,9 @@ class ArtifactCache(BaseCache):
oldref = element.get_artifact_name(oldkey)
newref = element.get_artifact_name(newkey)
- self.cas.link_ref(oldref, newref)
+ if not os.path.exists(os.path.join(self.artifactdir, newref)):
+ os.link(os.path.join(self.artifactdir, oldref),
+ os.path.join(self.artifactdir, newref))
# get_artifact_logs():
#
@@ -469,3 +482,182 @@ class ArtifactCache(BaseCache):
remote_missing_blobs_set.update(remote_missing_blobs)
return list(remote_missing_blobs_set)
+
+ def get_artifact_proto(self, ref):
+ artifact_path = os.path.join(self.artifactdir, ref)
+ artifact_proto = artifact_pb2.Artifact()
+ with open(artifact_path, 'rb') as f:
+ artifact_proto.ParseFromString(f.read())
+ return artifact_proto
+
+ ################################################
+ # Local Private Methods #
+ ################################################
+
+ # _reachable_directories()
+ #
+ # Returns:
+ # (iter): Iterator over directories digests available from artifacts.
+ #
+ def _reachable_directories(self):
+ for root, _, files in os.walk(self.artifactdir):
+ for artifact_file in files:
+ artifact = artifact_pb2.Artifact()
+ with open(os.path.join(root, artifact_file), 'r+b') as f:
+ artifact.ParseFromString(f.read())
+
+ if str(artifact.files):
+ yield artifact.files
+
+ if str(artifact.buildtree):
+ yield artifact.buildtree
+
+ # _reachable_digests()
+ #
+ # Returns:
+ # (iter): Iterator over single file digests in artifacts
+ #
+ def _reachable_digests(self):
+ for root, _, files in os.walk(self.artifactdir):
+ for artifact_file in files:
+ artifact = artifact_pb2.Artifact()
+ with open(os.path.join(root, artifact_file), 'r+b') as f:
+ artifact.ParseFromString(f.read())
+
+ if str(artifact.public_data):
+ yield artifact.public_data
+
+ for log_file in artifact.logs:
+ yield log_file.digest
+
+ # _push_artifact()
+ #
+ # Pushes relevant directories and then artifact proto to remote.
+ #
+ # Args:
+ # element (Element): element
+ # keys ([str]): keys to push
+ # remote (CASRemote): remote to push to
+ #
+ # Returns:
+ # (bool): whether the push was successful
+ #
+ def _push_artifact(self, element, keys, remote):
+ keys = list(keys)
+ if not keys:
+ keys = [element._get_cache_key()]
+ artifacts = list(map(self.get_artifact_proto, list(map(element.get_artifact_name, keys))))
+ # check the artifacts are the same for each key
+ # unsure how necessary this is
+ artifact = artifacts[0]
+ for check_artifact in artifacts:
+ assert artifact == check_artifact
+
+ # Check whether the artifact is on the server
+ present = False
+ for key in keys:
+ get_artifact = artifact_pb2.GetArtifactRequest()
+ get_artifact.cache_key = element.get_artifact_name(key)
+ try:
+ remote.artifact.GetArtifact(get_artifact)
+ except grpc.RpcError as e:
+ if e.code() != grpc.StatusCode.NOT_FOUND:
+ raise ArtifactError("Error checking artifact cache: {}"
+ .format(e.details()))
+ else:
+ present = True
+ if present:
+ return False
+
+ try:
+ self.cas._send_directory(remote, artifact.files)
+
+ if str(artifact.buildtree):
+ try:
+ self.cas._send_directory(remote, artifact.buildtree)
+ except FileNotFoundError:
+ pass
+
+ digests = []
+ if str(artifact.public_data):
+ digests.append(artifact.public_data)
+
+ for log_file in artifact.logs:
+ digests.append(log_file.digest)
+
+ self.cas.send_blobs(remote, digests)
+
+ except grpc.RpcError as e:
+ if e.code() != grpc.StatusCode.RESOURCE_EXHAUSTED:
+ raise ArtifactError("Failed to push artifact blobs: {}".format(e.details()))
+ return False
+
+ # finally need to send the artifact proto
+ for key in keys:
+ update_artifact = artifact_pb2.UpdateArtifactRequest()
+ update_artifact.cache_key = element.get_artifact_name(key)
+ update_artifact.artifact.CopyFrom(artifact)
+
+ try:
+ remote.artifact.UpdateArtifact(update_artifact)
+ except grpc.RpcError as e:
+ raise ArtifactError("Failed to push artifact: {}".format(e.details()))
+
+ return True
+
+ # _pull_artifact()
+ #
+ # Args:
+ # element (Element): element to pull
+ # key (str): specific key of element to pull
+ # remote (CASRemote): remote to pull from
+ # pull_buildtree (bool): whether to pull buildtrees or not
+ #
+ # Returns:
+ # (bool): whether the pull was successful
+ #
+ def _pull_artifact(self, element, key, remote, pull_buildtrees=False):
+
+ def __pull_digest(digest):
+ self.cas._fetch_directory(remote, digest)
+ required_blobs = self.cas.required_blobs_for_directory(digest)
+ missing_blobs = self.cas.local_missing_blobs(required_blobs)
+ if missing_blobs:
+ self.cas.fetch_blobs(remote, missing_blobs)
+
+ request = artifact_pb2.GetArtifactRequest()
+ request.cache_key = element.get_artifact_name(key=key)
+ try:
+ artifact = remote.artifact.GetArtifact(request)
+ except grpc.RpcError as e:
+ if e.code() != grpc.StatusCode.NOT_FOUND:
+ raise ArtifactError("Failed to pull artifact: {}".format(e.details()))
+ return False
+
+ try:
+ if str(artifact.files):
+ __pull_digest(artifact.files)
+
+ if pull_buildtrees and str(artifact.buildtree):
+ __pull_digest(artifact.buildtree)
+
+ digests = []
+ if str(artifact.public_data):
+ digests.append(artifact.public_data)
+
+ for log_digest in artifact.logs:
+ digests.append(log_digest.digest)
+
+ self.cas.fetch_blobs(remote, digests)
+ except grpc.RpcError as e:
+ if e.code() != grpc.StatusCode.NOT_FOUND:
+ raise ArtifactError("Failed to pull artifact: {}".format(e.details()))
+ return False
+
+ # Write the artifact proto to cache
+ artifact_path = os.path.join(self.artifactdir, request.cache_key)
+ os.makedirs(os.path.dirname(artifact_path), exist_ok=True)
+ with open(artifact_path, 'w+b') as f:
+ f.write(artifact.SerializeToString())
+
+ return True
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 10e8b3e..0bbeedd 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -87,6 +87,9 @@ class CASCache():
os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
os.makedirs(self.tmpdir, exist_ok=True)
+ self.__reachable_directory_callbacks = []
+ self.__reachable_digest_callbacks = []
+
# preflight():
#
# Preflight check.
@@ -578,6 +581,14 @@ class CASCache():
return None
+ # adds callback of iterator over reachable directory digests
+ def add_reachable_directories_callback(self, callback):
+ self.__reachable_directory_callbacks.append(callback)
+
+ # adds callbacks of iterator over reachable file digests
+ def add_reachable_digests_callback(self, callback):
+ self.__reachable_digest_callbacks.append(callback)
+
# prune():
#
# Prune unreachable objects from the repo.
@@ -597,6 +608,16 @@ class CASCache():
tree = self.resolve_ref(ref)
self._reachable_refs_dir(reachable, tree)
+ # check callback directory digests that are reachable
+ for digest_callback in self.__reachable_directory_callbacks:
+ for digest in digest_callback():
+ self._reachable_refs_dir(reachable, digest)
+
+ # check callback file digests that are reachable
+ for digest_callback in self.__reachable_digest_callbacks:
+ for digest in digest_callback():
+ reachable.add(digest.hash)
+
# Prune unreachable objects
for root, _, files in os.walk(os.path.join(self.casdir, 'objects')):
for filename in files:
@@ -1077,8 +1098,8 @@ class CASQuota:
self._message = context.message
- self._ref_callbacks = [] # Call backs to get required refs
- self._remove_callbacks = [] # Call backs to remove refs
+ self._remove_callbacks = [] # Callbacks to remove unrequired refs and their remove method
+ self._list_refs_callbacks = [] # Callbacks to all refs
self._calculate_cache_quota()
@@ -1156,6 +1177,21 @@ class CASQuota:
return False
+ # add_remove_callbacks()
+ #
+ # This adds tuples of iterators over unrequired objects (currently
+ # artifacts and source refs), and a callback to remove them.
+ #
+ # Args:
+ # callback (iter(unrequired), remove): tuple of iterator and remove
+ # method associated.
+ #
+ def add_remove_callbacks(self, list_unrequired, remove_method):
+ self._remove_callbacks.append((list_unrequired, remove_method))
+
+ def add_list_refs_callback(self, list_callback):
+ self._list_refs_callbacks.append(list_callback)
+
################################################
# Local Private Methods #
################################################
@@ -1312,28 +1348,25 @@ class CASQuota:
removed_ref_count = 0
space_saved = 0
- # get required refs
- refs = self.cas.list_refs()
- required_refs = set(
- required
- for callback in self._ref_callbacks
- for required in callback()
- )
+ total_refs = 0
+ for refs in self._list_refs_callbacks:
+ total_refs += len(list(refs()))
# Start off with an announcement with as much info as possible
volume_size, volume_avail = self._get_cache_volume_size()
self._message(Message(
None, MessageType.STATUS, "Starting cache cleanup",
- detail=("Elements required by the current build plan: {}\n" +
+ detail=("Elements required by the current build plan:\n" + "{}\n" +
"User specified quota: {} ({})\n" +
"Cache usage: {}\n" +
"Cache volume: {} total, {} available")
- .format(len(required_refs),
- context.config_cache_quota,
- utils._pretty_size(self._cache_quota, dec_places=2),
- utils._pretty_size(self.get_cache_size(), dec_places=2),
- utils._pretty_size(volume_size, dec_places=2),
- utils._pretty_size(volume_avail, dec_places=2))))
+ .format(
+ total_refs,
+ context.config_cache_quota,
+ utils._pretty_size(self._cache_quota, dec_places=2),
+ utils._pretty_size(self.get_cache_size(), dec_places=2),
+ utils._pretty_size(volume_size, dec_places=2),
+ utils._pretty_size(volume_avail, dec_places=2))))
# Do a real computation of the cache size once, just in case
self.compute_cache_size()
@@ -1341,67 +1374,63 @@ class CASQuota:
self._message(Message(None, MessageType.STATUS,
"Cache usage recomputed: {}".format(usage)))
- while self.get_cache_size() >= self._cache_lower_threshold:
- try:
- to_remove = refs.pop(0)
- except IndexError:
- # If too many artifacts are required, and we therefore
- # can't remove them, we have to abort the build.
- #
- # FIXME: Asking the user what to do may be neater
- #
- default_conf = os.path.join(os.environ['XDG_CONFIG_HOME'],
- 'buildstream.conf')
- detail = ("Aborted after removing {} refs and saving {} disk space.\n"
- "The remaining {} in the cache is required by the {} references in your build plan\n\n"
- "There is not enough space to complete the build.\n"
- "Please increase the cache-quota in {} and/or make more disk space."
- .format(removed_ref_count,
- utils._pretty_size(space_saved, dec_places=2),
- utils._pretty_size(self.get_cache_size(), dec_places=2),
- len(required_refs),
- (context.config_origin or default_conf)))
-
- if self.full():
- raise CASCacheError("Cache too full. Aborting.",
- detail=detail,
- reason="cache-too-full")
- else:
- break
-
- key = to_remove.rpartition('/')[2]
- if key not in required_refs:
-
- # Remove the actual artifact, if it's not required.
- size = 0
- removed_ref = False
- for (pred, remove) in self._remove_callbacks:
- if pred(to_remove):
- size = remove(to_remove)
- removed_ref = True
- break
-
- if not removed_ref:
- continue
-
- removed_ref_count += 1
- space_saved += size
-
- self._message(Message(
- None, MessageType.STATUS,
- "Freed {: <7} {}".format(
- utils._pretty_size(size, dec_places=2),
- to_remove)))
-
- self.set_cache_size(self._cache_size - size)
-
- # User callback
- #
- # Currently this process is fairly slow, but we should
- # think about throttling this progress() callback if this
- # becomes too intense.
- if progress:
- progress()
+ # Collect digests and their remove method
+ all_unrequired_refs = []
+ for (unrequired_refs, remove) in self._remove_callbacks:
+ for (mtime, ref) in unrequired_refs():
+ all_unrequired_refs.append((mtime, ref, remove))
+
+ # Pair refs and their remove method sorted in time order
+ all_unrequired_refs = [(ref, remove) for (_, ref, remove) in sorted(all_unrequired_refs)]
+
+ # Go through unrequired refs and remove them, oldest first
+ made_space = False
+ for (ref, remove) in all_unrequired_refs:
+ size = remove(ref)
+ removed_ref_count += 1
+ space_saved += size
+
+ self._message(Message(
+ None, MessageType.STATUS,
+ "Freed {: <7} {}".format(
+ utils._pretty_size(size, dec_places=2),
+ ref)))
+
+ self.set_cache_size(self._cache_size - size)
+
+ # User callback
+ #
+ # Currently this process is fairly slow, but we should
+ # think about throttling this progress() callback if this
+ # becomes too intense.
+ if progress:
+ progress()
+
+ if self.get_cache_size() < self._cache_lower_threshold:
+ made_space = True
+ break
+
+ if not made_space and self.full():
+ # If too many artifacts are required, and we therefore
+ # can't remove them, we have to abort the build.
+ #
+ # FIXME: Asking the user what to do may be neater
+ #
+ default_conf = os.path.join(os.environ['XDG_CONFIG_HOME'],
+ 'buildstream.conf')
+ detail = ("Aborted after removing {} refs and saving {} disk space.\n"
+ "The remaining {} in the cache is required by the {} references in your build plan\n\n"
+ "There is not enough space to complete the build.\n"
+ "Please increase the cache-quota in {} and/or make more disk space."
+ .format(removed_ref_count,
+ utils._pretty_size(space_saved, dec_places=2),
+ utils._pretty_size(self.get_cache_size(), dec_places=2),
+ total_refs,
+ (context.config_origin or default_conf)))
+
+ raise CASCacheError("Cache too full. Aborting.",
+ detail=detail,
+ reason="cache-too-full")
# Informational message about the side effects of the cleanup
self._message(Message(
@@ -1414,22 +1443,6 @@ class CASQuota:
return self.get_cache_size()
- # add_ref_callbacks()
- #
- # Args:
- # callback (Iterator): function that gives list of required refs
- def add_ref_callbacks(self, callback):
- self._ref_callbacks.append(callback)
-
- # add_remove_callbacks()
- #
- # Args:
- # callback (predicate, callback): The predicate says whether this is the
- # correct type to remove given a ref and the callback does actual
- # removing.
- def add_remove_callbacks(self, callback):
- self._remove_callbacks.append(callback)
-
def _grouper(iterable, n):
while True:
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 93e3f62..151ea63 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -233,7 +233,7 @@ class Context():
self.tmpdir = os.path.join(self.cachedir, 'tmp')
self.casdir = os.path.join(self.cachedir, 'cas')
self.builddir = os.path.join(self.cachedir, 'build')
- self.artifactdir = os.path.join(self.cachedir, 'artifacts')
+ self.artifactdir = os.path.join(self.cachedir, 'artifacts', 'refs')
# Move old artifact cas to cas if it exists and create symlink
old_casdir = os.path.join(self.cachedir, 'artifacts', 'cas')
diff --git a/buildstream/_sourcecache.py b/buildstream/_sourcecache.py
index 36f75d0..8dfa975 100644
--- a/buildstream/_sourcecache.py
+++ b/buildstream/_sourcecache.py
@@ -17,6 +17,8 @@
# Authors:
# Raoul Hidalgo Charman <ra...@codethink.co.uk>
#
+import os
+
from ._cas import CASRemoteSpec
from .storage._casbaseddirectory import CasBasedDirectory
from ._basecache import BaseCache
@@ -53,8 +55,8 @@ class SourceCache(BaseCache):
self._required_sources = set()
- self.casquota.add_ref_callbacks(self.required_sources)
- self.casquota.add_remove_callbacks((lambda x: x.startswith('@sources/'), self.cas.remove))
+ self.casquota.add_remove_callbacks(self.unrequired_sources, self.cas.remove)
+ self.casquota.add_list_refs_callback(self.list_sources)
# mark_required_sources()
#
@@ -81,14 +83,43 @@ class SourceCache(BaseCache):
# required_sources()
#
- # Yields the keys of all sources marked as required
+ # Yields the keys of all sources marked as required by the current build
+ # plan
#
# Returns:
- # iterable (str): iterable over the source keys
+ # iterable (str): iterable over the required source refs
#
def required_sources(self):
for source in self._required_sources:
- yield source._key
+ yield source._get_source_name()
+
+ # unrequired_sources()
+ #
+ # Yields the refs of all sources not required by the current build plan
+ #
+ # Returns:
+ # iter (str): iterable over unrequired source keys
+ #
+ def unrequired_sources(self):
+ required_source_names = set(map(
+ lambda x: x._get_source_name(), self._required_sources))
+ for (mtime, source) in utils._list_directory(
+ os.path.join(self.cas.casdir, 'refs', 'heads'),
+ glob_expr="@sources/*"):
+ if source not in required_source_names:
+ yield (mtime, source)
+
+ # list_sources()
+ #
+ # Get list of all sources in the `cas/refs/heads/@sources/` folder
+ #
+ # Returns:
+ # ([str]): iterable over all source refs
+ #
+ def list_sources(self):
+ return [ref for _, ref in utils._list_directory(
+ os.path.join(self.cas.casdir, 'refs', 'heads'),
+ glob_expr="@sources/*")]
# contains()
#
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index d4f26e4..2343c55 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -32,7 +32,7 @@ from contextlib import contextmanager, suppress
from fnmatch import fnmatch
from ._artifactelement import verify_artifact_ref
-from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, CASCacheError
+from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
from ._message import Message, MessageType
from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, \
SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue
@@ -587,7 +587,7 @@ class Stream():
for ref in remove_refs:
try:
self._artifacts.remove(ref, defer_prune=True)
- except CASCacheError as e:
+ except ArtifactError as e:
self._message(MessageType.WARN, str(e))
continue
diff --git a/buildstream/element.py b/buildstream/element.py
index 05884c0..7b7d2f8 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -752,12 +752,12 @@ class Element(Plugin):
if dep.name in old_dep_keys:
key_new = dep._get_cache_key()
- key_old = _yaml.node_get(old_dep_keys, str, dep.name)
+ key_old = old_dep_keys[dep.name]
# We only need to worry about modified and added
# files, since removed files will be picked up by
# build systems anyway.
- to_update, _, added = self.__artifacts.diff(dep, key_old, key_new, subdir='files')
+ to_update, _, added = self.__artifacts.diff(dep, key_old, key_new)
workspace.add_running_files(dep.name, to_update + added)
to_update.extend(workspace.running_files[dep.name])
@@ -1855,9 +1855,9 @@ class Element(Plugin):
# in user context, as to complete a partial artifact
subdir, _ = self.__pull_directories()
- if self.__strong_cached and subdir:
+ if self.__strong_cached and subdir == 'buildtree':
# If we've specified a subdir, check if the subdir is cached locally
- if self.__artifacts.contains_subdir_artifact(self, self.__strict_cache_key, subdir):
+ if self.__artifacts.contains_buildtree(self, self.__strict_cache_key):
return False
elif self.__strong_cached:
return False
diff --git a/buildstream/testing/runcli.py b/buildstream/testing/runcli.py
index 72bdce0..1b57d71 100644
--- a/buildstream/testing/runcli.py
+++ b/buildstream/testing/runcli.py
@@ -56,6 +56,7 @@ from buildstream._cas import CASCache
# Special private exception accessor, for test case purposes
from buildstream._exceptions import BstError, get_last_exception, get_last_task_error
+from buildstream._protos.buildstream.v2 import artifact_pb2
# Wrapper for the click.testing result
@@ -636,7 +637,7 @@ class TestArtifact():
#
def remove_artifact_from_cache(self, cache_dir, element_name):
- cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
+ cache_dir = os.path.join(cache_dir, 'artifacts', 'refs')
cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
shutil.rmtree(cache_dir)
@@ -655,13 +656,13 @@ class TestArtifact():
#
def is_cached(self, cache_dir, element, element_key):
- cas = CASCache(str(cache_dir))
+ # cas = CASCache(str(cache_dir))
artifact_ref = element.get_artifact_name(element_key)
- return cas.contains(artifact_ref)
+ return os.path.exists(os.path.join(cache_dir, 'artifacts', 'refs', artifact_ref))
# get_digest():
#
- # Get the digest for a given element's artifact
+ # Get the digest for a given element's artifact files
#
# Args:
# cache_dir (str): Specific cache dir to check
@@ -673,10 +674,12 @@ class TestArtifact():
#
def get_digest(self, cache_dir, element, element_key):
- cas = CASCache(str(cache_dir))
artifact_ref = element.get_artifact_name(element_key)
- digest = cas.resolve_ref(artifact_ref)
- return digest
+ artifact_dir = os.path.join(cache_dir, 'artifacts', 'refs')
+ artifact_proto = artifact_pb2.Artifact()
+ with open(os.path.join(artifact_dir, artifact_ref), 'rb') as f:
+ artifact_proto.ParseFromString(f.read())
+ return artifact_proto.files
# extract_buildtree():
#
@@ -691,9 +694,19 @@ class TestArtifact():
# (str): path to extracted buildtree directory, does not guarantee
# existence.
@contextmanager
- def extract_buildtree(self, tmpdir, digest):
- with self._extract_subdirectory(tmpdir, digest, 'buildtree') as extract:
- yield extract
+ def extract_buildtree(self, cache_dir, tmpdir, ref):
+ artifact = artifact_pb2.Artifact()
+ try:
+ with open(os.path.join(cache_dir, 'artifacts', 'refs', ref), 'rb') as f:
+ artifact.ParseFromString(f.read())
+ except FileNotFoundError:
+ yield None
+ else:
+ if str(artifact.buildtree):
+ with self._extract_subdirectory(tmpdir, artifact.buildtree) as f:
+ yield f
+ else:
+ yield None
# _extract_subdirectory():
#
@@ -709,12 +722,12 @@ class TestArtifact():
# (str): path to extracted subdir directory, does not guarantee
# existence.
@contextmanager
- def _extract_subdirectory(self, tmpdir, digest, subdir):
+ def _extract_subdirectory(self, tmpdir, digest):
with tempfile.TemporaryDirectory() as extractdir:
try:
cas = CASCache(str(tmpdir))
cas.checkout(extractdir, digest)
- yield os.path.join(extractdir, subdir)
+ yield extractdir
except FileNotFoundError:
yield None
diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py
index 1eb67b6..94aaccd 100644
--- a/tests/artifactcache/junctions.py
+++ b/tests/artifactcache/junctions.py
@@ -70,6 +70,8 @@ def test_push_pull(cli, tmpdir, datafiles):
#
cas = os.path.join(cli.directory, 'cas')
shutil.rmtree(cas)
+ artifact_dir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifact_dir)
# Assert that nothing is cached locally anymore
state = cli.get_element_state(project, 'target.bst')
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index 40fed76..c29afd5 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -259,7 +259,8 @@ def test_pull_tree(cli, tmpdir, datafiles):
utils._kill_process_tree(process.pid)
raise
- assert directory_hash and directory_size
+ # Directory size now zero with AaaP and stack element commit #1cbc5e63dc
+ assert directory_hash and not directory_size
directory_digest = remote_execution_pb2.Digest(hash=directory_hash,
size_bytes=directory_size)
diff --git a/tests/frontend/artifact.py b/tests/frontend/artifact.py
index 10cb4f5..6ca5621 100644
--- a/tests/frontend/artifact.py
+++ b/tests/frontend/artifact.py
@@ -92,9 +92,11 @@ def test_artifact_delete_artifact(cli, tmpdir, datafiles):
element = 'target.bst'
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'artifacts')
+ local_cache = os.path.join(str(tmpdir), 'cache')
cli.configure({'cachedir': local_cache})
+ print("local cache: {}".format(local_cache))
+
# First build an element so that we can find its artifact
result = cli.run(project=project, args=['build', element])
result.assert_success()
@@ -104,7 +106,7 @@ def test_artifact_delete_artifact(cli, tmpdir, datafiles):
artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
- assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+ assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact))
# Delete the artifact
result = cli.run(project=project, args=['artifact', 'delete', artifact])
@@ -122,7 +124,7 @@ def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
dep = 'compose-all.bst'
# Configure a local cache
- local_cache = os.path.join(str(tmpdir), 'artifacts')
+ local_cache = os.path.join(str(tmpdir), 'cache')
cli.configure({'cachedir': local_cache})
# First build an element so that we can find its artifact
@@ -138,14 +140,14 @@ def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
# Explicitly check that the ARTIFACT exists in the cache
- assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+ assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact))
# Delete the artifact
result = cli.run(project=project, args=['artifact', 'delete', artifact, dep])
result.assert_success()
# Check that the ARTIFACT is no longer in the cache
- assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
+ assert not os.path.exists(os.path.join(local_cache, 'artifacts', artifact))
# Check that the dependency ELEMENT is no longer cached
assert cli.get_element_state(project, dep) != 'cached'
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index cc62afe..4034de8 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -68,8 +68,10 @@ def test_push_pull_all(cli, tmpdir, datafiles):
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- cas = os.path.join(cli.directory, 'cas')
- shutil.rmtree(cas)
+ casdir = os.path.join(cli.directory, 'cas')
+ shutil.rmtree(casdir)
+ artifactdir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
states = cli.get_element_states(project, all_elements)
@@ -118,8 +120,10 @@ def test_push_pull_default_targets(cli, tmpdir, datafiles):
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- artifacts = os.path.join(cli.directory, 'cas')
- shutil.rmtree(artifacts)
+ casdir = os.path.join(cli.directory, 'cas')
+ shutil.rmtree(casdir)
+ artifactdir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
states = cli.get_element_states(project, all_elements)
@@ -160,8 +164,10 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles):
assert_shared(cli, share2, project, 'target.bst')
# Delete the user's local artifact cache.
- cas = os.path.join(cli.directory, 'cas')
- shutil.rmtree(cas)
+ casdir = os.path.join(cli.directory, 'cas')
+ shutil.rmtree(casdir)
+ artifactdir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifactdir)
# Assert that the element is not cached anymore.
assert cli.get_element_state(project, 'target.bst') != 'cached'
@@ -214,8 +220,10 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles):
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the good_share.
#
- cas = os.path.join(cli.directory, 'cas')
- shutil.rmtree(cas)
+ casdir = os.path.join(cli.directory, 'cas')
+ shutil.rmtree(casdir)
+ artifactdir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifactdir)
result = cli.run(project=project, args=['artifact', 'pull', 'target.bst', '--remote',
good_share.repo])
@@ -245,7 +253,7 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles):
result.assert_success()
assert cli.get_element_state(project, 'target.bst') == 'cached'
- # Assert that everything is now cached in the remote.
+ # Assert that everything is now cached in the reote.
all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst']
for element_name in all_elements:
assert_shared(cli, share, project, element_name)
@@ -253,8 +261,10 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles):
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- cas = os.path.join(cli.directory, 'cas')
- shutil.rmtree(cas)
+ casdir = os.path.join(cli.directory, 'cas')
+ shutil.rmtree(casdir)
+ artifactdir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
@@ -303,8 +313,10 @@ def test_push_pull_track_non_strict(cli, tmpdir, datafiles):
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- cas = os.path.join(cli.directory, 'cas')
- shutil.rmtree(cas)
+ casdir = os.path.join(cli.directory, 'cas')
+ shutil.rmtree(casdir)
+ artifactdir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
@@ -341,6 +353,8 @@ def test_push_pull_cross_junction(cli, tmpdir, datafiles):
cache_dir = os.path.join(project, 'cache', 'cas')
shutil.rmtree(cache_dir)
+ artifact_dir = os.path.join(project, 'cache', 'artifacts')
+ shutil.rmtree(artifact_dir)
assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable'
@@ -374,8 +388,10 @@ def test_pull_missing_blob(cli, tmpdir, datafiles):
# Now we've pushed, delete the user's local artifact cache
# directory and try to redownload it from the share
#
- cas = os.path.join(cli.directory, 'cas')
- shutil.rmtree(cas)
+ casdir = os.path.join(cli.directory, 'cas')
+ shutil.rmtree(casdir)
+ artifactdir = os.path.join(cli.directory, 'artifacts')
+ shutil.rmtree(artifactdir)
# Assert that nothing is cached locally anymore
for element_name in all_elements:
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 67c53f2..2ebcf98 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -352,6 +352,10 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
assert_shared(cli, share, project, 'element1.bst')
assert_shared(cli, share, project, 'element2.bst')
+ print("share repo: {}".format(share.directory))
+ for x, y, z in os.walk(share.directory):
+ print("{} {} {}".format(x, y, z))
+
# Remove element1 from the local cache
cli.remove_artifact_from_cache(project, 'element1.bst')
assert cli.get_element_state(project, 'element1.bst') != 'cached'
@@ -375,6 +379,10 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
assert cli.get_element_state(project, 'element3.bst') == 'cached'
assert_shared(cli, share, project, 'element3.bst')
+ print("share repo: {}".format(share.directory))
+ for x, y, z in os.walk(share.directory):
+ print("{} {} {}".format(x, y, z))
+
# Ensure that element2 was deleted from the share and element1 remains
assert_not_shared(cli, share, project, 'element2.bst')
assert_shared(cli, share, project, 'element1.bst')
diff --git a/tests/frontend/remote-caches.py b/tests/frontend/remote-caches.py
index e3f10e6..089cf96 100644
--- a/tests/frontend/remote-caches.py
+++ b/tests/frontend/remote-caches.py
@@ -80,8 +80,9 @@ def test_source_artifact_caches(cli, tmpdir, datafiles):
# remove the artifact from the repo and check it pulls sources, builds
# and then pushes the artifacts
shutil.rmtree(os.path.join(cachedir, 'cas'))
- print(os.listdir(os.path.join(share.repodir, 'cas', 'refs', 'heads')))
- shutil.rmtree(os.path.join(share.repodir, 'cas', 'refs', 'heads', 'test'))
+ shutil.rmtree(os.path.join(cachedir, 'artifacts'))
+ print(os.listdir(os.path.join(share.repodir, 'artifacts', 'refs')))
+ shutil.rmtree(os.path.join(share.repodir, 'artifacts', 'refs', 'test'))
res = cli.run(project=project_dir, args=['build', 'repo.bst'])
res.assert_success()
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index a5e1f4d..d36785a 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -27,6 +27,7 @@ import tempfile
from buildstream import utils
from buildstream.testing import cli_integration as cli
from tests.testutils import create_artifact_share
+from tests.testutils.element_name import element_ref_name
from tests.testutils.site import HAVE_SANDBOX
from buildstream._cas import CASCache
@@ -48,6 +49,7 @@ DATA_DIR = os.path.join(
def test_cache_buildtrees(cli, tmpdir, datafiles):
project = str(datafiles)
element_name = 'autotools/amhello.bst'
+ cwd = str(tmpdir)
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
@@ -69,19 +71,23 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
# The buildtree dir should not exist, as we set the config to not cache buildtrees.
cache_key = cli.get_element_key(project, element_name)
- elementdigest = share1.has_artifact('test', element_name, cache_key)
- with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
- assert not os.path.isdir(buildtreedir)
+ artifact_ref = "{}/{}/{}".format('test', element_ref_name(element_name), cache_key)
+
+ assert share1.has_artifact('test', element_name, cache_key)
+ with cli.artifact.extract_buildtree(cwd, cwd, artifact_ref) as buildtreedir:
+ assert not buildtreedir
# Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
# that is was cached in share1 as expected without a buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
assert cli.get_element_state(project, element_name) != 'cached'
result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
- assert not os.path.isdir(buildtreedir)
+ with cli.artifact.extract_buildtree(cwd, cwd, artifact_ref) as buildtreedir:
+ assert not buildtreedir
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
# Assert that the default behaviour of pull to not include buildtrees on the artifact
# in share1 which was purposely cached with an empty one behaves as expected. As such the
@@ -89,9 +95,10 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
# leading to no buildtreedir being extracted
result = cli.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
- assert not os.path.isdir(buildtreedir)
+ with cli.artifact.extract_buildtree(cwd, cwd, artifact_ref) as buildtreedir:
+ assert not buildtreedir
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
# Repeat building the artifacts, this time with cache-buildtrees set to
# 'always' via the cli, as such the buildtree dir should not be empty
@@ -105,21 +112,22 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
# Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
- elementdigest = share2.has_artifact('test', element_name, cache_key)
- with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(cwd, cwd, artifact_ref) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
# Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
# that it was cached in share2 as expected with a populated buildtree dir
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
assert cli.get_element_state(project, element_name) != 'cached'
result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(cwd, cwd, artifact_ref) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
# Clarify that the user config option for cache-buildtrees works as the cli
# main option does. Point to share3 which does not have the artifacts cached to force
@@ -133,7 +141,6 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
assert result.exit_code == 0
assert cli.get_element_state(project, element_name) == 'cached'
cache_key = cli.get_element_key(project, element_name)
- elementdigest = share3.has_artifact('test', element_name, cache_key)
- with cli.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
+ with cli.artifact.extract_buildtree(cwd, cwd, artifact_ref) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert os.listdir(buildtreedir)
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index 91acff4..8c34c5a 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -6,6 +6,7 @@ import tempfile
from tests.testutils import create_artifact_share
from tests.testutils.site import HAVE_SANDBOX
+from tests.testutils.element_name import element_ref_name
from buildstream import utils
from buildstream.testing import cli, cli_integration as cli2
@@ -40,6 +41,7 @@ def default_state(cli, tmpdir, share):
def test_pullbuildtrees(cli2, tmpdir, datafiles):
project = str(datafiles)
element_name = 'autotools/amhello.bst'
+ cwd = str(tmpdir)
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
@@ -58,6 +60,10 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
assert share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
default_state(cli2, tmpdir, share1)
+ element_ref = "test/{}/{}".format(
+ element_ref_name(element_name),
+ cli2.get_element_key(project, element_name))
+
# Pull artifact with default config, assert that pulling again
# doesn't create a pull job, then assert with buildtrees user
# config set creates a pull job.
@@ -75,12 +81,11 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
# Also assert that the buildtree is added to the local CAS.
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- elementdigest = share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
- with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(cwd, cwd, element_ref) as buildtreedir:
assert not buildtreedir
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
- with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(cwd, cwd, element_ref) as buildtreedir:
assert os.path.isdir(buildtreedir)
default_state(cli2, tmpdir, share1)
@@ -139,7 +144,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name not in result.get_pulled_elements()
- with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(cwd, cwd, element_ref) as buildtreedir:
assert not buildtreedir
assert element_name not in result.get_pushed_elements()
assert not share3.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
@@ -152,7 +157,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name in result.get_pulled_elements()
- with cli2.artifact.extract_buildtree(tmpdir, elementdigest) as buildtreedir:
+ with cli2.artifact.extract_buildtree(cwd, cwd, element_ref) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert element_name in result.get_pushed_elements()
assert share3.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 3d59c78..2bdb644 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -196,6 +196,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
# Discard the cache
shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
+ shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts')))
assert cli.get_element_state(project, element_name) != 'cached'
# Pull from cache, ensuring cli options is set to pull the buildtree
@@ -229,6 +230,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
# Discard the cache
shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
+ shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts')))
assert cli.get_element_state(project, element_name) != 'cached'
# Pull from cache, but do not include buildtrees.
@@ -269,6 +271,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
assert 'Attempting to fetch missing artifact buildtree' in res.stderr
assert 'Hi' in res.output
shutil.rmtree(os.path.join(os.path.join(str(tmpdir), 'cache', 'cas')))
+ shutil.rmtree(os.path.join(os.path.join(str(tmpdir), 'cache', 'artifacts')))
assert cli.get_element_state(project, element_name) != 'cached'
# Check it's not loading the shell at all with always set for the buildtree, when the
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index bc025cb..899e162 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -95,6 +95,7 @@ def test_source_fetch(cli, tmpdir, datafiles):
os.path.join(str(tmpdir), 'cache', 'cas'),
os.path.join(str(tmpdir), 'sourceshare', 'repo'))
shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'sources'))
+ shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'artifacts'))
digest = share.cas.resolve_ref(source._get_source_name())
assert share.has_object(digest)
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 80959e9..d4025d2 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -10,6 +10,7 @@ from buildstream._cas import CASCache
from buildstream._cas.casserver import create_server
from buildstream._exceptions import CASError
from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildstream._protos.buildstream.v2 import artifact_pb2
from tests.testutils.element_name import element_ref_name
@@ -44,6 +45,8 @@ class ArtifactShare():
#
self.repodir = os.path.join(self.directory, 'repo')
os.makedirs(self.repodir)
+ self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
+ os.makedirs(self.artifactdir)
self.cas = CASCache(self.repodir)
@@ -140,17 +143,44 @@ class ArtifactShare():
element_name = element_ref_name(element_name)
artifact_key = '{0}/{1}/{2}'.format(project_name, element_name, cache_key)
+ artifact_proto = artifact_pb2.Artifact()
+ artifact_path = os.path.join(self.artifactdir, artifact_key)
+
try:
- tree = self.cas.resolve_ref(artifact_key)
- reachable = set()
- try:
- self.cas._reachable_refs_dir(reachable, tree, update_mtime=False, check_exists=True)
- except FileNotFoundError:
- return None
- return tree
+ with open(artifact_path, 'rb') as f:
+ artifact_proto.ParseFromString(f.read())
+ except FileNotFoundError:
+ return None
+
+ reachable = set()
+
+ def reachable_dir(digest):
+ self.cas._reachable_refs_dir(
+ reachable, digest, update_mtime=False, check_exists=True)
+
+ try:
+ if str(artifact_proto.files):
+ reachable_dir(artifact_proto.files)
+
+ if str(artifact_proto.buildtree):
+ reachable_dir(artifact_proto.buildtree)
+
+ if str(artifact_proto.public_data):
+ if not os.path.exists(self.cas.objpath(artifact_proto.public_data)):
+ return None
+
+ for log_file in artifact_proto.logs:
+ if not os.path.exists(self.cas.objpath(log_file.digest)):
+ return None
+
+ return artifact_proto.files
+
except CASError:
return None
+ except FileNotFoundError:
+ return None
+
# close():
#
# Remove the artifact share.