[buildstream] 07/10: cascache.py: Add allow_partial parameter to fetch_blobs()

[no...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

not-in-ldap pushed a commit to branch juerg/public-data
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit 1298d0c485ddc0ccd9aa91b79b70199056117b38
Author: Jürg Billeter <j...@bitron.ch>
AuthorDate: Mon Mar 30 12:14:01 2020 +0200

    cascache.py: Add allow_partial parameter to fetch_blobs()
    
    This fixes handling of missing blobs in `ArtifactCache.pull()`.
---
 src/buildstream/_artifactcache.py         | 2 +-
 src/buildstream/_cas/cascache.py          | 9 ++++++---
 src/buildstream/_sourcecache.py           | 6 +-----
 src/buildstream/sandbox/_sandboxremote.py | 7 +------
 4 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py
index 9b800ac..9cebeb1 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -405,7 +405,7 @@ class ArtifactCache(BaseCache):
             remote.init()
 
             # fetch_blobs() will return the blobs that are still missing
-            missing_blobs = self.cas.fetch_blobs(remote, missing_blobs)
+            missing_blobs = self.cas.fetch_blobs(remote, missing_blobs, allow_partial=True)
 
         if missing_blobs:
             raise ArtifactError("Blobs not found on configured artifact servers")
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 03be75c..61a1a8f 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -647,16 +647,19 @@ class CASCache:
 
     # fetch_blobs():
     #
-    # Fetch blobs from remote CAS. Returns missing blobs that could not be fetched.
+    # Fetch blobs from remote CAS. Optionally returns missing blobs that could
+    # not be fetched.
     #
     # Args:
     #    remote (CASRemote): The remote repository to fetch from
     #    digests (list): The Digests of blobs to fetch
+    #    allow_partial (bool): True to return missing blobs, False to raise a
+    #                          BlobNotFound error if a blob is missing
     #
     # Returns: The Digests of the blobs that were not available on the remote CAS
     #
-    def fetch_blobs(self, remote, digests):
-        missing_blobs = []
+    def fetch_blobs(self, remote, digests, *, allow_partial=False):
+        missing_blobs = [] if allow_partial else None
 
         remote.init()
 
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py
index e485fbd..4533a25 100644
--- a/src/buildstream/_sourcecache.py
+++ b/src/buildstream/_sourcecache.py
@@ -242,11 +242,7 @@ class SourceCache(BaseCache):
                 self.cas._fetch_directory(remote, source_proto.files)
                 required_blobs = self.cas.required_blobs_for_directory(source_proto.files)
                 missing_blobs = self.cas.local_missing_blobs(required_blobs)
-                missing_blobs = self.cas.fetch_blobs(remote, missing_blobs)
-
-                if missing_blobs:
-                    source.info("Remote cas ({}) does not have source {} cached".format(remote, display_key))
-                    continue
+                self.cas.fetch_blobs(remote, missing_blobs)
 
                 source.info("Pulled source {} <- {}".format(display_key, remote))
                 return True
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index 3dcbb2c..5b03852 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -291,12 +291,7 @@ class SandboxRemote(SandboxREAPI):
                     blobs_to_fetch = artifactcache.find_missing_blobs(project, local_missing_blobs)
 
                 with CASRemote(self.storage_remote_spec, cascache) as casremote:
-                    remote_missing_blobs = cascache.fetch_blobs(casremote, blobs_to_fetch)
-
-                if remote_missing_blobs:
-                    raise SandboxError(
-                        "{} output files are missing on the CAS server".format(len(remote_missing_blobs))
-                    )
+                    cascache.fetch_blobs(casremote, blobs_to_fetch)
 
     def _execute_action(self, action, flags):
         stdout, stderr = self._get_output()