You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:04:32 UTC
[buildstream] 04/06: tmpdir: add tmpdir to context for CASRemote
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch raoul/802-refactor-artifactcache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 9944dddbb7b2f93ec9af4442bf825c6723ddb761
Author: Raoul Hidalgo Charman <ra...@codethink.co.uk>
AuthorDate: Fri Dec 14 11:20:40 2018 +0000
tmpdir: add tmpdir to context for CASRemote
As CASRemote is used in other places such as SandboxRemote it makes sense for
there to be a tmpdir it uses. This dir is passed to CASRemote when initialized.
This is currently in the artifactdir, but should be moved when artifactdir and
builddir options are deprecated in favour of one top level directory containing
the two.
Part of #802
---
buildstream/_artifactcache.py | 4 ++--
buildstream/_cas/casremote.py | 11 ++++++++---
buildstream/_context.py | 7 +++++--
buildstream/sandbox/_sandboxremote.py | 7 ++++---
tests/testutils/runcli.py | 2 +-
5 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index cdbf2d9..dd5b4b5 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -374,7 +374,7 @@ class ArtifactCache():
q = multiprocessing.Queue()
for remote_spec in remote_specs:
- error = CASRemote.check_remote(remote_spec, q)
+ error = CASRemote.check_remote(remote_spec, self.context.tmpdir, q)
if error and on_failure:
on_failure(remote_spec.url, error)
@@ -385,7 +385,7 @@ class ArtifactCache():
if remote_spec.push:
self._has_push_remotes = True
- remotes[remote_spec.url] = CASRemote(remote_spec)
+ remotes[remote_spec.url] = CASRemote(remote_spec, self.context.tmpdir)
for project in self.context.get_projects():
remote_specs = self.global_remote_specs
diff --git a/buildstream/_cas/casremote.py b/buildstream/_cas/casremote.py
index 56ba4c5..f7af253 100644
--- a/buildstream/_cas/casremote.py
+++ b/buildstream/_cas/casremote.py
@@ -79,7 +79,7 @@ class BlobNotFound(CASRemoteError):
# Represents a single remote CAS cache.
#
class CASRemote():
- def __init__(self, spec):
+ def __init__(self, spec, tmpdir):
self.spec = spec
self._initialized = False
self.channel = None
@@ -91,6 +91,11 @@ class CASRemote():
self.capabilities = None
self.max_batch_total_size_bytes = None
+ # Need str because python 3.5 and lower doesn't deal with path like
+ # objects here.
+ self.tmpdir = str(tmpdir)
+ os.makedirs(self.tmpdir, exist_ok=True)
+
def init(self):
if not self._initialized:
url = urlparse(self.spec.url)
@@ -172,11 +177,11 @@ class CASRemote():
# in the main BuildStream process
# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
@classmethod
- def check_remote(cls, remote_spec, q):
+ def check_remote(cls, remote_spec, tmpdir, q):
def __check_remote():
try:
- remote = cls(remote_spec)
+ remote = cls(remote_spec, tmpdir)
remote.init()
request = buildstream_pb2.StatusRequest()
diff --git a/buildstream/_context.py b/buildstream/_context.py
index 324b455..f81a6b0 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -191,10 +191,13 @@ class Context():
_yaml.node_validate(defaults, [
'sourcedir', 'builddir', 'artifactdir', 'logdir',
'scheduler', 'artifacts', 'logging', 'projects',
- 'cache', 'prompt', 'workspacedir', 'remote-execution'
+ 'cache', 'prompt', 'workspacedir', 'remote-execution',
])
- for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
+ defaults['tmpdir'] = os.path.join(defaults['artifactdir'], 'tmp')
+
+ for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir',
+ 'tmpdir', 'workspacedir']:
# Allow the ~ tilde expansion and any environment variables in
# path specification in the config files.
#
diff --git a/buildstream/sandbox/_sandboxremote.py b/buildstream/sandbox/_sandboxremote.py
index 6a1f6f2..8c21041 100644
--- a/buildstream/sandbox/_sandboxremote.py
+++ b/buildstream/sandbox/_sandboxremote.py
@@ -279,7 +279,7 @@ class SandboxRemote(Sandbox):
context = self._get_context()
cascache = context.get_cascache()
- casremote = CASRemote(self.storage_remote_spec)
+ casremote = CASRemote(self.storage_remote_spec, context.tmpdir)
# Now do a pull to ensure we have the necessary parts.
dir_digest = cascache.pull_tree(casremote, tree_digest)
@@ -306,8 +306,9 @@ class SandboxRemote(Sandbox):
def _run(self, command, flags, *, cwd, env):
# set up virtual dircetory
+ context = self._get_context()
upload_vdir = self.get_virtual_directory()
- cascache = self._get_context().get_cascache()
+ cascache = context.get_cascache()
if isinstance(upload_vdir, FileBasedDirectory):
# Make a new temporary directory to put source in
upload_vdir = CasBasedDirectory(cascache, ref=None)
@@ -340,7 +341,7 @@ class SandboxRemote(Sandbox):
action_result = self._check_action_cache(action_digest)
if not action_result:
- casremote = CASRemote(self.storage_remote_spec)
+ casremote = CASRemote(self.storage_remote_spec, context.tmpdir)
# Now, push that key (without necessarily needing a ref) to the remote.
try:
diff --git a/tests/testutils/runcli.py b/tests/testutils/runcli.py
index 0c8e962..67a874e 100644
--- a/tests/testutils/runcli.py
+++ b/tests/testutils/runcli.py
@@ -526,7 +526,7 @@ def cli_integration(tmpdir, integration_cache):
# to avoid downloading the huge base-sdk repeatedly
fixture.configure({
'sourcedir': os.path.join(integration_cache, 'sources'),
- 'artifactdir': os.path.join(integration_cache, 'artifacts')
+ 'artifactdir': os.path.join(integration_cache, 'artifacts'),
})
return fixture