You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/02/04 08:00:59 UTC
[buildstream] branch chiaratolentino/refactor-setup-remotes created
(now 79fbf28)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch chiaratolentino/refactor-setup-remotes
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 79fbf28 _stream.py: Fix type annotation for artifact_url and source_url
This branch includes the following new commits:
new 0f49b11 _basecache.py: Remove redundant use_config argument in setup_remotes
new 79fbf28 _stream.py: Fix type annotation for artifact_url and source_url
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/02: _stream.py: Fix type annotation for
artifact_url and source_url
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch chiaratolentino/refactor-setup-remotes
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 79fbf28213f78f3e8ea76479bc423b870cfb9b12
Author: ctolentino8 <ct...@bloomberg.net>
AuthorDate: Thu Nov 28 16:56:20 2019 +0000
_stream.py: Fix type annotation for artifact_url and source_url
---
src/buildstream/_stream.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 176662d..ca7e0b0 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -29,7 +29,7 @@ import tempfile
from contextlib import contextmanager, suppress
from fnmatch import fnmatch
from collections import deque
-from typing import List, Tuple
+from typing import List, Tuple, Optional
from ._artifactelement import verify_artifact_ref, ArtifactElement
from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
@@ -1137,7 +1137,7 @@ class Stream:
# source_url - The url of the source server to connect to.
#
def __connect_remotes(
- self, artifact_url: str, source_url: str
+ self, artifact_url: Optional[str], source_url: Optional[str]
):
# ArtifactCache.setup_remotes expects all projects to be fully loaded
for project in self._context.get_projects():
[buildstream] 01/02: _basecache.py: Remove redundant use_config
argument in setup_remotes
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch chiaratolentino/refactor-setup-remotes
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0f49b11944bcb58a9e2ac5dde2b1e5f329e3d22a
Author: ctolentino8 <ct...@bloomberg.net>
AuthorDate: Thu Nov 28 16:27:46 2019 +0000
_basecache.py: Remove redundant use_config argument in setup_remotes
---
src/buildstream/_basecache.py | 9 ++++----
src/buildstream/_frontend/cli.py | 2 +-
src/buildstream/_stream.py | 42 ++++++-------------------------------
tests/artifactcache/capabilities.py | 2 +-
tests/artifactcache/pull.py | 4 ++--
tests/artifactcache/push.py | 4 ++--
tests/sourcecache/capabilities.py | 2 +-
7 files changed, 18 insertions(+), 47 deletions(-)
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index 516119c..8a0d7b9 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -155,13 +155,12 @@ class BaseCache:
# Sets up which remotes to use
#
# Args:
- # use_config (bool): Whether to use project configuration
# remote_url (str): Remote cache URL
#
# This requires that all of the projects which are to be processed in the session
# have already been loaded and are observable in the Context.
#
- def setup_remotes(self, *, use_config=False, remote_url=None):
+ def setup_remotes(self, *, remote_url=None):
# Ensure we do not double-initialise since this can be expensive
if self._remotes_setup:
@@ -172,15 +171,17 @@ class BaseCache:
# Initialize remote caches. We allow the commandline to override
# the user config in some cases (for example `bst artifact push --remote=...`).
has_remote_caches = False
- if remote_url:
+
+ if remote_url is not None:
self._set_remotes([RemoteSpec(remote_url, push=True)])
has_remote_caches = True
- if use_config:
+ else:
for project in self.context.get_projects():
caches = self._configured_remote_cache_specs(self.context, project)
if caches: # caches is a list of RemoteSpec instances
self._set_remotes(caches, project=project)
has_remote_caches = True
+
if has_remote_caches:
self._initialize_remotes()
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 051f5d7..01d067c 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -655,7 +655,7 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c
if not element:
raise AppError('Missing argument "ELEMENT".')
- elements = app.stream.load_selection((element,), selection=selection, use_artifact_config=True)
+ elements = app.stream.load_selection((element,), selection=selection)
# last one will be the element we want to stage, previous ones are
# elements to try and pull
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 4b84093..176662d 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -145,7 +145,6 @@ class Stream:
# targets (list of str): Targets to pull
# selection (PipelineSelection): The selection mode for the specified targets
# except_targets (list of str): Specified targets to except from fetching
- # use_artifact_config (bool): If artifact remote configs should be loaded
#
# Returns:
# (list of Element): The selected elements
@@ -155,7 +154,6 @@ class Stream:
*,
selection=PipelineSelection.NONE,
except_targets=(),
- use_artifact_config=False,
load_refs=False
):
with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)):
@@ -163,7 +161,6 @@ class Stream:
targets,
selection=selection,
except_targets=except_targets,
- use_artifact_config=use_artifact_config,
load_refs=load_refs,
)
@@ -276,17 +273,11 @@ class Stream:
#
def build(self, targets, *, selection=PipelineSelection.PLAN, ignore_junction_targets=False, remote=None):
- use_config = True
- if remote:
- use_config = False
-
elements = self._load(
targets,
selection=selection,
ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
artifact_remote_url=remote,
- use_source_config=True,
dynamic_plan=True,
)
@@ -337,15 +328,10 @@ class Stream:
#
def fetch(self, targets, *, selection=PipelineSelection.PLAN, except_targets=None, remote=None):
- use_source_config = True
- if remote:
- use_source_config = False
-
elements = self._load(
targets,
selection=selection,
except_targets=except_targets,
- use_source_config=use_source_config,
source_remote_url=remote,
)
@@ -398,15 +384,10 @@ class Stream:
#
def pull(self, targets, *, selection=PipelineSelection.NONE, ignore_junction_targets=False, remote=None):
- use_config = True
- if remote:
- use_config = False
-
elements = self._load(
targets,
selection=selection,
ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
artifact_remote_url=remote,
load_refs=True,
)
@@ -439,15 +420,10 @@ class Stream:
#
def push(self, targets, *, selection=PipelineSelection.NONE, ignore_junction_targets=False, remote=None):
- use_config = True
- if remote:
- use_config = False
-
elements = self._load(
targets,
selection=selection,
ignore_junction_targets=ignore_junction_targets,
- use_artifact_config=use_config,
artifact_remote_url=remote,
load_refs=True,
)
@@ -535,7 +511,7 @@ class Stream:
tar=False
):
- elements = self._load((target,), selection=selection, use_artifact_config=True, load_refs=True)
+ elements = self._load((target,), selection=selection, load_refs=True)
# self.targets contains a list of the loaded target objects
# if we specify --deps build, Stream._load() will return a list
@@ -614,7 +590,7 @@ class Stream:
#
def artifact_show(self, targets, *, selection=PipelineSelection.NONE):
# Obtain list of Element and/or ArtifactElement objects
- target_objects = self.load_selection(targets, selection=selection, use_artifact_config=True, load_refs=True)
+ target_objects = self.load_selection(targets, selection=selection, load_refs=True)
if self._artifacts.has_fetch_remotes():
self._pipeline.check_remotes(target_objects)
@@ -1159,19 +1135,17 @@ class Stream:
# Args:
# artifact_url - The url of the artifact server to connect to.
# source_url - The url of the source server to connect to.
- # use_artifact_config - Whether to use the artifact config.
- # use_source_config - Whether to use the source config.
#
def __connect_remotes(
- self, artifact_url: str, source_url: str, use_artifact_config: bool, use_source_config: bool
+ self, artifact_url: str, source_url: str
):
# ArtifactCache.setup_remotes expects all projects to be fully loaded
for project in self._context.get_projects():
project.ensure_fully_loaded()
# Connect to remote caches, this needs to be done before resolving element state
- self._artifacts.setup_remotes(use_config=use_artifact_config, remote_url=artifact_url)
- self._sourcecache.setup_remotes(use_config=use_source_config, remote_url=source_url)
+ self._artifacts.setup_remotes(remote_url=artifact_url)
+ self._sourcecache.setup_remotes(remote_url=source_url)
# _load_tracking()
#
@@ -1237,8 +1211,6 @@ class Stream:
# selection (PipelineSelection): The selection mode for the specified targets
# except_targets (list of str): Specified targets to except from fetching
# ignore_junction_targets (bool): Whether junction targets should be filtered out
- # use_artifact_config (bool): Whether to initialize artifacts with the config
- # use_source_config (bool): Whether to initialize remote source caches with the config
# artifact_remote_url (str): A remote url for initializing the artifacts
# source_remote_url (str): A remote url for initializing source caches
#
@@ -1252,8 +1224,6 @@ class Stream:
selection=PipelineSelection.NONE,
except_targets=(),
ignore_junction_targets=False,
- use_artifact_config=False,
- use_source_config=False,
artifact_remote_url=None,
source_remote_url=None,
dynamic_plan=False,
@@ -1277,7 +1247,7 @@ class Stream:
self.targets = elements + artifacts
# Connect to remote caches, this needs to be done before resolving element state
- self.__connect_remotes(artifact_remote_url, source_remote_url, use_artifact_config, use_source_config)
+ self.__connect_remotes(artifact_remote_url, source_remote_url)
# Now move on to loading primary selection.
#
diff --git a/tests/artifactcache/capabilities.py b/tests/artifactcache/capabilities.py
index c8a49f9..8f206b6 100644
--- a/tests/artifactcache/capabilities.py
+++ b/tests/artifactcache/capabilities.py
@@ -41,7 +41,7 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
artifactcache = context.artifactcache
# Manually setup the CAS remote
- artifactcache.setup_remotes(use_config=True)
+ artifactcache.setup_remotes()
assert (
not artifactcache.has_fetch_remotes()
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index e6eaec9..01c0ff8 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -87,7 +87,7 @@ def test_pull(cli, tmpdir, datafiles):
artifactcache = context.artifactcache
# Manually setup the CAS remote
- artifactcache.setup_remotes(use_config=True)
+ artifactcache.setup_remotes()
assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst"
assert artifactcache.pull(element, element_key), "Pull operation failed"
@@ -140,7 +140,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
artifactcache = context.artifactcache
# Manually setup the CAS remote
- artifactcache.setup_remotes(use_config=True)
+ artifactcache.setup_remotes()
assert artifactcache.has_push_remotes()
directory = remote_execution_pb2.Directory()
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 7160e05..4278511 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -41,7 +41,7 @@ def _push(cli, cache_dir, project_dir, config_file, target):
e._initialize_state()
# Manually setup the CAS remotes
- artifactcache.setup_remotes(use_config=True)
+ artifactcache.setup_remotes()
artifactcache.initialize_remotes()
assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst"
@@ -141,7 +141,7 @@ def test_push_message(tmpdir, datafiles):
artifactcache = context.artifactcache
# Manually setup the artifact remote
- artifactcache.setup_remotes(use_config=True)
+ artifactcache.setup_remotes()
artifactcache.initialize_remotes()
assert artifactcache.has_push_remotes()
diff --git a/tests/sourcecache/capabilities.py b/tests/sourcecache/capabilities.py
index 9d41eba..681aaad 100644
--- a/tests/sourcecache/capabilities.py
+++ b/tests/sourcecache/capabilities.py
@@ -37,7 +37,7 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi
sourcecache = context.sourcecache
# Manually setup the CAS remote
- sourcecache.setup_remotes(use_config=True)
+ sourcecache.setup_remotes()
assert (
not sourcecache.has_fetch_remotes()