You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2021/12/07 10:44:00 UTC
[buildstream] branch tristan/mirror-policy created (now ad069e9)
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a change to branch tristan/mirror-policy
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at ad069e9 NEWS: Document addition of fetch/track source user configuration
This branch includes the following new commits:
new ea4bb14 _project.py: Change get_alias_uri() -> alias_exists()
new 62dac2b Add fetch and track source URL policy
new fff93ef doc/source/using_config.rst: Document new source URL policy configurations
new ad069e9 NEWS: Document addition of fetch/track source user configuration
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/04: Add fetch and track source URL policy
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/mirror-policy
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 62dac2b6d77973edc1569161704ed7f05b21a9ef
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Fri Nov 12 17:12:31 2021 +0900
Add fetch and track source URL policy
This adds a new feature which allows the user configuration to specify
what URIs are allowed to be accessed while fetching and tracking.
---
src/buildstream/_context.py | 20 +++++++++++++++++++-
src/buildstream/_project.py | 33 ++++++++++++++++++++++-----------
src/buildstream/data/userconfig.yaml | 22 ++++++++++++++++++++++
src/buildstream/source.py | 6 +++---
src/buildstream/types.py | 23 +++++++++++++++++++++++
5 files changed, 89 insertions(+), 15 deletions(-)
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index bdc780b..911d278 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -34,7 +34,7 @@ from ._elementsourcescache import ElementSourcesCache
from ._remotespec import RemoteSpec, RemoteExecutionSpec
from ._sourcecache import SourceCache
from ._cas import CASCache, CASLogLevel
-from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction
+from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction, _SourceUriPolicy
from ._workspaces import Workspaces, WorkspaceProjectCache
from .node import Node, MappingNode
@@ -169,6 +169,12 @@ class Context:
# Control which dependencies to build
self.build_dependencies: Optional[_PipelineSelection] = None
+ # Control which URIs can be accessed when fetching sources
+ self.fetch_source: Optional[str] = None
+
+ # Control which URIs can be accessed when tracking sources
+ self.track_source: Optional[str] = None
+
# Size of the artifact cache in bytes
self.config_cache_quota: Optional[int] = None
@@ -298,6 +304,8 @@ class Context:
"logdir",
"scheduler",
"build",
+ "fetch",
+ "track",
"artifacts",
"source-caches",
"logging",
@@ -431,6 +439,16 @@ class Context:
)
self.build_dependencies = _PipelineSelection(dependencies)
+ # Load fetch config
+ fetch = defaults.get_mapping("fetch")
+ fetch.validate_keys(["source"])
+ self.fetch_source = fetch.get_enum("source", _SourceUriPolicy)
+
+ # Load track config
+ track = defaults.get_mapping("track")
+ track.validate_keys(["source"])
+ self.track_source = track.get_enum("source", _SourceUriPolicy)
+
# Load per-projects overrides
self._project_overrides = defaults.get_mapping("projects", default={})
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 24783ea..c699f87 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -37,7 +37,7 @@ from .exceptions import LoadErrorReason
from ._options import OptionPool
from .node import ScalarNode, MappingNode, ProvenanceInformation, _assert_symbol_name
from ._pluginfactory import ElementFactory, SourceFactory, load_plugin_origin
-from .types import CoreWarnings, _HostMount, _SourceMirror
+from .types import CoreWarnings, _HostMount, _SourceMirror, _SourceUriPolicy
from ._projectrefs import ProjectRefs, ProjectRefStorage
from ._loader import Loader, LoadContext
from .element import Element
@@ -141,6 +141,7 @@ class Project:
self._shell_command: List[str] = [] # The default interactive shell command
self._shell_environment: Dict[str, str] = {} # Statically set environment vars
self._shell_host_files: List[_HostMount] = [] # A list of HostMount objects
+ self._mirror_override: bool = False # Whether mirrors have been declared in user configuration
# This is a lookup table of lists indexed by project,
# the child dictionaries are lists of ScalarNodes indicating
@@ -393,9 +394,10 @@ class Project:
# Args:
# alias (str): The alias.
# first_pass (bool): Whether to use first pass configuration (for junctions)
+ # tracking (bool): Whether we want the aliases for tracking (otherwise assume fetching)
#
# Returns a list of every URI to replace an alias with
- def get_alias_uris(self, alias, *, first_pass=False):
+ def get_alias_uris(self, alias, *, first_pass=False, tracking=False):
if first_pass:
config = self.first_pass_config
else:
@@ -404,16 +406,23 @@ class Project:
if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test
return [None]
- mirror_list = []
- for mirror_name, mirror in config.mirrors.items():
- if alias in mirror.aliases:
- if mirror_name == config.default_mirror:
- mirror_list = mirror.aliases[alias] + mirror_list
- else:
- mirror_list += mirror.aliases[alias]
+ uri_list = []
+ policy = self._context.track_source if tracking else self._context.fetch_source
- mirror_list.append(config._aliases.get_str(alias))
- return mirror_list
+ if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.MIRRORS) or (
+ policy == _SourceUriPolicy.USER and self._mirror_override
+ ):
+ for mirror_name, mirror in config.mirrors.items():
+ if alias in mirror.aliases:
+ if mirror_name == config.default_mirror:
+ uri_list = mirror.aliases[alias] + uri_list
+ else:
+ uri_list += mirror.aliases[alias]
+
+ if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.ALIASES):
+ uri_list.append(config._aliases.get_str(alias))
+
+ return uri_list
# load_elements()
#
@@ -1002,6 +1011,8 @@ class Project:
mirrors_node = overrides.get_sequence("mirrors", default=None)
if mirrors_node is None:
mirrors_node = config.get_sequence("mirrors", default=[])
+ else:
+ self._mirror_override = True
# Perform variable substitutions in source mirror definitions,
# even if the mirrors are specified in user configuration.
diff --git a/src/buildstream/data/userconfig.yaml b/src/buildstream/data/userconfig.yaml
index 3232de3..3fc29f7 100644
--- a/src/buildstream/data/userconfig.yaml
+++ b/src/buildstream/data/userconfig.yaml
@@ -72,6 +72,28 @@ build:
#
+# Source fetch related configuration
+#
+fetch:
+
+ #
+ # Which URIs are allowed to be accessed
+ #
+ source: all
+
+
+#
+# Source track related configuration
+#
+track:
+
+ #
+ # Which URIs are allowed to be accessed
+ #
+ source: all
+
+
+#
# Logging
#
logging:
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index afc2092..b2dfd60 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -1256,7 +1256,7 @@ class Source(Plugin):
break
alias = fetcher._get_alias()
- for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
+ for uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False):
try:
fetcher.fetch(uri)
# FIXME: Need to consider temporary vs. permanent failures,
@@ -1284,7 +1284,7 @@ class Source(Plugin):
self.fetch(**kwargs)
return
- for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
+ for uri in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False):
new_source = self.__clone_for_uri(uri)
try:
new_source.fetch(**kwargs)
@@ -1314,7 +1314,7 @@ class Source(Plugin):
# NOTE: We are assuming here that tracking only requires substituting the
# first alias used
- for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass)):
+ for uri in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)):
new_source = self.__clone_for_uri(uri)
try:
ref = new_source.track(**kwargs) # pylint: disable=assignment-from-none
diff --git a/src/buildstream/types.py b/src/buildstream/types.py
index 6fa4fe8..f278820 100644
--- a/src/buildstream/types.py
+++ b/src/buildstream/types.py
@@ -255,6 +255,29 @@ class _CacheBuildTrees(FastEnum):
NEVER = "never"
+# _SourceUriPolicy()
+#
+# A policy for which URIs to access when fetching and tracking
+#
+class _SourceUriPolicy(FastEnum):
+
+ # Use all URIs from default aliases and mirrors
+ ALL = "all"
+
+ # Use only the base source aliases defined in project configuration
+ #
+ ALIASES = "aliases"
+
+ # Use only URIs from source mirrors (whether they are found
+ # in project configuration or user configuration)
+ MIRRORS = "mirrors"
+
+ # Use only URIs from user configuration, intentionally causing
+ # a failure if we try to access a source for which the user
+ # configuration has not provided a mirror
+ USER = "user"
+
+
# _PipelineSelection()
#
# Defines the kind of pipeline selection to make when the pipeline
[buildstream] 04/04: NEWS: Document addition of fetch/track source user configuration
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/mirror-policy
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ad069e9e7a32f799c73b10653de4c7ecb72ffb7e
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Dec 7 19:43:20 2021 +0900
NEWS: Document addition of fetch/track source user configuration
---
NEWS | 3 +++
1 file changed, 3 insertions(+)
diff --git a/NEWS b/NEWS
index c44f756..98e25f6 100644
--- a/NEWS
+++ b/NEWS
@@ -15,6 +15,9 @@ Format
o User configuration now has the ability to override source mirrors defined
by project configuration.
+ o User configuration now has fetch source and track source configurations to
+ control what mirrors/URIs can be accessed to obtain sources.
+
CLI
---
o BREAKING CHANGE: Removed `--deps plan` from all commands
[buildstream] 01/04: _project.py: Change get_alias_uri() -> alias_exists()
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/mirror-policy
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ea4bb14aed38a3998a17a65d2435397a0242fa22
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Nov 11 16:37:42 2021 +0900
_project.py: Change get_alias_uri() -> alias_exists()
This clarifies the purpose of this API, which is only used to check
the presence of an alias.
- Project.alias_exists() now checks if the alias is declared
- Project.get_alias_uris() is the only valid way to actually expand the URIs
---
src/buildstream/_project.py | 10 +++++-----
src/buildstream/source.py | 5 ++---
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 882f2c7..24783ea 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -369,7 +369,7 @@ class Project:
def create_source(self, meta, variables):
return self.source_factory.create(self._context, self, meta, variables)
- # get_alias_uri()
+ # alias_exists()
#
# Returns the URI for a given alias, if it exists
#
@@ -378,15 +378,15 @@ class Project:
# first_pass (bool): Whether to use first pass configuration (for junctions)
#
# Returns:
- # str: The URI for the given alias; or None: if there is no URI for
- # that alias.
- def get_alias_uri(self, alias, *, first_pass=False):
+ # bool: Whether the alias is declared in the scope of this project
+ #
+ def alias_exists(self, alias, *, first_pass=False):
if first_pass:
config = self.first_pass_config
else:
config = self.config
- return config._aliases.get_str(alias, default=None)
+ return config._aliases.get_str(alias, default=None) is not None
# get_alias_uris()
#
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index fa8042c..afc2092 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -682,8 +682,7 @@ class Source(Plugin):
# If there is an alias in use, ensure that it exists in the project
if alias:
project = self._get_project()
- alias_uri = project.get_alias_uri(alias, first_pass=self.__first_pass)
- if alias_uri is None:
+ if not project.alias_exists(alias, first_pass=self.__first_pass):
raise SourceError(
"{}: Invalid alias '{}' specified in URL: {}".format(self, alias, url),
reason="invalid-source-alias",
@@ -1117,7 +1116,7 @@ class Source(Plugin):
def _get_alias(self):
alias = self.__expected_alias
project = self._get_project()
- if project.get_alias_uri(alias, first_pass=self.__first_pass):
+ if project.alias_exists(alias, first_pass=self.__first_pass):
# The alias must already be defined in the project's aliases
# otherwise http://foo gets treated like it contains an alias
return alias
[buildstream] 03/04: doc/source/using_config.rst: Document new source URL policy configurations
Posted by tv...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/mirror-policy
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit fff93ef02c3636f99ffa788daabcba8aa1a18511
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Dec 7 19:41:13 2021 +0900
doc/source/using_config.rst: Document new source URL policy configurations
---
doc/source/using_config.rst | 70 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 70 insertions(+)
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index 1f5bbf3..996fddf 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -328,6 +328,76 @@ Attributes
* ``all``: Build elements even if they are build dependencies of artifacts which are already cached
+Fetch controls
+--------------
+Some aspects about how sources get fetched can be controlled by attributes of the ``fetch``
+dictionary at the toplevel, like so:
+
+.. code:: yaml
+
+ #
+ # Fetch controls
+ #
+ fetch:
+
+ #
+ # Don't allow fetching from project defined alias or mirror URIs
+ #
+ source: user
+
+
+Attributes
+~~~~~~~~~~
+
+* ``source``
+
+ This controls what URIs are allowed to be accessed when fetching sources,
+ valid values for this attribute are:
+
+ * ``all``: Fetch from mirrors defined in :ref:`user configuration <config_mirrors>` and
+ :ref:`project configuration <project_essentials_mirrors>`, and also project defined
+ :ref:`default alias URIs <project_source_aliases>`.
+ * ``aliases``: Only allow fetching from project defined :ref:`default alias URIs <project_source_aliases>`.
+ * ``mirrors``: Only allow fetching from mirrors defined in :ref:`user configuration <config_mirrors>` and
+ :ref:`project configuration <project_essentials_mirrors>`
+ * ``user``: Only allow fetching from mirrors defined in :ref:`user configuration <config_mirrors>`
+
+
+Track controls
+--------------
+Some aspects about how sources get tracked can be controlled by attributes of the ``track``
+dictionary at the toplevel, like so:
+
+.. code:: yaml
+
+ #
+ # Track controls
+ #
+ track:
+
+ #
+ # Only track sources for new refs from project defined default alias URIs
+ #
+ source: aliases
+
+
+Attributes
+~~~~~~~~~~
+
+* ``source``
+
+ This controls what URIs are allowed to be accessed when tracking sources
+ for new refs, valid values for this attribute are:
+
+ * ``all``: Track from mirrors defined in :ref:`user configuration <config_mirrors>` and
+ :ref:`project configuration <project_essentials_mirrors>`, and also project defined
+ :ref:`default alias URIs <project_source_aliases>`.
+ * ``aliases``: Only allow tracking from project defined :ref:`default alias URIs <project_source_aliases>`.
+ * ``mirrors``: Only allow tracking from mirrors defined in :ref:`user configuration <config_mirrors>` and
+ :ref:`project configuration <project_essentials_mirrors>`
+ * ``user``: Only allow tracking from mirrors defined in :ref:`user configuration <config_mirrors>`
+
+
Logging controls
----------------
Various aspects of how BuildStream presents output and UI can be controlled with