You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:25:55 UTC
[buildstream] branch tristan/multi-location-overlaps created (now
f145c87)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at f145c87 tests/frontend/overlaps.py: Test CoreWarnings.UNSTAGED_FILES
This branch includes the following new commits:
new 3d799d4 compose plugin: Stage the artifacts in Element.stage()
new 2857936 filter plugin: Stage the artifacts in Element.stage()
new bd23f98 tests/elements/filter/basic/element_plugins/dynamic.py: Stage in Element.stage()
new cfdbdf2 types.py: Adding OverlapAction enumeration
new 79e0a3a types.py: Adding CoreWarnings.UNSTAGED_FILES
new 0828474 _overlapcollector.py: Adding the OverlapCollector as a separate module
new 2e15309 element.py, _elementproxy.py: Use new OverlapCollector
new 2de37e9 tests/frontend/overlaps.py: Minor cleanup and refactoring
new f145c87 tests/frontend/overlaps.py: Test CoreWarnings.UNSTAGED_FILES
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 06/09: _overlapcollector.py: Adding the
OverlapCollector as a separate module
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 082847492a35941e2336daaaa1d41c3abb328da7
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Sep 9 17:05:28 2020 +0900
_overlapcollector.py: Adding the OverlapCollector as a separate module
---
src/buildstream/_overlapcollector.py | 328 +++++++++++++++++++++++++++++++++++
1 file changed, 328 insertions(+)
diff --git a/src/buildstream/_overlapcollector.py b/src/buildstream/_overlapcollector.py
new file mode 100644
index 0000000..30ecfa3
--- /dev/null
+++ b/src/buildstream/_overlapcollector.py
@@ -0,0 +1,328 @@
+#
+# Copyright (C) 2020 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Tristan Van Berkom <tr...@codethink.co.uk>
+
+import os
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, Optional, List, Tuple
+from .plugin import Plugin
+from .types import CoreWarnings, OverlapAction
+from .utils import FileListResult
+
+if TYPE_CHECKING:
+ from typing import Dict
+
+ # pylint: disable=cyclic-import
+ from .element import Element
+
+ # pylint: enable=cyclic-import
+
+
+# OverlapCollector()
+#
+# Collects results of Element.stage_artifact() and saves
+# them in order to raise a proper overlap error at the end
+# of staging.
+#
+# Args:
+# element (Element): The element for which we are staging artifacts
+#
+class OverlapCollector:
+ def __init__(self, element: "Element"):
+
+ # The Element we are staging for, on which we'll issue warnings
+ self._element = element # type: Element
+
+ # The list of sessions
+ self._sessions = [] # type: List[OverlapCollectorSession]
+
+ # The active session, if any
+ self._session = None # type: Optional[OverlapCollectorSession]
+
+ # session()
+ #
+ # Create a session for collecting overlaps, calls to OverlapCollector.collect_stage_result()
+ # are expected to always occur within the context of a session (this context manager).
+ #
+ # Upon exiting this context, warnings and/or errors will be issued for any overlaps
+ # which occurred either as a result of overlapping files within this session, or
+ # as a result of files staged during this session, overlapping with files staged in
+ # previous sessions in this OverlapCollector.
+ #
+ # Args:
+ # action (OverlapAction): The action to take for this overall session's overlaps with other sessions
+ # location (str): The Sandbox relative location this session was created for
+ #
+ @contextmanager
+ def session(self, action: str, location: Optional[str]):
+ assert self._session is None, "Stage session already started"
+
+ if location is None:
+ location = "/"
+
+ self._session = OverlapCollectorSession(self._element, action, location)
+
+ # Run code body where staging results can be collected.
+ yield
+
+ # Issue warnings for the current session, passing along previously completed sessions
+ self._session.warnings(self._sessions)
+
+ # Store the newly ended session and end the session
+ self._sessions.append(self._session)
+ self._session = None
+
+ # collect_stage_result()
+ #
+ # Collect and accumulate results of Element.stage_artifact()
+ #
+ # Args:
+ # element (Element): The name of the element staged
+ # result (FileListResult): The result of Element.stage_artifact()
+ #
+ def collect_stage_result(self, element: "Element", result: FileListResult):
+ assert self._session is not None, "Staging files outside of staging session"
+
+ self._session.collect_stage_result(element, result)
+
+
+# OverlapCollectorSession()
+#
+# Collect the results of a single session
+#
+# Args:
+# element (Element): The element for which we are staging artifacts
+# action (OverlapAction): The action to take for this overall session's overlaps with other sessions
+# location (str): The Sandbox relative location this session was created for
+#
+class OverlapCollectorSession:
+ def __init__(self, element: "Element", action: str, location: str):
+
+ # The Element we are staging for, on which we'll issue warnings
+ self._element = element # type: Element
+
+ # The OverlapAction for this session
+ self._action = action # type: str
+
+ # The Sandbox relative directory this session was created for
+ self._location = location # type: str
+
+ # Dictionary of files which were ignored (See FileListResult()), keyed by element unique ID
+ self._ignored = {} # type: Dict[int, List[str]]
+
+ # Dictionary of files which were staged, keyed by element unique ID
+ self._files_written = {} # type: Dict[int, List[str]]
+
+ # Dictionary of element IDs which overlapped, keyed by the file they overlap on
+ self._overlaps = {} # type: Dict[str, List[int]]
+
+ # collect_stage_result()
+ #
+ # Collect and accumulate results of Element.stage_artifact()
+ #
+ # Args:
+ # element (Element): The name of the element staged
+ # result (FileListResult): The result of Element.stage_artifact()
+ #
+ def collect_stage_result(self, element: "Element", result: FileListResult):
+
+ for overwritten_file in result.overwritten:
+
+ overlap_list = None
+ try:
+ overlap_list = self._overlaps[overwritten_file]
+ except KeyError:
+
+ # Create a fresh list
+ #
+ self._overlaps[overwritten_file] = overlap_list = []
+
+ # Search files which were staged in this session, start the
+ # list off with the bottom most element
+ #
+ for element_id, staged_files in self._files_written.items():
+ if overwritten_file in staged_files:
+ overlap_list.append(element_id)
+ break
+
+ # Add the currently staged element to the overlap list, it might be
+ # the only element in the list if it overlaps with a file staged
+ # from a previous session.
+ #
+ overlap_list.append(element._unique_id)
+
+ # Record written files and ignored files.
+ #
+ self._files_written[element._unique_id] = result.files_written
+ if result.ignored:
+ self._ignored[element._unique_id] = result.ignored
+
+ # warnings()
+ #
+ # Issue any warnings as a batch as a result of staging artifacts,
+ # based on the results collected with collect_stage_result().
+ #
+ # Args:
+ # sessions (list): List of previously completed sessions
+ #
+ def warnings(self, sessions: List["OverlapCollectorSession"]):
+
+ # Collect a table of filenames which overlapped something from outside of this session.
+ #
+ external_overlaps = {} # type: Dict[str, int]
+
+ #
+ # First issue the warnings for this session
+ #
+ if self._overlaps:
+ overlap_warning = False
+ detail = "Staged files overwrite existing files in staging area: {}\n".format(self._location)
+ for filename, element_ids in self._overlaps.items():
+
+ # If there is only one element in the overlap list, it means it has
+ # overlapped a file from a previous session.
+ #
+ # Ignore it and handle the warning below
+ #
+ if len(element_ids) == 1:
+ external_overlaps[filename] = element_ids[0]
+ continue
+
+ # Filter whitelisted elements out of the list of overlapping elements
+ #
+ # Ignore the bottom-most element as it does not overlap anything.
+ #
+ overlapping_element_ids = element_ids[1:]
+ warning_elements = self._filter_whitelisted(filename, overlapping_element_ids)
+
+ if warning_elements:
+ overlap_warning = True
+
+ detail += self._overlap_detail(filename, warning_elements, element_ids)
+
+ if overlap_warning:
+ self._element.warn(
+ "Non-whitelisted overlaps detected", detail=detail, warning_token=CoreWarnings.OVERLAPS
+ )
+
+ if self._ignored:
+ detail = "Not staging files which would replace non-empty directories in staging area: {}\n".format(
+ self._location
+ )
+ for element_id, ignored_filenames in self._ignored.items():
+ element = Plugin._lookup(element_id)
+ detail += "\nFrom {}:\n".format(element._get_full_name())
+ detail += " " + " ".join(
+ ["{}\n".format(os.path.join(self._location, filename)) for filename in ignored_filenames]
+ )
+ self._element.warn(
+ "Not staging files which would have replaced non-empty directories",
+ detail=detail,
+ warning_token=CoreWarnings.UNSTAGED_FILES,
+ )
+
+ if external_overlaps and self._action != OverlapAction.IGNORE:
+ detail = "Detected file overlaps while staging elements into: {}\n".format(self._location)
+
+ # Find the session responsible for the overlap
+ #
+ for filename, element_id in external_overlaps.items():
+ absolute_filename = os.path.join(self._location, filename)
+ overlapped_id, location = self._search_stage_element(absolute_filename, sessions)
+ element = Plugin._lookup(element_id)
+ overlapped = Plugin._lookup(overlapped_id)
+ detail += "{}: {} overlaps files previously staged by {} in: {}\n".format(
+ absolute_filename, element._get_full_name(), overlapped._get_full_name(), location
+ )
+
+ if self._action == OverlapAction.WARNING:
+ self._element.warn("Overlaps detected", detail=detail, warning_token=CoreWarnings.OVERLAPS)
+ else:
+ from .element import ElementError
+
+ raise ElementError("Overlaps detected", detail=detail, reason="overlaps")
+
+ # _search_stage_element()
+ #
+ # Search the sessions list for the element responsible for staging the given file
+ #
+ # Args:
+ # filename (str): The sandbox relative file which was overwritten
+ # sessions (List[OverlapCollectorSession])
+ #
+ # Returns:
+ # element_id (int): The unique ID of the element responsible
+ # location (str): The sandbox relative staging location where element_id was staged
+ #
+ def _search_stage_element(self, filename: str, sessions: List["OverlapCollectorSession"]) -> Tuple[int, str]:
+ for session in reversed(sessions):
+ for element_id, staged_files in session._files_written.items():
+ if any(
+ staged_file
+ for staged_file in staged_files
+ if os.path.join(session._location, staged_file) == filename
+ ):
+ return element_id, session._location
+
+ assert False, "Could not find element responsible for staging: {}".format(filename)
+
+ # Silence the linter with an unreachable return statement
+ return None, None
+
+ # _filter_whitelisted()
+ #
+ # Args:
+ # filename (str): The staging session relative filename
+ # element_ids (List[int]): Ordered list of elements
+ #
+ # Returns:
+ # (List[Element]): The list of element objects which are not whitelisted
+ #
+ def _filter_whitelisted(self, filename: str, element_ids: List[int]):
+ overlap_elements = []
+
+ for element_id in element_ids:
+ element = Plugin._lookup(element_id)
+ if not element._file_is_whitelisted(filename):
+ overlap_elements.append(element)
+
+ return overlap_elements
+
+ # _overlap_detail()
+ #
+ # Get a string to describe overlaps on a filename
+ #
+ # Args:
+ # filename (str): The filename being overlapped
+ # overlap_elements (List[Element]): A list of Elements overlapping
+ # element_ids (List[int]): The ordered ID list of elements which staged this file
+ #
+ def _overlap_detail(self, filename, overlap_elements, element_ids):
+ filename = os.path.join(self._location, filename)
+ if overlap_elements:
+ overlap_element_names = [element._get_full_name() for element in overlap_elements]
+ overlap_order_elements = [Plugin._lookup(element_id) for element_id in element_ids]
+ overlap_order_names = [element._get_full_name() for element in overlap_order_elements]
+ return "{}: {} {} not permitted to overlap other elements, order {} \n".format(
+ filename,
+ " and ".join(overlap_element_names),
+ "is" if len(overlap_element_names) == 1 else "are",
+ " above ".join(reversed(overlap_order_names)),
+ )
+ else:
+ return ""
[buildstream] 04/09: types.py: Adding OverlapAction enumeration
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cfdbdf2f4f8cb8d45ff56aa3dfdcbcc8c6d31f15
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Sep 10 16:09:43 2020 +0900
types.py: Adding OverlapAction enumeration
Used to define the behavior of multiple calls to Element.stage_artifact()
and Element.stage_dependency_artifacts()
---
src/buildstream/__init__.py | 2 +-
src/buildstream/types.py | 40 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/__init__.py b/src/buildstream/__init__.py
index 4d15187..606353a 100644
--- a/src/buildstream/__init__.py
+++ b/src/buildstream/__init__.py
@@ -30,7 +30,7 @@ if "_BST_COMPLETION" not in os.environ:
from .utils import UtilError, ProgramNotFoundError
from .sandbox import Sandbox, SandboxFlags, SandboxCommandError
- from .types import CoreWarnings
+ from .types import CoreWarnings, OverlapAction
from .node import MappingNode, Node, ProvenanceInformation, ScalarNode, SequenceNode
from .plugin import Plugin
from .source import Source, SourceError, SourceFetcher
diff --git a/src/buildstream/types.py b/src/buildstream/types.py
index 3b1f7a4..7ba2793 100644
--- a/src/buildstream/types.py
+++ b/src/buildstream/types.py
@@ -119,6 +119,46 @@ class CoreWarnings:
"""
+class OverlapAction(FastEnum):
+ """OverlapAction()
+
+ Defines what action to take when files staged into the sandbox overlap.
+
+ .. note::
+
+ This only dictates what happens when functions such as
+ :func:`Element.stage_artifact() <buildstream.element.Element.stage_artifact>` and
+ :func:`Element.stage_dependency_artifacts() <buildstream.element.Element.stage_dependency_artifacts>`
+ are called multiple times in an Element's :func:`Element.stage() <buildstream.element.Element.stage>`
+ implementation, and the files staged from one function call result in overlapping files staged
+ from previous invocations.
+
+ If multiple staged elements overlap eachother within a single call to
+ :func:`Element.stage_dependency_artifacts() <buildstream.element.Element.stage_dependency_artifacts>`,
+ then the :ref:`overlap whitelist <public_overlap_whitelist>` will be ovserved, and warnings will
+ be issued for overlapping files, which will be fatal warnings if
+ :attr:`CoreWarnings.OVERLAPS <buildstream.types.CoreWarnings.OVERLAPS>` is specified
+ as a :ref:`fatal warning <configurable_warnings>`.
+ """
+
+ ERROR = "error"
+ """
+ It is an error to overlap previously staged files
+ """
+
+ WARNING = "warning"
+ """
+ A warning will be issued for previously staged files, which will fatal if
+ :attr:`CoreWarnings.OVERLAPS <buildstream.types.CoreWarnings.OVERLAPS>` is specified
+ as a :ref:`fatal warning <configurable_warnings>` in the project.
+ """
+
+ IGNORE = "ignore"
+ """
+ Overlapping files are acceptable, and do not cause any warning or error.
+ """
+
+
# _Scope():
#
# Defines the scope of dependencies to include for a given element
[buildstream] 01/09: compose plugin: Stage the artifacts in
Element.stage()
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3d799d4804e0c0d1438c80b49dc859debeb196e4
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Sep 9 17:06:02 2020 +0900
compose plugin: Stage the artifacts in Element.stage()
It will now be illegal to call Element.stage_dependency_artifacts() outside
of the Element.stage() abstract method.
---
src/buildstream/plugins/elements/compose.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py
index 8084196..0d49884 100644
--- a/src/buildstream/plugins/elements/compose.py
+++ b/src/buildstream/plugins/elements/compose.py
@@ -82,17 +82,15 @@ class ComposeElement(Element):
pass
def stage(self, sandbox):
- pass
-
- def assemble(self, sandbox):
-
- require_split = self.include or self.exclude or not self.include_orphans
# Stage deps in the sandbox root
with self.timed_activity("Staging dependencies", silent_nested=True):
self.stage_dependency_artifacts(sandbox)
+ def assemble(self, sandbox):
manifest = set()
+
+ require_split = self.include or self.exclude or not self.include_orphans
if require_split:
with self.timed_activity("Computing split", silent_nested=True):
for dep in self.dependencies():
[buildstream] 05/09: types.py: Adding CoreWarnings.UNSTAGED_FILES
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 79e0a3ad4f6374e56009eb051939ff49959206bb
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Sat Sep 12 17:11:33 2020 +0900
types.py: Adding CoreWarnings.UNSTAGED_FILES
Makes the warning fatal if we fail to stage a file because it would have
otherwise overwritten a non-empty directory.
---
src/buildstream/types.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/types.py b/src/buildstream/types.py
index 7ba2793..48dd5de 100644
--- a/src/buildstream/types.py
+++ b/src/buildstream/types.py
@@ -100,6 +100,12 @@ class CoreWarnings:
which is not whitelisted. See :ref:`Overlap Whitelist <public_overlap_whitelist>`
"""
+ UNSTAGED_FILES = "unstaged-files"
+ """
+ This warning will be produced when a file cannot be staged. This can happen when
+ a file overlaps with a directory in the sandbox that is not empty.
+ """
+
REF_NOT_IN_TRACK = "ref-not-in-track"
"""
This warning will be produced when a source is configured with a reference
@@ -114,7 +120,7 @@ class CoreWarnings:
BAD_CHARACTERS_IN_NAME = "bad-characters-in-name"
"""
- This warning will be produces when filename for a target contains invalid
+ This warning will be produced when a filename for a target contains invalid
characters in its name.
"""
[buildstream] 08/09: tests/frontend/overlaps.py: Minor cleanup and
refactoring
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2de37e91599ed42482922c13b2405f75ce46e428
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Fri Sep 11 20:15:39 2020 +0900
tests/frontend/overlaps.py: Minor cleanup and refactoring
This commit:
* Removes testing of the deprecated `fail-on-overlap` project configuration
option, this is going away soon and unneeded.
* Tests that warnings are issued whenever they should be (some tests
were happy to see a successful run but failed to check for an
expected warning).
* Test error/warning more evenly across tests, some were missing the
warning mode.
* Use `bst show` instead of `bst build` for the undefined_variable
test, it should fail without needing a build.
---
tests/frontend/overlaps.py | 69 +++++++++++++++++++++-------------------------
1 file changed, 32 insertions(+), 37 deletions(-)
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index 28bf8a7..1bf22ab7 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -13,68 +13,64 @@ from tests.testutils import generate_junction
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "overlaps")
-def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True, project_name="test"):
+def gen_project(project_dir, fail_on_overlap, *, project_name="test"):
template = {"name": project_name, "min-version": "2.0"}
- if use_fatal_warnings:
- template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else []
- else:
- template["fail-on-overlap"] = fail_on_overlap
+ template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else []
projectfile = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump(template, projectfile)
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("use_fatal_warnings", [True, False])
-def test_overlaps(cli, datafiles, use_fatal_warnings):
- project_dir = str(datafiles)
- gen_project(project_dir, False, use_fatal_warnings)
- result = cli.run(project=project_dir, silent=True, args=["build", "collect.bst"])
- result.assert_success()
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("use_fatal_warnings", [True, False])
-def test_overlaps_error(cli, datafiles, use_fatal_warnings):
+@pytest.mark.parametrize("error", [False, True], ids=["warning", "error"])
+def test_overlaps(cli, datafiles, error):
project_dir = str(datafiles)
- gen_project(project_dir, True, use_fatal_warnings)
+ gen_project(project_dir, error)
result = cli.run(project=project_dir, silent=True, args=["build", "collect.bst"])
- result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_overlaps_whitelist(cli, datafiles):
- project_dir = str(datafiles)
- gen_project(project_dir, True)
- result = cli.run(project=project_dir, silent=True, args=["build", "collect-whitelisted.bst"])
- result.assert_success()
+ if error:
+ result.assert_main_error(ErrorDomain.STREAM, None)
+ result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
+ else:
+ result.assert_success()
+ assert "WARNING [overlaps]" in result.stderr
+#
+# When the overlap is whitelisted, there is no warning or error.
+#
+# Still test this in fatal/nonfatal warning modes
+#
@pytest.mark.datafiles(DATA_DIR)
-def test_overlaps_whitelist_ignored(cli, datafiles):
+@pytest.mark.parametrize("error", [False, True], ids=["warning", "error"])
+def test_overlaps_whitelisted(cli, datafiles, error):
project_dir = str(datafiles)
- gen_project(project_dir, False)
+ gen_project(project_dir, error)
result = cli.run(project=project_dir, silent=True, args=["build", "collect-whitelisted.bst"])
result.assert_success()
+ assert "WARNING [overlaps]" not in result.stderr
@pytest.mark.datafiles(DATA_DIR)
-def test_overlaps_whitelist_on_overlapper(cli, datafiles):
+@pytest.mark.parametrize("error", [False, True], ids=["warning", "error"])
+def test_overlaps_whitelist_on_overlapper(cli, datafiles, error):
# Tests that the overlapping element is responsible for whitelisting,
# i.e. that if A overlaps B overlaps C, and the B->C overlap is permitted,
# it'll still fail because A doesn't permit overlaps.
project_dir = str(datafiles)
- gen_project(project_dir, True)
+ gen_project(project_dir, error)
result = cli.run(project=project_dir, silent=True, args=["build", "collect-partially-whitelisted.bst"])
- result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
+ if error:
+ result.assert_main_error(ErrorDomain.STREAM, None)
+ result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
+ else:
+ result.assert_success()
+ assert "WARNING [overlaps]" in result.stderr
@pytest.mark.datafiles(DATA_DIR)
def test_overlaps_whitelist_undefined_variable(cli, datafiles):
project_dir = str(datafiles)
gen_project(project_dir, False)
- result = cli.run(project=project_dir, silent=True, args=["build", "whitelist-undefined.bst"])
+ result = cli.run(project=project_dir, silent=True, args=["show", "whitelist-undefined.bst"])
# Assert that we get the expected undefined variable error,
# and that it has the provenance we expect from whitelist-undefined.bst
@@ -84,12 +80,11 @@ def test_overlaps_whitelist_undefined_variable(cli, datafiles):
@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("use_fatal_warnings", [True, False])
-def test_overlaps_script(cli, datafiles, use_fatal_warnings):
+def test_overlaps_script(cli, datafiles):
# Test overlaps with script element to test
# Element.stage_dependency_artifacts() with Scope.RUN
project_dir = str(datafiles)
- gen_project(project_dir, False, use_fatal_warnings)
+ gen_project(project_dir, False)
result = cli.run(project=project_dir, silent=True, args=["build", "script.bst"])
result.assert_success()
[buildstream] 02/09: filter plugin: Stage the artifacts in
Element.stage()
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 285793686b214c006ec0a23aa30a5a6267bab3e4
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Sep 10 21:34:50 2020 +0900
filter plugin: Stage the artifacts in Element.stage()
Staging artifacts at Element.assemble() time is now illegal
---
src/buildstream/plugins/elements/filter.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py
index c817ca4..783079c 100644
--- a/src/buildstream/plugins/elements/filter.py
+++ b/src/buildstream/plugins/elements/filter.py
@@ -219,9 +219,6 @@ class FilterElement(Element):
pass
def stage(self, sandbox):
- pass
-
- def assemble(self, sandbox):
with self.timed_activity("Staging artifact", silent_nested=True):
for dep in self.dependencies(recurse=False):
# Check that all the included/excluded domains exist
@@ -250,6 +247,8 @@ class FilterElement(Element):
raise ElementError("Unknown domains declared.", detail=detail)
dep.stage_artifact(sandbox, include=self.include, exclude=self.exclude, orphans=self.include_orphans)
+
+ def assemble(self, sandbox):
return ""
def _get_source_element(self):
[buildstream] 09/09: tests/frontend/overlaps.py: Test
CoreWarnings.UNSTAGED_FILES
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit f145c87cad12bde43a110c3a0c47effd54f0dffe
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Sat Sep 12 17:26:29 2020 +0900
tests/frontend/overlaps.py: Test CoreWarnings.UNSTAGED_FILES
---
tests/frontend/overlaps.py | 18 ++++++++++++++++--
tests/frontend/overlaps/directory-file.bst | 9 +++++++++
tests/frontend/overlaps/directory-file/directory-file | 1 +
tests/frontend/overlaps/unstaged.bst | 4 ++++
tests/frontend/overlaps/with-directory.bst | 7 +++++++
.../overlaps/with-directory/directory-file/file | 1 +
6 files changed, 38 insertions(+), 2 deletions(-)
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index 1bf22ab7..77e9a82 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -13,15 +13,29 @@ from tests.testutils import generate_junction
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "overlaps")
-def gen_project(project_dir, fail_on_overlap, *, project_name="test"):
+def gen_project(project_dir, fatal_warnings, *, project_name="test"):
template = {"name": project_name, "min-version": "2.0"}
- template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else []
+ template["fatal-warnings"] = [CoreWarnings.OVERLAPS, CoreWarnings.UNSTAGED_FILES] if fatal_warnings else []
projectfile = os.path.join(project_dir, "project.conf")
_yaml.roundtrip_dump(template, projectfile)
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("error", [False, True], ids=["warning", "error"])
+def test_unstaged_files(cli, datafiles, error):
+ project_dir = str(datafiles)
+ gen_project(project_dir, error)
+ result = cli.run(project=project_dir, silent=True, args=["build", "unstaged.bst"])
+ if error:
+ result.assert_main_error(ErrorDomain.STREAM, None)
+ result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.UNSTAGED_FILES)
+ else:
+ result.assert_success()
+ assert "WARNING [unstaged-files]" in result.stderr
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("error", [False, True], ids=["warning", "error"])
def test_overlaps(cli, datafiles, error):
project_dir = str(datafiles)
gen_project(project_dir, error)
diff --git a/tests/frontend/overlaps/directory-file.bst b/tests/frontend/overlaps/directory-file.bst
new file mode 100644
index 0000000..ab3e98a
--- /dev/null
+++ b/tests/frontend/overlaps/directory-file.bst
@@ -0,0 +1,9 @@
+kind: import
+config:
+ source: /
+ target: /
+depends:
+- with-directory.bst
+sources:
+- kind: local
+ path: "directory-file"
diff --git a/tests/frontend/overlaps/directory-file/directory-file b/tests/frontend/overlaps/directory-file/directory-file
new file mode 100644
index 0000000..f73f309
--- /dev/null
+++ b/tests/frontend/overlaps/directory-file/directory-file
@@ -0,0 +1 @@
+file
diff --git a/tests/frontend/overlaps/unstaged.bst b/tests/frontend/overlaps/unstaged.bst
new file mode 100644
index 0000000..974bc3f
--- /dev/null
+++ b/tests/frontend/overlaps/unstaged.bst
@@ -0,0 +1,4 @@
+kind: compose
+
+build-depends:
+- directory-file.bst
diff --git a/tests/frontend/overlaps/with-directory.bst b/tests/frontend/overlaps/with-directory.bst
new file mode 100644
index 0000000..3963258
--- /dev/null
+++ b/tests/frontend/overlaps/with-directory.bst
@@ -0,0 +1,7 @@
+kind: import
+config:
+ source: /
+ target: /
+sources:
+- kind: local
+ path: "with-directory"
diff --git a/tests/frontend/overlaps/with-directory/directory-file/file b/tests/frontend/overlaps/with-directory/directory-file/file
new file mode 100644
index 0000000..f73f309
--- /dev/null
+++ b/tests/frontend/overlaps/with-directory/directory-file/file
@@ -0,0 +1 @@
+file
[buildstream] 07/09: element.py,
_elementproxy.py: Use new OverlapCollector
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 2e15309a63d43a3c7a1a8ddc2792ef69d6d8adef
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Wed Sep 9 17:11:33 2020 +0900
element.py, _elementproxy.py: Use new OverlapCollector
Setup the OverlapCollector in Element.stage() routines, and ensure we
call OverlapCollector.start_session() and OverlapCollector.end_session()
in the right places.
This adds the OverlapAction `action` parameter to the Element.stage_artifact()
and Element.stage_dependency_artifacts() APIs so that Elements can choose
how to behave when multiple artifact staging calls overlap with files staged
by previous artifact staging calls.
---
src/buildstream/_elementproxy.py | 38 ++++-
src/buildstream/element.py | 289 ++++++++++++++++++---------------------
2 files changed, 163 insertions(+), 164 deletions(-)
diff --git a/src/buildstream/_elementproxy.py b/src/buildstream/_elementproxy.py
index acb08ce..a7b1f09 100644
--- a/src/buildstream/_elementproxy.py
+++ b/src/buildstream/_elementproxy.py
@@ -18,7 +18,7 @@
# Tristan Van Berkom <tr...@codethink.co.uk>
from typing import TYPE_CHECKING, cast, Optional, Iterator, Dict, List, Sequence
-from .types import _Scope
+from .types import _Scope, OverlapAction
from .utils import FileListResult
from ._pluginproxy import PluginProxy
@@ -96,13 +96,23 @@ class ElementProxy(PluginProxy):
sandbox: "Sandbox",
*,
path: str = None,
+ action: str = OverlapAction.WARNING,
include: Optional[List[str]] = None,
exclude: Optional[List[str]] = None,
orphans: bool = True
) -> FileListResult:
- return cast("Element", self._plugin).stage_artifact(
- sandbox, path=path, include=include, exclude=exclude, orphans=orphans
- )
+
+ owner = cast("Element", self._owner)
+ element = cast("Element", self._plugin)
+
+ assert owner._overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()"
+
+ with owner._overlap_collector.session(action, path):
+ result = element._stage_artifact(
+ sandbox, path=path, action=action, include=include, exclude=exclude, orphans=orphans, owner=owner
+ )
+
+ return result
def stage_dependency_artifacts(
self,
@@ -110,6 +120,7 @@ class ElementProxy(PluginProxy):
selection: Sequence["Element"] = None,
*,
path: str = None,
+ action: str = OverlapAction.WARNING,
include: Optional[List[str]] = None,
exclude: Optional[List[str]] = None,
orphans: bool = True
@@ -120,7 +131,7 @@ class ElementProxy(PluginProxy):
if selection is None:
selection = [cast("Element", self._plugin)]
cast("Element", self._owner).stage_dependency_artifacts(
- sandbox, selection, path=path, include=include, exclude=exclude, orphans=orphans
+ sandbox, selection, path=path, action=action, include=include, exclude=exclude, orphans=orphans
)
def integrate(self, sandbox: "Sandbox") -> None:
@@ -154,3 +165,20 @@ class ElementProxy(PluginProxy):
def _file_is_whitelisted(self, path):
return cast("Element", self._plugin)._file_is_whitelisted(path)
+
+ def _stage_artifact(
+ self,
+ sandbox: "Sandbox",
+ *,
+ path: str = None,
+ action: str = OverlapAction.WARNING,
+ include: Optional[List[str]] = None,
+ exclude: Optional[List[str]] = None,
+ orphans: bool = True,
+ owner: Optional["Element"] = None
+ ) -> FileListResult:
+ owner = cast("Element", self._owner)
+ element = cast("Element", self._plugin)
+ return element._stage_artifact(
+ sandbox, path=path, action=action, include=include, exclude=exclude, orphans=orphans, owner=owner
+ )
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 3316d8a..e03b889 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1,5 +1,5 @@
#
-# Copyright (C) 2016-2018 Codethink Limited
+# Copyright (C) 2016-2020 Codethink Limited
# Copyright (C) 2017-2020 Bloomberg Finance LP
#
# This program is free software; you can redistribute it and/or
@@ -103,11 +103,12 @@ from .plugin import Plugin
from .sandbox import SandboxFlags, SandboxCommandError
from .sandbox._config import SandboxConfig
from .sandbox._sandboxremote import SandboxRemote
-from .types import CoreWarnings, _Scope, _CacheBuildTrees, _KeyStrength
+from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction
from ._artifact import Artifact
from ._elementproxy import ElementProxy
from ._elementsources import ElementSources
from ._loader import Symbol, MetaSource
+from ._overlapcollector import OverlapCollector
from .storage.directory import Directory
from .storage._filebaseddirectory import FileBasedDirectory
@@ -235,6 +236,7 @@ class Element(Plugin):
# Internal instance properties
#
self._depth = None # Depth of Element in its current dependency graph
+ self._overlap_collector = None # type: Optional[OverlapCollector]
#
# Private instance properties
@@ -601,9 +603,10 @@ class Element(Plugin):
sandbox: "Sandbox",
*,
path: str = None,
+ action: str = OverlapAction.WARNING,
include: Optional[List[str]] = None,
exclude: Optional[List[str]] = None,
- orphans: bool = True
+ orphans: bool = True,
) -> FileListResult:
"""Stage this element's output artifact in the sandbox
@@ -615,6 +618,7 @@ class Element(Plugin):
Args:
sandbox: The build sandbox
path: An optional sandbox relative path
+ action (OverlapAction): The action to take when overlapping with previous invocations
include: An optional list of domains to include files from
exclude: An optional list of domains to exclude files from
orphans: Whether to include files not spoken for by split domains
@@ -631,38 +635,24 @@ class Element(Plugin):
unless the existing directory in `dest` is not empty in
which case the path will be reported in the return value.
- **Example:**
-
- .. code:: python
+ .. attention::
- # Stage the dependencies for a build of 'self'
- for dep in self.dependencies():
- dep.stage_artifact(sandbox)
+ When staging artifacts with their dependencies, use
+ :func:`Element.stage_dependency_artifacts() <buildstream.element.Element.stage_dependency_artifacts>`
+ instead.
"""
+ assert self._overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()"
- if not self._cached():
- detail = (
- "No artifacts have been cached yet for that element\n"
- + "Try building the element first with `bst build`\n"
- )
- raise ElementError("No artifacts to stage", detail=detail, reason="uncached-checkout-attempt")
-
- # Time to use the artifact, check once more that it's there
- self.__assert_cached()
-
- self.status("Staging {}/{}".format(self.name, self._get_brief_display_key()))
- # Disable type checking since we can't easily tell mypy that
- # `self.__artifact` can't be None at this stage.
- files_vdir = self.__artifact.get_files() # type: ignore
-
- # Hard link it into the staging area
#
- vbasedir = sandbox.get_virtual_directory()
- vstagedir = vbasedir if path is None else vbasedir.descend(*path.lstrip(os.sep).split(os.sep), create=True)
-
- split_filter = self.__split_filter_func(include, exclude, orphans)
-
- result = vstagedir.import_files(files_vdir, filter_callback=split_filter, report_written=True, can_link=True)
+ # The public API can only be called on the implementing plugin itself.
+ #
+ # ElementProxy calls to stage_artifact() are routed directly to _stage_artifact(),
+ # and the ElementProxy takes care of starting and ending the OverlapCollector session.
+ #
+ with self._overlap_collector.session(action, path):
+ result = self._stage_artifact(
+ sandbox, path=path, action=action, include=include, exclude=exclude, orphans=orphans
+ )
return result
@@ -672,9 +662,10 @@ class Element(Plugin):
selection: Sequence["Element"] = None,
*,
path: str = None,
+ action: str = OverlapAction.WARNING,
include: Optional[List[str]] = None,
exclude: Optional[List[str]] = None,
- orphans: bool = True
+ orphans: bool = True,
) -> None:
"""Stage element dependencies in scope
@@ -689,23 +680,22 @@ class Element(Plugin):
is called is used as the `selection`.
Args:
- sandbox: The build sandbox
+ sandbox (Sandbox): The build sandbox
selection (Sequence[Element]): A list of dependencies to select, or None
- path An optional sandbox relative path
- include: An optional list of domains to include files from
- exclude: An optional list of domains to exclude files from
- orphans: Whether to include files not spoken for by split domains
+ path (str): An optional sandbox relative path
+ action (OverlapAction): The action to take when overlapping with previous invocations
+ include (List[str]): An optional list of domains to include files from
+ exclude (List[str]): An optional list of domains to exclude files from
+ orphans (bool): Whether to include files not spoken for by split domains
Raises:
(:class:`.ElementError`): if forbidden overlaps occur.
"""
- overlaps = _OverlapCollector(self)
-
- for dep in self.dependencies(selection):
- result = dep.stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans)
- overlaps.collect_stage_result(dep, result)
+ assert self._overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()"
- overlaps.overlap_warnings()
+ with self._overlap_collector.session(action, path):
+ for dep in self.dependencies(selection):
+ dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self)
def integrate(self, sandbox: "Sandbox") -> None:
"""Integrate currently staged filesystem against this artifact.
@@ -928,6 +918,73 @@ class Element(Plugin):
return None
+ # _stage_artifact()
+ #
+ # Stage this element's output artifact in the sandbox
+ #
+ # This will stage the files from the artifact to the sandbox at specified location.
+ # The files are selected for staging according to the `include`, `exclude` and `orphans`
+ # parameters; if `include` is not specified then all files spoken for by any domain
+ # are included unless explicitly excluded with an `exclude` domain.
+ #
+ # Args:
+ # sandbox: The build sandbox
+ # path: An optional sandbox relative path
+ # action (OverlapAction): The action to take when overlapping with previous invocations
+ # include: An optional list of domains to include files from
+ # exclude: An optional list of domains to exclude files from
+ # orphans: Whether to include files not spoken for by split domains
+ # owner: The session element currently running Element.stage()
+ #
+ # Raises:
+ # (:class:`.ElementError`): If the element has not yet produced an artifact.
+ #
+ # Returns:
+ # The result describing what happened while staging
+ #
+ def _stage_artifact(
+ self,
+ sandbox: "Sandbox",
+ *,
+ path: str = None,
+ action: str = OverlapAction.WARNING,
+ include: Optional[List[str]] = None,
+ exclude: Optional[List[str]] = None,
+ orphans: bool = True,
+ owner: Optional["Element"] = None,
+ ) -> FileListResult:
+
+ owner = owner or self
+ assert owner._overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()"
+
+ if not self._cached():
+ detail = (
+ "No artifacts have been cached yet for that element\n"
+ + "Try building the element first with `bst build`\n"
+ )
+ raise ElementError("No artifacts to stage", detail=detail, reason="uncached-checkout-attempt")
+
+ # Time to use the artifact, check once more that it's there
+ self.__assert_cached()
+
+ self.status("Staging {}/{}".format(self.name, self._get_brief_display_key()))
+ # Disable type checking since we can't easily tell mypy that
+ # `self.__artifact` can't be None at this stage.
+ files_vdir = self.__artifact.get_files() # type: ignore
+
+ # Hard link it into the staging area
+ #
+ vbasedir = sandbox.get_virtual_directory()
+ vstagedir = vbasedir if path is None else vbasedir.descend(*path.lstrip(os.sep).split(os.sep), create=True)
+
+ split_filter = self.__split_filter_func(include, exclude, orphans)
+
+ result = vstagedir.import_files(files_vdir, filter_callback=split_filter, report_written=True, can_link=True)
+
+ owner._overlap_collector.collect_stage_result(self, result)
+
+ return result
+
# _stage_dependency_artifacts()
#
# Stage element dependencies in scope, this is used for core
@@ -948,13 +1005,9 @@ class Element(Plugin):
# occur.
#
def _stage_dependency_artifacts(self, sandbox, scope, *, path=None, include=None, exclude=None, orphans=True):
- overlaps = _OverlapCollector(self)
-
- for dep in self._dependencies(scope):
- result = dep.stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans)
- overlaps.collect_stage_result(dep, result)
-
- overlaps.overlap_warnings()
+ with self._overlap_collector.session(OverlapAction.WARNING, path):
+ for dep in self._dependencies(scope):
+ dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self)
# _new_from_load_element():
#
@@ -1332,10 +1385,10 @@ class Element(Plugin):
# Stage what we need
if shell and scope == _Scope.BUILD:
- self.stage(sandbox)
+ self.__stage(sandbox)
else:
# Stage deps in the sandbox root
- with self.timed_activity("Staging dependencies", silent_nested=True):
+ with self.timed_activity("Staging dependencies", silent_nested=True), self.__collect_overlaps():
self._stage_dependency_artifacts(sandbox, scope)
# Run any integration commands provided by the dependencies
@@ -1642,7 +1695,7 @@ class Element(Plugin):
# Step 1 - Configure
self.__configure_sandbox(sandbox)
# Step 2 - Stage
- self.stage(sandbox)
+ self.__stage(sandbox)
try:
if self.__batch_prepare_assemble:
cm = sandbox.batch(
@@ -2435,6 +2488,16 @@ class Element(Plugin):
self.configure_sandbox(sandbox)
+ # __stage():
+ #
+ # Internal method for calling public abstract stage() method.
+ #
+ def __stage(self, sandbox):
+
+ # Enable the overlap collector during the staging process
+ with self.__collect_overlaps():
+ self.stage(sandbox)
+
# __prepare():
#
# Internal method for calling public abstract prepare() method.
@@ -2541,6 +2604,20 @@ class Element(Plugin):
def __use_remote_execution(self):
return bool(self.__remote_execution_specs)
+ # __collect_overlaps():
+ #
+ # A context manager for collecting overlap warnings and errors.
+ #
+ # Any places where code might call Element.stage_artifact()
+ # or Element.stage_dependency_artifacts() should be run in
+ # this context manager.
+ #
+ @contextmanager
+ def __collect_overlaps(self):
+ self._overlap_collector = OverlapCollector(self)
+ yield
+ self._overlap_collector = None
+
# __sandbox():
#
# A context manager to prepare a Sandbox object at the specified directory,
@@ -3213,112 +3290,6 @@ class Element(Plugin):
self._update_ready_for_runtime_and_cached()
-# _OverlapCollector()
-#
-# Collects results of Element.stage_artifact() and saves
-# them in order to raise a proper overlap error at the end
-# of staging.
-#
-# Args:
-# element (Element): The element for which we are staging artifacts
-#
-class _OverlapCollector:
- def __init__(self, element):
- self.element = element
-
- # Dictionary of files which were ignored (See FileListResult()), keyed by element unique ID
- self.ignored = {} # type: Dict[int, List[str]]
-
- # Dictionary of files which were staged, keyed by element unique ID
- self.files_written = {} # type: Dict[int, List[str]]
-
- # Dictionary of element IDs which overlapped, keyed by the file they overlap on
- self.overlaps = {} # type: Dict[str, List[int]]
-
- # collect_stage_result()
- #
- # Collect and accumulate results of Element.stage_artifact()
- #
- # Args:
- # element (Element): The name of the element staged
- # result (FileListResult): The result of Element.stage_artifact()
- #
- def collect_stage_result(self, element: Element, result: FileListResult):
- if result.overwritten:
- for overwritten_file in result.overwritten:
- # Completely new overwrite
- if overwritten_file not in self.overlaps:
- # Search for the element we've overwritten in self.written_files
- for element_id, staged_files in self.files_written.items():
- if overwritten_file in staged_files:
- self.overlaps[overwritten_file] = [element_id, element._unique_id]
- break
- # Record the new overwrite in the list
- else:
- self.overlaps[overwritten_file].append(element._unique_id)
-
- self.files_written[element._unique_id] = result.files_written
- if result.ignored:
- self.ignored[element._unique_id] = result.ignored
-
- # overlap_warnings()
- #
- # Issue any warnings as a batch as a result of staging artifacts,
- # based on the results collected with collect_stage_result().
- #
- def overlap_warnings(self):
- if self.overlaps:
- overlap_warning = False
- warning_detail = "Staged files overwrite existing files in staging area:\n"
- for filename, element_ids in self.overlaps.items():
- overlap_warning_elements = []
- # The bottom item overlaps nothing
- overlapping_element_ids = element_ids[1:]
- for element_id in overlapping_element_ids:
- element = Plugin._lookup(element_id)
- if not element._file_is_whitelisted(filename):
- overlap_warning_elements.append(element)
- overlap_warning = True
-
- warning_detail += self._overlap_error_detail(filename, overlap_warning_elements, element_ids)
-
- if overlap_warning:
- self.element.warn(
- "Non-whitelisted overlaps detected", detail=warning_detail, warning_token=CoreWarnings.OVERLAPS
- )
-
- if self.ignored:
- detail = "Not staging files which would replace non-empty directories:\n"
- for element_id, ignored_filenames in self.ignored.items():
- element = Plugin._lookup(element_id)
- detail += "\nFrom {}:\n".format(element._get_full_name())
- detail += " " + " ".join(["/" + filename + "\n" for filename in ignored_filenames])
- self.element.warn("Ignored files", detail=detail)
-
- # _overlap_error_detail()
- #
- # Get a string to describe overlaps on a filename
- #
- # Args:
- # filename (str): The filename being overlapped
- # overlap_elements (List[Element]): A list of Elements overlapping
- # element_ids (List[int]): The ordered ID list of elements which staged this file
- #
- def _overlap_error_detail(self, filename, overlap_elements, element_ids):
- if overlap_elements:
- overlap_element_names = [element._get_full_name() for element in overlap_elements]
- overlap_order_elements = [Plugin._lookup(element_id) for element_id in element_ids]
- overlap_order_names = [element._get_full_name() for element in overlap_order_elements]
- return "/{}: {} {} not permitted to overlap other elements, order {} \n".format(
- filename,
- " and ".join(overlap_element_names),
- "is" if len(overlap_element_names) == 1 else "are",
- " above ".join(reversed(overlap_order_names)),
- )
- else:
- return ""
-
-
# _get_normal_name():
#
# Get the element name without path separators or
[buildstream] 03/09:
tests/elements/filter/basic/element_plugins/dynamic.py: Stage in
Element.stage()
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch tristan/multi-location-overlaps
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit bd23f982be498aae9389ddfb01c3d01c53f14ff0
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Thu Sep 10 21:35:25 2020 +0900
tests/elements/filter/basic/element_plugins/dynamic.py: Stage in Element.stage()
This test element was also staging artifacts in Element.assemble(), which is
now illegal.
---
tests/elements/filter/basic/element_plugins/dynamic.py | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/tests/elements/filter/basic/element_plugins/dynamic.py b/tests/elements/filter/basic/element_plugins/dynamic.py
index 16a6008..401c6b1 100644
--- a/tests/elements/filter/basic/element_plugins/dynamic.py
+++ b/tests/elements/filter/basic/element_plugins/dynamic.py
@@ -20,13 +20,10 @@ class DynamicElement(Element):
pass
def stage(self, sandbox):
- pass
-
- def assemble(self, sandbox):
with self.timed_activity("Staging artifact", silent_nested=True):
- for dep in self.dependencies():
- dep.stage_artifact(sandbox)
+ self.stage_dependency_artifacts(sandbox)
+ def assemble(self, sandbox):
bstdata = self.get_public_data("bst")
bstdata["split-rules"] = self.split_rules
self.set_public_data("bst", bstdata)