You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:05:59 UTC
[buildstream] branch Qinusty/235-manifest created (now 7e0dc95)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch Qinusty/235-manifest
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 7e0dc95 tests: Add tests for build manifests
This branch includes the following new commits:
new 38d5a4c cli.py: Add --build-manifest to build command
new 7e0dc95 tests: Add tests for build manifests
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 02/02: tests: Add tests for build manifests
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch Qinusty/235-manifest
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 7e0dc95aa5a976eb2915b013070521fa958c1551
Author: Josh Smith <jo...@codethink.co.uk>
AuthorDate: Wed Aug 1 17:07:46 2018 +0100
tests: Add tests for build manifests
Tests ensure that a build manifest is created when it should be, and
isn't when it shouldn't be.
---
tests/manifest/manifest.py | 61 +++++++++++++++++++++++++++++
tests/manifest/project/elements/base.bst | 6 +++
tests/manifest/project/files/hello/file.txt | 0
tests/manifest/project/project.conf | 4 ++
4 files changed, 71 insertions(+)
diff --git a/tests/manifest/manifest.py b/tests/manifest/manifest.py
new file mode 100644
index 0000000..dbf7eab
--- /dev/null
+++ b/tests/manifest/manifest.py
@@ -0,0 +1,61 @@
+import pytest
+import os
+from ruamel import yaml
+
+from tests.testutils import cli
+
+# Project directory
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ "project",
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("specify_path, build_manifest", [
+ (True, True), (True, False), (False, True)
+])
+def test_manifest_created(tmpdir, cli, datafiles, specify_path, build_manifest):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ manifest_path = os.path.join(str(tmpdir), "build_manifest.yaml")
+
+ args = ['build', "base.bst"]
+
+ if specify_path:
+ args += ["--manifest-path", manifest_path]
+ if build_manifest:
+ args.append("--build-manifest")
+
+ result = cli.run(project=project, args=args)
+ result.assert_success()
+
+ with open(manifest_path) as f:
+ manifest = yaml.load(f, Loader=yaml.loader.RoundTripLoader)
+
+ assert len(manifest["Elements"]["base"]["Sources"]) == 1
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("extension, valid", [
+ (".yaml", True),
+ (".yml", True),
+ (".bst", False),
+ (".ynl", False),
+ (".xml", False),
+ (".mnf", False),
+ (".txt", False),
+ (".abc", False),
+ (".json", False)
+])
+def test_manifest_extensions(tmpdir, cli, datafiles, extension, valid):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+
+ manifest_path = os.path.join(str(tmpdir), "build_manifest{}" + extension)
+
+ result = cli.run(project=project, args=['build', "base.bst", "--manifest-path", manifest_path])
+
+ if valid:
+ result.assert_success()
+ else:
+ assert result.exit_code == 2
diff --git a/tests/manifest/project/elements/base.bst b/tests/manifest/project/elements/base.bst
new file mode 100644
index 0000000..a41e59a
--- /dev/null
+++ b/tests/manifest/project/elements/base.bst
@@ -0,0 +1,6 @@
+kind: import
+description: Custom foo element
+
+sources:
+ - kind: local
+ path: files/hello
\ No newline at end of file
diff --git a/tests/manifest/project/files/hello/file.txt b/tests/manifest/project/files/hello/file.txt
new file mode 100644
index 0000000..e69de29
diff --git a/tests/manifest/project/project.conf b/tests/manifest/project/project.conf
new file mode 100644
index 0000000..854e386
--- /dev/null
+++ b/tests/manifest/project/project.conf
@@ -0,0 +1,4 @@
+# Project config for frontend build test
+name: test
+
+element-path: elements
[buildstream] 01/02: cli.py: Add --build-manifest to build command
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch Qinusty/235-manifest
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 38d5a4cf24956c4dc573add021f513a4975ff693
Author: Josh Smith <jo...@codethink.co.uk>
AuthorDate: Tue Jul 31 17:47:59 2018 +0100
cli.py: Add --build-manifest to build command
This allows the user to opt in generating a build manifest containing
all of the elements and sources used for their build alongside a buildstream
version and build date/time.
_manifest.py: New module to provide functionality for producing a
manifest from a collection of elements.
source.py: Add _get_element_index() to allow access to
Source.__element_index.
---
buildstream/_frontend/cli.py | 25 ++++++++-
buildstream/_manifest.py | 124 +++++++++++++++++++++++++++++++++++++++++++
buildstream/source.py | 3 ++
3 files changed, 151 insertions(+), 1 deletion(-)
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index 20624e2..b883956 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -3,6 +3,7 @@ import sys
import click
from .. import _yaml
+from .. import _manifest
from .._exceptions import BstError, LoadError, AppError
from .._versions import BST_FORMAT_VERSION
from .complete import main_bashcomplete, complete_path, CompleteUnhandled
@@ -289,6 +290,15 @@ def init(app, project_name, format_version, element_path, force):
##################################################################
# Build Command #
##################################################################
+def _validate_manifest_path(ctx, param, value):
+ if not value:
+ return
+ if value.lower().endswith(".yaml") or value.lower().endswith(".yml"):
+ return os.path.abspath(value)
+ else:
+ raise click.BadParameter("Manifest files are outputted as YAML\n\t" +
+ "Please provide a path with a valid file extension (yml, yaml)")
+
@cli.command(short_help="Build elements in a pipeline")
@click.option('--all', 'all_', default=False, is_flag=True,
help="Build elements that would not be needed for the current build plan")
@@ -305,10 +315,16 @@ def init(app, project_name, format_version, element_path, force):
help="Allow tracking to cross junction boundaries")
@click.option('--track-save', default=False, is_flag=True,
help="Deprecated: This is ignored")
+@click.option('--build-manifest', default=False, is_flag=True,
+ help="Produces a build manifest containing elements and sources.")
+@click.option('--manifest-path', default=None, type=click.Path(readable=False),
+ help="Provides a path for a build manifest to be written to.",
+ callback=_validate_manifest_path)
@click.argument('elements', nargs=-1,
type=click.Path(readable=False))
@click.pass_obj
-def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
+def build(app, elements, all_, track_, track_save, build_manifest,
+ manifest_path, track_all, track_except, track_cross_junctions):
"""Build elements in a pipeline"""
if (track_except or track_cross_junctions) and not (track_ or track_all):
@@ -329,6 +345,13 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
track_cross_junctions=track_cross_junctions,
build_all=all_)
+ if build_manifest and not manifest_path:
+ manifest_path = os.path.join(app.project.directory,
+ "build_manifest.yaml")
+ if manifest_path:
+ _manifest.generate(app.context, app.stream.total_elements,
+ app._session_start, manifest_path)
+
##################################################################
# Fetch Command #
diff --git a/buildstream/_manifest.py b/buildstream/_manifest.py
new file mode 100644
index 0000000..a897866
--- /dev/null
+++ b/buildstream/_manifest.py
@@ -0,0 +1,124 @@
+#
+# Copyright (C) 2018 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+# Josh Smith <jo...@codethink.co.uk>
+
+from ruamel import yaml
+from . import __version__ as bst_version
+from . import _yaml
+from ._message import MessageType, Message
+
+# generate():
+#
+# Generates a manfiest file from the collection of elements.
+#
+# Args:
+# context (Context): Application context
+# elements (list of Element): Collection of elements to build the manifest from
+# build_start_datetime (datetime): The start of the build assosciated with this manifest
+# manifest_path (str): Absolute path to write the manifest file to.
+#
+def generate(context, elements, build_start_datetime, manifest_path):
+ with context.timed_activity("Building Manifest"):
+ manifest = _build(elements, build_start_datetime)
+
+ _yaml.dump(manifest, manifest_path)
+ context.message(Message(None, MessageType.STATUS,
+ "Manifest saved to {}".format(manifest_path)))
+
+# _build():
+#
+# Builds a manifest (dictionary) using the provided elements to be stored.
+#
+# Args:
+# elements (list of Element): Collection of elements to build the manifest from
+# build_start_datetime (datetime): The start of the build assosciated with this manifest
+#
+# Returns:
+# (CommentedMap): A dictionary containing the entire
+# manifest produced from the provided elements.
+#
+def _build(elements, build_start_datetime):
+ manifest = yaml.comments.CommentedMap()
+
+ # Add BuildStream Version
+ manifest['BuildStream_Version'] = "{}".format(bst_version)
+ # Add Build Date
+ manifest['Build_Date'] = build_start_datetime.isoformat()
+ manifest['Elements'] = yaml.comments.CommentedMap()
+
+ # Sort elements
+ elements = sorted(elements, key=lambda e: e.normal_name)
+
+ # Add Elements
+ for elem in elements:
+ manifest['Elements'][elem.normal_name] = _build_element(elem)
+
+ return manifest
+
+# _build_element():
+#
+# Builds a manifest segment for an individual element.
+#
+# Args:
+# element (Element): Element to extract information from
+#
+# Returns:
+# (CommentedMap): A dictionary containing the information
+# extracted from the provided element
+#
+def _build_element(element):
+ element_dict = yaml.comments.CommentedMap()
+ sources = yaml.comments.CommentedMap()
+ # Add Cache Key
+ cache_key = element._get_cache_key()
+ if cache_key:
+ element_dict["Cache_Key"] = cache_key
+
+ # Add sources
+ for source in element.sources():
+ src = _build_source(source)
+ if src:
+ source_desc = "{}({})".format(source._get_element_index(), type(source).__name__)
+ sources[source_desc] = src
+ if sources:
+ element_dict['Sources'] = sources
+
+
+ return element_dict
+
+# _build_source():
+#
+# Builds a manifest segment for an individual source.
+#
+# Args:
+# source (Source): Source to extract information from
+#
+# Returns:
+# (CommentedMap): A dictionary containing the information
+# extracted from the provided source
+#
+def _build_source(source):
+ src = yaml.comments.CommentedMap()
+ if hasattr(source, "url") and source.url:
+ src["url"] = source.url
+ if hasattr(source, "ref") and source.ref:
+ src["ref"] = source.ref
+ if hasattr(source, "path") and source.path:
+ src["path"] = source.path
+
+ return src if src else None
diff --git a/buildstream/source.py b/buildstream/source.py
index 2f3f1c2..8f45ce0 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -786,6 +786,9 @@ class Source(Plugin):
else:
return None
+ def _get_element_index(self):
+ return self.__element_index
+
#############################################################
# Local Private Methods #
#############################################################