You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/03/30 05:40:46 UTC
[buildstream] 16/37: tests/plugins/sample-plugins: Adding git source to sample plugins
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/remove-plugins
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit d178798c4189a5a735183fafc514065165c17155
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Tue Mar 29 15:49:07 2022 +0900
tests/plugins/sample-plugins: Adding git source to sample plugins
---
tests/plugins/sample-plugins/setup.py | 1 +
.../src/sample_plugins/sources/git.py | 1059 ++++++++++++++++++++
2 files changed, 1060 insertions(+)
diff --git a/tests/plugins/sample-plugins/setup.py b/tests/plugins/sample-plugins/setup.py
index 070f8c7..0db7481 100755
--- a/tests/plugins/sample-plugins/setup.py
+++ b/tests/plugins/sample-plugins/setup.py
@@ -32,6 +32,7 @@ setup(
],
"buildstream.plugins.sources": [
"sample = sample_plugins.sources.sample",
+ "git = sample_plugins.sources.git",
],
},
zip_safe=False,
diff --git a/tests/plugins/sample-plugins/src/sample_plugins/sources/git.py b/tests/plugins/sample-plugins/src/sample_plugins/sources/git.py
new file mode 100644
index 0000000..decd2ec
--- /dev/null
+++ b/tests/plugins/sample-plugins/src/sample_plugins/sources/git.py
@@ -0,0 +1,1059 @@
+#
+# Copyright (C) 2016 Codethink Limited
+# Copyright (C) 2018 Bloomberg Finance LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# Tristan Van Berkom <tr...@codethink.co.uk>
+# Chandan Singh <cs...@bloomberg.net>
+# Tom Mewett <to...@codethink.co.uk>
+
+"""
+git - stage files from a git repository
+=======================================
+
+**Host dependencies:**
+
+ * git
+
+.. attention::
+
+ Note that this plugin **will checkout git submodules by default**; even if
+ they are not specified in the `.bst` file.
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the git source kind
+ kind: git
+
+ # Specify the repository url, using an alias defined
+ # in your project configuration is recommended.
+ url: upstream:foo.git
+
+ # Optionally specify a symbolic tracking branch or tag, this
+ # will be used to update the 'ref' when refreshing the pipeline.
+ track: master
+
+ # Optionally specify the ref format used for tracking.
+ # The default is 'sha1' for the raw commit hash.
+ # If you specify 'git-describe', the commit hash will be prefixed
+ # with the closest tag.
+ ref-format: sha1
+
+ # Specify the commit ref, this must be specified in order to
+ # checkout sources and build, but can be automatically updated
+ # if the 'track' attribute was specified.
+ ref: d63cbb6fdc0bbdadc4a1b92284826a6d63a7ebcd
+
+ # Optionally specify whether submodules should be checked-out.
+ # This is done recursively, as with `git clone --recurse-submodules`.
+ # If not set, this will default to 'True'
+ checkout-submodules: True
+
+ # If your repository has submodules, explicitly specifying the
+ # url from which they are to be fetched allows you to easily
+ # rebuild the same sources from a different location. This is
+ # especially handy when used with project defined aliases which
+ # can be redefined at a later time.
+ # You may also explicitly specify whether to check out this
+ # submodule. If 'checkout' is set, it will control whether to
+ # checkout that submodule and recurse into it. It defaults to the
+ # value of 'checkout-submodules'.
+ submodules:
+ plugins/bar:
+ url: upstream:bar.git
+ checkout: True
+ plugins/bar/quux:
+ checkout: False
+ plugins/baz:
+ url: upstream:baz.git
+ checkout: False
+
+ # Enable tag tracking.
+ #
+ # This causes the `tags` metadata to be populated automatically
+ # as a result of tracking the git source.
+ #
+ # By default this is 'False'.
+ #
+ track-tags: True
+
+ # If the list of tags below is set, then a lightweight dummy
+ # git repository will be staged along with the content at
+ # build time.
+ #
+ # This is useful for a growing number of modules which use
+ # `git describe` at build time in order to determine the version
+ # which will be encoded into the built software.
+ #
+ # The 'tags' below is considered as a part of the git source
+ # reference and will be stored in the 'project.refs' file if
+ # that has been selected as your project's ref-storage.
+ #
+ # Migration notes:
+ #
+ # If you are upgrading from BuildStream 1.2, which used to
+ # stage the entire repository by default, you will notice that
+ # some modules which use `git describe` are broken, and will
+ # need to enable this feature in order to fix them.
+ #
+ # If you need to enable this feature without changing the
+ # the specific commit that you are building, then we recommend
+ # the following migration steps for any git sources where
+ # `git describe` is required:
+ #
+ # o Enable `track-tags` feature
+ # o Set the `track` parameter to the desired commit sha which
+ # the current `ref` points to
+ # o Run `bst source track` for these elements, this will result in
+ # populating the `tags` portion of the refs without changing
+ # the refs
+ # o Restore the `track` parameter to the branches which you have
+ # previously been tracking afterwards.
+ #
+ tags:
+ - tag: lightweight-example
+ commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78
+ annotated: false
+ - tag: annotated-example
+ commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a
+ annotated: true
+
+See `built-in functionality doumentation
+<https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
+details on common configuration options for sources.
+
+
+**Configurable Warnings:**
+
+This plugin provides the following
+`configurable warnings <https://docs.buildstream.build/master/format_project.html#configurable-warnings>`_:
+
+- ``git:inconsistent-submodule`` - A submodule present in the git repository's .gitmodules was never
+ added with `git submodule add`.
+
+- ``git:unlisted-submodule`` - A submodule is present in the git repository but was not specified in
+ the source configuration and was not disabled for checkout.
+
+- ``git:invalid-submodule`` - A submodule is specified in the source configuration but does not exist
+ in the repository.
+
+This plugin also utilises the following configurable
+`core warnings <https://docs.buildstream.build/master/buildstream.types.html#buildstream.types.CoreWarnings>`_:
+
+- `ref-not-in-track <https://docs.buildstream.build/master/buildstream.types.html#buildstream.types.CoreWarnings.REF_NOT_IN_TRACK>`_ -
+ The provided ref was not found in the provided track in the element's git repository.
+"""
+
+
+import os
+import re
+import shutil
+from io import StringIO
+from tempfile import TemporaryFile
+
+from configparser import RawConfigParser
+
+from buildstream import Source, SourceError, SourceFetcher
+from buildstream import CoreWarnings, FastEnum
+from buildstream import utils
+from buildstream.utils import DirectoryExistsError
+
+GIT_MODULES = ".gitmodules"
+EXACT_TAG_PATTERN = r"(?P<tag>.*)-0-g(?P<commit>.*)"
+
+# Warnings
+WARN_INCONSISTENT_SUBMODULE = "inconsistent-submodule"
+WARN_UNLISTED_SUBMODULE = "unlisted-submodule"
+WARN_INVALID_SUBMODULE = "invalid-submodule"
+
+
+class _RefFormat(FastEnum):
+ SHA1 = "sha1"
+ GIT_DESCRIBE = "git-describe"
+
+
+def _strip_tag(rev):
+ return rev.split("-g")[-1]
+
+
+# This class represents a single Git repository. The Git source needs to account for
+# submodules, but we don't want to cache them all under the umbrella of the
+# superproject - so we use this class which caches them independently, according
+# to their URL. Instances keep reference to their "parent" GitSource,
+# and if applicable, where in the superproject they are found.
+#
+# Args:
+# source (GitSource): The parent source
+# path (str): The relative location of the submodule in the superproject;
+# the empty string for the superproject itself
+# url (str): Where to clone the repo from
+# ref (str): Specified 'ref' from the source configuration
+# primary (bool): Whether this is the primary URL for the source
+# tags (list): Tag configuration; see GitSource._load_tags
+#
+class GitMirror(SourceFetcher):
+ def __init__(self, source, path, url, ref, *, primary=False, tags=None):
+
+ super().__init__()
+ self.source = source
+ self.path = path
+ self.url = url
+ self.ref = ref
+ self.tags = tags or []
+ self.primary = primary
+ self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
+
+ # _ensure_repo():
+ #
+ # Ensures that the Git repository exists at the mirror location and is configured
+ # to fetch from the given URL
+ #
+ def _ensure_repo(self):
+ if not os.path.exists(self.mirror):
+ with self.source.tempdir() as tmpdir:
+ self.source.call(
+ [self.source.host_git, "init", "--bare", tmpdir],
+ fail="Failed to initialise repository",
+ )
+
+ try:
+ utils.move_atomic(tmpdir, self.mirror)
+ except DirectoryExistsError:
+ # Another process was quicker to download this repository.
+ # Let's discard our own
+ self.source.status("{}: Discarding duplicate repository".format(self.source))
+ except OSError as e:
+ raise SourceError(
+ "{}: Failed to move created repository from '{}' to '{}': {}".format(
+ self.source, tmpdir, self.mirror, e
+ )
+ ) from e
+
+ def _fetch(self, url, fetch_all=False):
+ self._ensure_repo()
+
+ # Work out whether we can fetch a specific tag: are we given a ref which
+ # 1. is in git-describe format
+ # 2. refers to an exact tag (is "...-0-g...")
+ # 3. is available on the remote and tags the specified commit?
+ # And lastly: are we on a new-enough Git which allows cloning from our potentially shallow cache?
+ if fetch_all:
+ pass
+ # Fetching from a shallow-cloned repo was first supported in v1.9.0
+ elif not self.ref or self.source.host_git_version is not None and self.source.host_git_version < (1, 9, 0):
+ fetch_all = True
+ else:
+ m = re.match(EXACT_TAG_PATTERN, self.ref)
+ if m is None:
+ fetch_all = True
+ else:
+ tag = m.group("tag")
+ commit = m.group("commit")
+
+ if not self.remote_has_tag(url, tag, commit):
+ self.source.status(
+ "{}: {} is not advertised on {}. Fetching all Git refs".format(self.source, self.ref, url)
+ )
+ fetch_all = True
+ else:
+ exit_code = self.source.call(
+ [
+ self.source.host_git,
+ "fetch",
+ "--depth=1",
+ url,
+ "+refs/tags/{tag}:refs/tags/{tag}".format(tag=tag),
+ ],
+ cwd=self.mirror,
+ )
+ if exit_code != 0:
+ self.source.status(
+ "{}: Failed to fetch tag '{}' from {}. Fetching all Git refs".format(self.source, tag, url)
+ )
+ fetch_all = True
+
+ if fetch_all:
+ self.source.call(
+ [
+ self.source.host_git,
+ "fetch",
+ "--prune",
+ url,
+ "+refs/heads/*:refs/heads/*",
+ "+refs/tags/*:refs/tags/*",
+ ],
+ fail="Failed to fetch from remote git repository: {}".format(url),
+ fail_temporarily=True,
+ cwd=self.mirror,
+ )
+
+ def fetch(self, alias_override=None): # pylint: disable=arguments-differ
+ resolved_url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
+
+ with self.source.timed_activity("Fetching from {}".format(resolved_url), silent_nested=True):
+ if not self.has_ref():
+ self._fetch(resolved_url)
+ self.assert_ref()
+
+ def has_ref(self):
+ if not self.ref:
+ return False
+
+ # If the mirror doesnt exist, we also dont have the ref
+ if not os.path.exists(self.mirror):
+ return False
+
+ # Check if the ref is really there
+ rc = self.source.call([self.source.host_git, "cat-file", "-t", self.ref], cwd=self.mirror)
+ return rc == 0
+
+ def assert_ref(self):
+ if not self.has_ref():
+ raise SourceError(
+ "{}: expected ref '{}' was not found in git repository: '{}'".format(self.source, self.ref, self.url)
+ )
+
+ # remote_has_tag():
+ #
+ # Args:
+ # url (str)
+ # tag (str)
+ # commit (str)
+ #
+ # Returns:
+ # (bool): Whether the remote at `url` has the tag `tag` attached to `commit`
+ #
+ def remote_has_tag(self, url, tag, commit):
+ _, ls_remote = self.source.check_output(
+ [self.source.host_git, "ls-remote", url],
+ cwd=self.mirror,
+ fail="Failed to list advertised remote refs from git repository {}".format(url),
+ )
+
+ line = "{commit}\trefs/tags/{tag}".format(commit=commit, tag=tag)
+ return line in ls_remote or line + "^{}" in ls_remote
+
+ # to_commit():
+ #
+ # Args:
+ # rev (str): A Git "commit-ish" rev
+ #
+ # Returns:
+ # (str): The full revision ID of the commit
+ #
+ def to_commit(self, rev):
+ _, output = self.source.check_output(
+ [self.source.host_git, "rev-list", "-n", "1", rev],
+ fail="Unable to find revision {}".format(rev),
+ cwd=self.mirror,
+ )
+
+ return output.strip()
+
+ # describe():
+ #
+ # Args:
+ # rev (str): A Git "commit-ish" rev
+ #
+ # Returns:
+ # (str): The full revision ID of the commit given by rev, prepended
+ # with tag information as given by git-describe (where available)
+ #
+ def describe(self, rev):
+ _, output = self.source.check_output(
+ [
+ self.source.host_git,
+ "describe",
+ "--tags",
+ "--abbrev=40",
+ "--long",
+ "--always",
+ rev,
+ ],
+ fail="Unable to find revision {}".format(rev),
+ cwd=self.mirror,
+ )
+
+ return output.strip()
+
+ # reachable_tags():
+ #
+ # Args:
+ # rev (str): A Git "commit-ish" rev
+ #
+ # Returns:
+ # (list): A list of tags in the ancestry of rev. Each entry is a triple of the form
+ # (tag name (str), commit ref (str), annotated (bool)) describing a tag,
+ # its tagged commit and whether it's annotated
+ #
+ def reachable_tags(self, rev):
+ tags = set()
+ for options in [
+ [],
+ ["--first-parent"],
+ ["--tags"],
+ ["--tags", "--first-parent"],
+ ]:
+ exit_code, output = self.source.check_output(
+ [
+ self.source.host_git,
+ "describe",
+ "--abbrev=0",
+ rev,
+ *options,
+ ],
+ cwd=self.mirror,
+ )
+ if exit_code == 0:
+ tag = output.strip()
+ _, commit_ref = self.source.check_output(
+ [self.source.host_git, "rev-parse", tag + "^{commit}"],
+ fail="Unable to resolve tag '{}'".format(tag),
+ cwd=self.mirror,
+ )
+ exit_code = self.source.call(
+ [self.source.host_git, "cat-file", "tag", tag],
+ cwd=self.mirror,
+ )
+ annotated = exit_code == 0
+
+ tags.add((tag, commit_ref.strip(), annotated))
+
+ return list(tags)
+
+ def stage(self, directory):
+ fullpath = os.path.join(directory, self.path)
+
+ # Using --shared here avoids copying the objects into the checkout, in any
+ # case we're just checking out a specific commit and then removing the .git/
+ # directory.
+ self.source.call(
+ [
+ self.source.host_git,
+ "clone",
+ "--no-checkout",
+ "--shared",
+ self.mirror,
+ fullpath,
+ ],
+ fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
+ fail_temporarily=True,
+ )
+
+ self.source.call(
+ [self.source.host_git, "checkout", "--force", self.ref],
+ fail="Failed to checkout git ref {}".format(self.ref),
+ cwd=fullpath,
+ )
+
+ # Remove .git dir
+ shutil.rmtree(os.path.join(fullpath, ".git"))
+
+ self._rebuild_git(fullpath)
+
+ def init_workspace(self, directory):
+ fullpath = os.path.join(directory, self.path)
+ url = self.source.translate_url(self.url)
+
+ self.source.call(
+ [
+ self.source.host_git,
+ "clone",
+ "--no-checkout",
+ self.mirror,
+ fullpath,
+ ],
+ fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath),
+ fail_temporarily=True,
+ )
+
+ self.source.call(
+ [self.source.host_git, "remote", "set-url", "origin", url],
+ fail='Failed to add remote origin "{}"'.format(url),
+ cwd=fullpath,
+ )
+
+ self.source.call(
+ [self.source.host_git, "checkout", "--force", self.ref],
+ fail="Failed to checkout git ref {}".format(self.ref),
+ cwd=fullpath,
+ )
+
+ # get_submodule_mirrors():
+ #
+ # Returns:
+ # An iterator through new instances of this class, one of each submodule
+ # in the repo
+ #
+ def get_submodule_mirrors(self):
+ for path, url in self.submodule_list():
+ ref = self.submodule_ref(path)
+ if ref is not None:
+ mirror = self.__class__(self.source, os.path.join(self.path, path), url, ref)
+ yield mirror
+
+ # List the submodules (path/url tuples) present at the given ref of this repo
+ def submodule_list(self):
+ modules = "{}:{}".format(self.ref, GIT_MODULES)
+ exit_code, output = self.source.check_output([self.source.host_git, "show", modules], cwd=self.mirror)
+
+ # If git show reports error code 128 here, we take it to mean there is
+ # no .gitmodules file to display for the given revision.
+ if exit_code == 128:
+ return
+ elif exit_code != 0:
+ raise SourceError("{plugin}: Failed to show gitmodules at ref {ref}".format(plugin=self, ref=self.ref))
+
+ content = "\n".join([l.strip() for l in output.splitlines()])
+
+ io = StringIO(content)
+ parser = RawConfigParser()
+ parser.read_file(io)
+
+ for section in parser.sections():
+ # validate section name against the 'submodule "foo"' pattern
+ if re.match(r'submodule "(.*)"', section):
+ path = parser.get(section, "path")
+ url = parser.get(section, "url")
+
+ yield (path, url)
+
+ # Fetch the ref which this mirror requires its submodule to have,
+ # at the given ref of this mirror.
+ def submodule_ref(self, submodule, ref=None):
+ if not ref:
+ ref = self.ref
+
+ # list objects in the parent repo tree to find the commit
+ # object that corresponds to the submodule
+ _, output = self.source.check_output(
+ [self.source.host_git, "ls-tree", ref, submodule],
+ fail="ls-tree failed for commit {} and submodule: {}".format(ref, submodule),
+ cwd=self.mirror,
+ )
+
+ # read the commit hash from the output
+ fields = output.split()
+ if len(fields) >= 2 and fields[1] == "commit":
+ submodule_commit = output.split()[2]
+
+ # fail if the commit hash is invalid
+ if len(submodule_commit) != 40:
+ raise SourceError(
+ "{}: Error reading commit information for submodule '{}'".format(self.source, submodule)
+ )
+
+ return submodule_commit
+
+ else:
+ detail = (
+ "The submodule '{}' is defined either in the BuildStream source\n".format(submodule)
+ + "definition, or in a .gitmodules file. But the submodule was never added to the\n"
+ + "underlying git repository with `git submodule add`."
+ )
+
+ self.source.warn(
+ "{}: Ignoring inconsistent submodule '{}'".format(self.source, submodule),
+ detail=detail,
+ warning_token=WARN_INCONSISTENT_SUBMODULE,
+ )
+
+ return None
+
+ def _rebuild_git(self, fullpath):
+ if not self.tags:
+ return
+
+ with self.source.tempdir() as tmpdir:
+ included = set()
+ shallow = set()
+ for _, commit_ref, _ in self.tags:
+
+ if commit_ref == self.ref:
+ # rev-list does not work in case of same rev
+ shallow.add(self.ref)
+ else:
+ _, out = self.source.check_output(
+ [
+ self.source.host_git,
+ "rev-list",
+ "--ancestry-path",
+ "--boundary",
+ "{}..{}".format(commit_ref, self.ref),
+ ],
+ fail="Failed to get git history {}..{} in directory: {}".format(
+ commit_ref, self.ref, fullpath
+ ),
+ fail_temporarily=True,
+ cwd=self.mirror,
+ )
+ self.source.warn("refs {}..{}: {}".format(commit_ref, self.ref, out.splitlines()))
+ for line in out.splitlines():
+ rev = line.lstrip("-")
+ if line[0] == "-":
+ shallow.add(rev)
+ else:
+ included.add(rev)
+
+ shallow -= included
+ included |= shallow
+
+ self.source.call(
+ [self.source.host_git, "init"],
+ fail="Cannot initialize git repository: {}".format(fullpath),
+ cwd=fullpath,
+ )
+
+ for rev in included:
+ with TemporaryFile(dir=tmpdir) as commit_file:
+ self.source.call(
+ [self.source.host_git, "cat-file", "commit", rev],
+ stdout=commit_file,
+ fail="Failed to get commit {}".format(rev),
+ cwd=self.mirror,
+ )
+ commit_file.seek(0, 0)
+ self.source.call(
+ [
+ self.source.host_git,
+ "hash-object",
+ "-w",
+ "-t",
+ "commit",
+ "--stdin",
+ ],
+ stdin=commit_file,
+ fail="Failed to add commit object {}".format(rev),
+ cwd=fullpath,
+ )
+
+ with open(
+ os.path.join(fullpath, ".git", "shallow"),
+ "w",
+ encoding="utf-8",
+ ) as shallow_file:
+ for rev in shallow:
+ shallow_file.write("{}\n".format(rev))
+
+ for tag, commit_ref, annotated in self.tags:
+ if annotated:
+ with TemporaryFile(dir=tmpdir) as tag_file:
+ tag_data = "object {}\ntype commit\ntag {}\n".format(commit_ref, tag)
+ tag_file.write(tag_data.encode("ascii"))
+ tag_file.seek(0, 0)
+ _, tag_ref = self.source.check_output(
+ [
+ self.source.host_git,
+ "hash-object",
+ "-w",
+ "-t",
+ "tag",
+ "--stdin",
+ ],
+ stdin=tag_file,
+ fail="Failed to add tag object {}".format(tag),
+ cwd=fullpath,
+ )
+
+ self.source.call(
+ [self.source.host_git, "tag", tag, tag_ref.strip()],
+ fail="Failed to tag: {}".format(tag),
+ cwd=fullpath,
+ )
+ else:
+ self.source.call(
+ [self.source.host_git, "tag", tag, commit_ref],
+ fail="Failed to tag: {}".format(tag),
+ cwd=fullpath,
+ )
+
+ with open(os.path.join(fullpath, ".git", "HEAD"), "w", encoding="utf-8") as head:
+ self.source.call(
+ [self.source.host_git, "rev-parse", self.ref],
+ stdout=head,
+ fail="Failed to parse commit {}".format(self.ref),
+ cwd=self.mirror,
+ )
+
+
+class GitSource(Source):
+ # pylint: disable=attribute-defined-outside-init
+
+ BST_MIN_VERSION = "2.0"
+
+ def configure(self, node):
+ ref = node.get_str("ref", None)
+
+ config_keys = [
+ "url",
+ "track",
+ "ref",
+ "submodules",
+ "checkout-submodules",
+ "ref-format",
+ "track-tags",
+ "tags",
+ ]
+ node.validate_keys(config_keys + Source.COMMON_CONFIG_KEYS)
+
+ tags_node = node.get_sequence("tags", [])
+ for tag_node in tags_node:
+ tag_node.validate_keys(["tag", "commit", "annotated"])
+
+ tags = self._load_tags(node)
+ self.track_tags = node.get_bool("track-tags", default=False)
+
+ self.original_url = node.get_str("url")
+ self.mirror = GitMirror(self, "", self.original_url, ref, tags=tags, primary=True)
+ self.tracking = node.get_str("track", None)
+
+ self.ref_format = node.get_enum("ref-format", _RefFormat, _RefFormat.SHA1)
+
+ # At this point we now know if the source has a ref and/or a track.
+ # If it is missing both then we will be unable to track or build.
+ if self.mirror.ref is None and self.tracking is None:
+ raise SourceError(
+ "{}: Git sources require a ref and/or track".format(self),
+ reason="missing-track-and-ref",
+ )
+
+ self.checkout_submodules = node.get_bool("checkout-submodules", default=True)
+
+ # Parse a dict of submodule overrides, stored in the submodule_overrides
+ # and submodule_checkout_overrides dictionaries.
+ self.submodule_overrides = {}
+ self.submodule_checkout_overrides = {}
+ modules = node.get_mapping("submodules", {})
+ for path in modules.keys():
+ submodule = modules.get_mapping(path)
+ url = submodule.get_str("url", None)
+
+ # Make sure to mark all URLs that are specified in the configuration
+ if url:
+ self.mark_download_url(url, primary=False)
+
+ self.submodule_overrides[path] = url
+ if "checkout" in submodule:
+ checkout = submodule.get_bool("checkout")
+ self.submodule_checkout_overrides[path] = checkout
+
+ self.mark_download_url(self.original_url)
+
+ def preflight(self):
+ # Check if git is installed, get the binary at the same time
+ self.host_git = utils.get_host_tool("git")
+
+ rc, version_str = self.check_output([self.host_git, "--version"])
+ # e.g. on Git for Windows we get "git version 2.21.0.windows.1".
+ # e.g. on Mac via Homebrew we get "git version 2.19.0".
+ if rc == 0:
+ self.host_git_version = tuple(int(x) for x in version_str.split(" ")[2].split(".")[:3])
+ else:
+ self.host_git_version = None
+
+ def get_unique_key(self):
+ ref = self.mirror.ref
+ if ref is not None:
+ # Strip any (arbitary) tag information, leaving just the commit ID
+ ref = _strip_tag(ref)
+
+ # Here we want to encode the local name of the repository and
+ # the ref, if the user changes the alias to fetch the same sources
+ # from another location, it should not affect the cache key.
+ key = [self.original_url, ref]
+ if self.mirror.tags:
+ tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags}
+ key.append({"tags": tags})
+
+ # Only modify the cache key with checkout_submodules if it's something
+ # other than the default behaviour.
+ if self.checkout_submodules is False:
+ key.append({"checkout_submodules": self.checkout_submodules})
+
+ # We want the cache key to change if the source was
+ # configured differently, and submodules count.
+ if self.submodule_overrides:
+ key.append(self.submodule_overrides)
+
+ if self.submodule_checkout_overrides:
+ key.append({"submodule_checkout_overrides": self.submodule_checkout_overrides})
+
+ return key
+
+ def is_resolved(self):
+ return self.mirror.ref is not None
+
+ def is_cached(self):
+ return self._have_all_refs()
+
+ def load_ref(self, node):
+ self.mirror.ref = node.get_str("ref", None)
+ self.mirror.tags = self._load_tags(node)
+
+ def get_ref(self):
+ if self.mirror.ref is None:
+ return None
+ return self.mirror.ref, self.mirror.tags
+
+ def set_ref(self, ref, node):
+ if not ref:
+ self.mirror.ref = None
+ if "ref" in node:
+ del node["ref"]
+ self.mirror.tags = []
+ if "tags" in node:
+ del node["tags"]
+ else:
+ actual_ref, tags = ref
+ node["ref"] = self.mirror.ref = actual_ref
+ self.mirror.tags = tags
+ if tags:
+ node["tags"] = []
+ for tag, commit_ref, annotated in tags:
+ data = {
+ "tag": tag,
+ "commit": commit_ref,
+ "annotated": annotated,
+ }
+ node["tags"].append(data)
+ else:
+ if "tags" in node:
+ del node["tags"]
+
+ def track(self): # pylint: disable=arguments-differ
+
+ # If self.tracking is not specified it's not an error, just silently return
+ if not self.tracking:
+ # Is there a better way to check if a ref is given.
+ if self.mirror.ref is None:
+ detail = "Without a tracking branch ref can not be updated. Please " + "provide a ref or a track."
+ raise SourceError(
+ "{}: No track or ref".format(self),
+ detail=detail,
+ reason="track-attempt-no-track",
+ )
+ return None
+
+ # Resolve the URL for the message
+ resolved_url = self.translate_url(self.mirror.url)
+ with self.timed_activity(
+ "Tracking {} from {}".format(self.tracking, resolved_url),
+ silent_nested=True,
+ ):
+ self.mirror._fetch(resolved_url, fetch_all=True)
+
+ ref = self.mirror.to_commit(self.tracking)
+ tags = self.mirror.reachable_tags(ref) if self.track_tags else []
+
+ if self.ref_format == _RefFormat.GIT_DESCRIBE:
+ ref = self.mirror.describe(ref)
+
+ return ref, tags
+
+ def init_workspace(self, directory):
+ with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
+ self.mirror.init_workspace(directory)
+ for mirror in self._recurse_submodules(configure=True):
+ mirror.init_workspace(directory)
+
+ def stage(self, directory):
+ # Stage the main repo in the specified directory
+ #
+ with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
+ self.mirror.stage(directory)
+ for mirror in self._recurse_submodules(configure=True):
+ mirror.stage(directory)
+
+ def get_source_fetchers(self):
+ self.mirror.mark_download_url(self.mirror.url)
+ yield self.mirror
+ # _recurse_submodules only iterates those which are known at the current
+ # cached state - but fetch is called on each result as we go, so this will
+ # yield all configured submodules
+ for submodule in self._recurse_submodules(configure=True):
+ submodule.mark_download_url(submodule.url)
+ yield submodule
+
+ def validate_cache(self):
+ discovered_submodules = {}
+ unlisted_submodules = []
+ invalid_submodules = []
+
+ for submodule in self._recurse_submodules(configure=False):
+ discovered_submodules[submodule.path] = submodule.url
+ if self._ignoring_submodule(submodule.path):
+ continue
+
+ if submodule.path not in self.submodule_overrides:
+ unlisted_submodules.append((submodule.path, submodule.url))
+
+ # Warn about submodules which are explicitly configured but do not exist
+ for path, url in self.submodule_overrides.items():
+ if path not in discovered_submodules:
+ invalid_submodules.append((path, url))
+
+ if invalid_submodules:
+ detail = []
+ for path, url in invalid_submodules:
+ detail.append(" Submodule URL '{}' at path '{}'".format(url, path))
+
+ self.warn(
+ "{}: Invalid submodules specified".format(self),
+ warning_token=WARN_INVALID_SUBMODULE,
+ detail="The following submodules are specified in the source "
+ "description but do not exist according to the repository\n\n" + "\n".join(detail),
+ )
+
+ # Warn about submodules which exist but have not been explicitly configured
+ if unlisted_submodules:
+ detail = []
+ for path, url in unlisted_submodules:
+ detail.append(" Submodule URL '{}' at path '{}'".format(url, path))
+
+ self.warn(
+ "{}: Unlisted submodules exist".format(self),
+ warning_token=WARN_UNLISTED_SUBMODULE,
+ detail="The following submodules exist but are not specified "
+ + "in the source description\n\n"
+ + "\n".join(detail),
+ )
+
+ # Assert that the ref exists in the track tag/branch, if track has been specified.
+ # Also don't do this check if an exact tag ref is given, as we probably didn't fetch
+ # any branch refs
+ ref_in_track = False
+ if not re.match(EXACT_TAG_PATTERN, self.mirror.ref) and self.tracking:
+ _, branch = self.check_output(
+ [
+ self.host_git,
+ "branch",
+ "--list",
+ self.tracking,
+ "--contains",
+ self.mirror.ref,
+ ],
+ cwd=self.mirror.mirror,
+ )
+ if branch:
+ ref_in_track = True
+ else:
+ _, tag = self.check_output(
+ [
+ self.host_git,
+ "tag",
+ "--list",
+ self.tracking,
+ "--contains",
+ self.mirror.ref,
+ ],
+ cwd=self.mirror.mirror,
+ )
+ if tag:
+ ref_in_track = True
+
+ if not ref_in_track:
+ detail = (
+ "The ref provided for the element does not exist locally "
+ + "in the provided track branch / tag '{}'.\n".format(self.tracking)
+ + "You may wish to track the element to update the ref from '{}' ".format(self.tracking)
+ + "with `bst source track`,\n"
+ + "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url)
+ )
+
+ self.warn(
+ "{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n".format(
+ self, self.mirror.ref, self.tracking, self.mirror.url
+ ),
+ detail=detail,
+ warning_token=CoreWarnings.REF_NOT_IN_TRACK,
+ )
+
+ ###########################################################
+ # Local Functions #
+ ###########################################################
+
+ def _have_all_refs(self):
+ return self.mirror.has_ref() and all(
+ submodule.has_ref() for submodule in self._recurse_submodules(configure=True)
+ )
+
+ # _configure_submodules():
+ #
+ # Args:
+ # submodules: An iterator of GitMirror (or similar) objects for submodules
+ #
+ # Returns:
+ # An iterator through `submodules` but filtered of any ignored submodules
+ # and modified to use any custom URLs configured in the source
+ #
+ def _configure_submodules(self, submodules):
+ for submodule in submodules:
+ if self._ignoring_submodule(submodule.path):
+ continue
+ # Allow configuration to override the upstream location of the submodules.
+ submodule.url = self.submodule_overrides.get(submodule.path, submodule.url)
+ yield submodule
+
+ # _recurse_submodules():
+ #
+ # Recursively iterates through GitMirrors for submodules of the main repo. Only
+ # submodules that are cached are recursed into - but this is decided at
+ # iteration time, so you can fetch in a for loop over this function to fetch
+ # all submodules.
+ #
+ # Args:
+ # configure (bool): Whether to apply the 'submodule' config while recursing
+ # (URL changing and 'checkout' overrides)
+ #
+ def _recurse_submodules(self, configure):
+ def recurse(mirror):
+ submodules = mirror.get_submodule_mirrors()
+ if configure:
+ submodules = self._configure_submodules(submodules)
+
+ for submodule in submodules:
+ yield submodule
+ if submodule.has_ref():
+ yield from recurse(submodule)
+
+ yield from recurse(self.mirror)
+
+ def _load_tags(self, node):
+ tags = []
+ tags_node = node.get_sequence("tags", [])
+ for tag_node in tags_node:
+ tag = tag_node.get_str("tag")
+ commit_ref = tag_node.get_str("commit")
+ annotated = tag_node.get_bool("annotated")
+ tags.append((tag, commit_ref, annotated))
+ return tags
+
+ # _ignoring_submodule():
+ #
+ # Args:
+ # path (str): The path of a submodule in the superproject
+ #
+ # Returns:
+ # (bool): Whether to not clone/checkout this submodule
+ #
+ def _ignoring_submodule(self, path):
+ return not self.submodule_checkout_overrides.get(path, self.checkout_submodules)
+
+
+# Plugin entry point
+def setup():
+ return GitSource