You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:49:38 UTC
[buildstream] branch tmewett/merge-git-tag created (now 0182080)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 0182080 Document my changes to the Git source plugin in NEWS
This branch includes the following new commits:
new 417c182 _gitsourcebase.py: Refactor fetching and tracking
new c171026 _gitsourcebase.py: Fetch with depth=1 when an available tag is given
new 410a279 _gitsourcebase.py: Strip git-describe tag info from cache key
new 4b63d15 _gitsourcebase.py: Refactor latest_commit_with_tags
new c3f1a1f repo/git.py: Add optional date argument to add_commit
new cb96eb8 repo/git.py: Remove PWD from test Git environment
new 946fb13 _gitsourcebase.py: Add support for tracking multiple branches
new 4edac5d Add Git source test for tracking multiple branches
new 31a0ff6 sources/git.py: Document tracking multiple branches
new 6e34512 _gitsourcebase.py: Add 'track-latest-tag' option
new 5125ef4 Add Git source tests for tracking latest tag
new ad9cfda sources/git.py: Document tracking latest tag
new 0182080 Document my changes to the Git source plugin in NEWS
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 06/13: repo/git.py: Remove PWD from test Git
environment
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit cb96eb82df412e348539b514edc6e84af053f648
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Thu Jan 9 14:05:08 2020 +0000
repo/git.py: Remove PWD from test Git environment
It is not used by Git or any of the tests.
---
tests/testutils/repo/git.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index 779eb76..14eb6f4 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -23,8 +23,7 @@ class Git(Repo):
def _run_git(self, *args, **kwargs):
argv = [GIT]
argv.extend(args)
- if "env" not in kwargs:
- kwargs["env"] = dict(self.env, PWD=self.repo)
+ kwargs.setdefault("env", self.env)
kwargs.setdefault("cwd", self.repo)
kwargs.setdefault("check", True)
return subprocess.run(argv, **kwargs) # pylint: disable=subprocess-run-check
[buildstream] 09/13: sources/git.py: Document tracking multiple
branches
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 31a0ff61f13d20ce6f683801160959e1f5e539f7
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Mon Jan 13 15:33:29 2020 +0000
sources/git.py: Document tracking multiple branches
---
src/buildstream/plugins/sources/git.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/src/buildstream/plugins/sources/git.py b/src/buildstream/plugins/sources/git.py
index d04ccfe..ce69895 100644
--- a/src/buildstream/plugins/sources/git.py
+++ b/src/buildstream/plugins/sources/git.py
@@ -41,8 +41,13 @@ git - stage files from a git repository
# in your project configuration is recommended.
url: upstream:foo.git
- # Optionally specify a symbolic tracking branch or tag, this
- # will be used to update the 'ref' when refreshing the pipeline.
+ # Optionally specify a symbolic tracking branch, tag, or other Git
+ # revision. When tracking, the rev is checked and the 'ref' config option
+ # is updated to the commit it points to.
+ #
+ # 'track' can also be a list, in which case all the revs are checked and
+ # the overall latest commit is used.
+ #
track: master
# Optionally specify the ref format used for tracking.
[buildstream] 02/13: _gitsourcebase.py: Fetch with depth=1 when an
available tag is given
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c171026e988fa79383843c4880e9733a2acb1cf4
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Wed Dec 18 17:15:36 2019 +0000
_gitsourcebase.py: Fetch with depth=1 when an available tag is given
---
src/buildstream/_gitsourcebase.py | 89 +++++++++++++++++++++++++++++++++------
1 file changed, 76 insertions(+), 13 deletions(-)
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index afbf992..7d1858a 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -48,6 +48,21 @@ class _RefFormat(FastEnum):
GIT_DESCRIBE = "git-describe"
+# _has_matching_ref():
+#
+# Args:
+# refs: Iterable of string (ref id, ref name) pairs
+# tag (str): Tag name
+# commit (str): Commit ID
+#
+# Returns:
+# (bool): Whether the given tag is found in `refs` and points to ID `commit`
+#
+def _has_matching_ref(refs, tag, commit):
+ names = ("refs/tags/{tag}^{{}}".format(tag=tag), "refs/tags/{tag}".format(tag=tag))
+ return any(ref_commit == commit and ref_name in names for ref_commit, ref_name in refs)
+
+
# This class represents a single Git repository. The Git source needs to account for
# submodules, but we don't want to cache them all under the umbrella of the
# superproject - so we use this class which caches them independently, according
@@ -103,19 +118,67 @@ class _GitMirror(SourceFetcher):
def _fetch(self, url):
self._ensure_repo()
- self.source.call(
- [
- self.source.host_git,
- "fetch",
- "--prune",
- url,
- "+refs/heads/*:refs/heads/*",
- "+refs/tags/*:refs/tags/*",
- ],
- fail="Failed to fetch from remote git repository: {}".format(url),
- fail_temporarily=True,
- cwd=self.mirror,
- )
+ fetch_all = False
+
+ # Work out whether we can fetch a specific tag: are we given a ref which
+ # 1. is in git-describe format
+ # 2. refers to an exact tag (is "...-0-g...")
+ # 3. is available on the remote and tags the specified commit?
+ if not self.ref:
+ fetch_all = True
+ else:
+ m = re.match(r"(?P<tag>.*)-0-g(?P<commit>.*)", self.ref)
+ if m is None:
+ fetch_all = True
+ else:
+ tag = m.group("tag")
+ commit = m.group("commit")
+
+ _, ls_remote = self.source.check_output(
+ [self.source.host_git, "ls-remote", url],
+ cwd=self.mirror,
+ fail="Failed to list advertised remote refs from git repository {}".format(url),
+ )
+
+ refs = [line.split("\t", 1) for line in ls_remote.splitlines()]
+ has_ref = _has_matching_ref(refs, tag, commit)
+
+ if not has_ref:
+ self.source.status(
+ "{}: {} is not advertised on {}. Fetching all Git refs".format(self.source, self.ref, url)
+ )
+ fetch_all = True
+ else:
+ exit_code = self.source.call(
+ [
+ self.source.host_git,
+ "fetch",
+ "--depth=1",
+ url,
+ "+refs/tags/{tag}:refs/tags/{tag}".format(tag=tag),
+ ],
+ cwd=self.mirror,
+ )
+ if exit_code != 0:
+ self.source.status(
+ "{}: Failed to fetch tag '{}' from {}. Fetching all Git refs".format(self.source, tag, url)
+ )
+ fetch_all = True
+
+ if fetch_all:
+ self.source.call(
+ [
+ self.source.host_git,
+ "fetch",
+ "--prune",
+ url,
+ "+refs/heads/*:refs/heads/*",
+ "+refs/tags/*:refs/tags/*",
+ ],
+ fail="Failed to fetch from remote git repository: {}".format(url),
+ fail_temporarily=True,
+ cwd=self.mirror,
+ )
def fetch(self, alias_override=None): # pylint: disable=arguments-differ
resolved_url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
[buildstream] 03/13: _gitsourcebase.py: Strip git-describe tag info
from cache key
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 410a27995dee2141eeed48a12e604574e3e56e15
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Wed Jan 8 13:11:48 2020 +0000
_gitsourcebase.py: Strip git-describe tag info from cache key
The beginning parts of git-describe labels are completely arbitrary.
They can be changed either manually or by a track (e.g. if a tag is
moved, added or deleted) even if the referenced commit is the same.
Hence, only the commit ID part of the label should factor into the cache
key.
This commit, of course, breaks cache keys for artifacts built with the
'git' source with git-describe refs.
---
src/buildstream/_gitsourcebase.py | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 7d1858a..271a08e 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -530,10 +530,16 @@ class _GitSourceBase(Source):
self.host_git = utils.get_host_tool("git")
def get_unique_key(self):
+ ref = self.mirror.ref
+ if ref is not None:
+ # If the ref contains "-g" (is in git-describe format),
+ # only choose the part after, which is the commit ID
+ ref = ref.split("-g")[-1]
+
# Here we want to encode the local name of the repository and
# the ref, if the user changes the alias to fetch the same sources
# from another location, it should not affect the cache key.
- key = [self.original_url, self.mirror.ref]
+ key = [self.original_url, ref]
if self.mirror.tags:
tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags}
key.append({"tags": tags})
[buildstream] 11/13: Add Git source tests for tracking latest tag
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 5125ef4cf970c1621adda12bff0a7602676ff3c5
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Tue Jan 14 10:38:04 2020 +0000
Add Git source tests for tracking latest tag
---
tests/sources/git.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 81 insertions(+)
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 0cf8b5b..2d53ce2 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -744,6 +744,87 @@ def test_track_multiple_branches(cli, tmpdir, datafiles):
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+def test_track_latest_tag(cli, tmpdir, datafiles):
+ project = str(datafiles)
+
+ # Create the repo from 'repofiles' subdir
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
+ repo.add_tag("tag")
+ expected_ref = repo.latest_commit() # store the commit we expect to track
+ repo.add_commit()
+
+ # Write out our test target
+ config = repo.source_config()
+ config["track-latest-tag"] = True
+ element = {"kind": "import", "sources": [config]}
+ element_path = os.path.join(project, "target.bst")
+ _yaml.roundtrip_dump(element, element_path)
+
+ # Track it
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
+ result.assert_success()
+
+ element = _yaml.load(element_path)
+ new_ref = element.get_sequence("sources").mapping_at(0).get_str("ref")
+
+ assert new_ref == expected_ref
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+def test_track_latest_matching_tag(cli, tmpdir, datafiles):
+ project = str(datafiles)
+
+ # Create the repo from 'repofiles' subdir
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
+ repo.add_tag("A")
+ expected_ref = repo.latest_commit() # store the commit we expect to track
+ repo.add_commit()
+ repo.add_tag("B")
+
+ # Write out our test target
+ config = repo.source_config()
+ config["track-latest-tag"] = {"match": ["A"]}
+ element = {"kind": "import", "sources": [config]}
+ element_path = os.path.join(project, "target.bst")
+ _yaml.roundtrip_dump(element, element_path)
+
+ # Track it
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
+ result.assert_success()
+
+ element = _yaml.load(element_path)
+ new_ref = element.get_sequence("sources").mapping_at(0).get_str("ref")
+
+ assert new_ref == expected_ref
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+def test_track_no_tag(cli, tmpdir, datafiles):
+ project = str(datafiles)
+
+ # Create the repo from 'repofiles' subdir
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
+ repo.add_tag("B")
+
+ # Write out our test target
+ config = repo.source_config()
+ config["track-latest-tag"] = {"match": ["A"]}
+ element = {"kind": "import", "sources": [config]}
+ element_path = os.path.join(project, "target.bst")
+ _yaml.roundtrip_dump(element, element_path)
+
+ # Track it
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
+ result.assert_task_error(ErrorDomain.SOURCE, None)
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.skipif(HAVE_OLD_GIT, reason="old git describe lacks --first-parent")
@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
[buildstream] 08/13: Add Git source test for tracking multiple
branches
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4edac5df1d93fa9f2ddbce6d175d03906238089d
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Thu Jan 9 14:40:43 2020 +0000
Add Git source test for tracking multiple branches
---
tests/sources/git.py | 31 +++++++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 096338b..0cf8b5b 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -713,6 +713,37 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
+def test_track_multiple_branches(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ now = 1320966000 # arbitrary, but taken from testing/_utils/site.py GIT_ENV
+
+ # Create the repo from 'repofiles' subdir
+ repo = create_repo("git", str(tmpdir))
+ repo.create(os.path.join(project, "repofiles"))
+ repo.add_commit(date=now)
+ repo.branch("branch")
+ repo.add_commit(date=now + 100)
+ expected_ref = repo.latest_commit() # store the commit we expect to track
+
+ # Write out our test target
+ config = repo.source_config()
+ config["track"] = [config["track"], "branch"]
+ element = {"kind": "import", "sources": [config]}
+ element_path = os.path.join(project, "target.bst")
+ _yaml.roundtrip_dump(element, element_path)
+
+ # Track it
+ result = cli.run(project=project, args=["source", "track", "target.bst"])
+ result.assert_success()
+
+ element = _yaml.load(element_path)
+ new_ref = element.get_sequence("sources").mapping_at(0).get_str("ref")
+
+ assert new_ref == expected_ref
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.skipif(HAVE_OLD_GIT, reason="old git describe lacks --first-parent")
@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")])
[buildstream] 12/13: sources/git.py: Document tracking latest tag
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ad9cfda2a045ffeecffdb2807f0c4a4877917459
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Tue Jan 14 11:27:07 2020 +0000
sources/git.py: Document tracking latest tag
---
src/buildstream/plugins/sources/git.py | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/src/buildstream/plugins/sources/git.py b/src/buildstream/plugins/sources/git.py
index ce69895..e18f136 100644
--- a/src/buildstream/plugins/sources/git.py
+++ b/src/buildstream/plugins/sources/git.py
@@ -50,6 +50,20 @@ git - stage files from a git repository
#
track: master
+ # Optionally specify whether to instead choose the latest tagged commit
+ # when tracking. If a tracking revision has no tags, it is ignored.
+ #
+ # This option can either be True/False or a mapping with the optional keys
+ # 'match' and 'exclude'. These sub-options provide a list of glob expressions
+ # to filter which tags are considered when tracking. All tags matching one
+ # or more 'match' patterns and zero 'exclude' patterns are considered.
+ #
+ track-latest-tag:
+ match:
+ - v1.*
+ exclude:
+ - v1.4-pre*
+
# Optionally specify the ref format used for tracking.
# The default is 'sha1' for the raw commit hash.
# If you specify 'git-describe', the commit hash will be prefixed
@@ -142,6 +156,9 @@ details on common configuration options for sources.
This plugin provides the following :ref:`configurable warnings <configurable_warnings>`:
+- ``git:tag-not-found`` - 'track-latest-tag' is enabled but one or more
+ tracking targets has no matching tags in its history.
+
- ``git:inconsistent-submodule`` - A submodule present in the git repository's .gitmodules was never
added with `git submodule add`.
[buildstream] 13/13: Document my changes to the Git source plugin
in NEWS
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0182080d01738996e8eb25bb019fd401e00eff0b
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Tue Jan 14 11:55:48 2020 +0000
Document my changes to the Git source plugin in NEWS
This includes adding an entry for the recursive submodules changes from !1765,
and adding myself to the authors of _gitsourcebase.py.
---
NEWS | 15 +++++++++++++++
src/buildstream/_gitsourcebase.py | 1 +
2 files changed, 16 insertions(+)
diff --git a/NEWS b/NEWS
index 602b14b..43393fb 100644
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,15 @@ CLI
of `buildable` for the state of junction elements, as they can't
be built.
+Plugins
+-------
+
+ o The 'git' source can now track multiple branches/revisions. The overall
+ latest commit is chosen as the new ref.
+
+ o The 'git' source now has the 'track-latest-tag' option. When enabled,
+ tracking will choose the latest tagged commit for the new ref.
+
API
---
@@ -43,6 +52,12 @@ CLI
* `bst workspace open --track`
* `bst workspace reset --track`
+Plugins
+-------
+
+ o The 'git' source will now fetch submodules recursively when
+ its 'checkout-submodules' option is enabled.
+
API
---
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 365a459..240b78c 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -18,6 +18,7 @@
# Authors:
# Tristan Van Berkom <tr...@codethink.co.uk>
# Chandan Singh <cs...@bloomberg.net>
+# Tom Mewett <to...@codethink.co.uk>
"""Abstract base class for source implementations that work with a Git repository"""
[buildstream] 01/13: _gitsourcebase.py: Refactor fetching and
tracking
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 417c18263cdcf129b5688728b739aeca25cf9f1d
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Wed Dec 18 14:56:01 2019 +0000
_gitsourcebase.py: Refactor fetching and tracking
This changes how the Git repo in the source cache is managed.
The mirror is not created with an initial clone, but is set up manually from
an empty repo. This is to support shallow cloning in future changes.
To simplifiy code and Git calls, remotes are no longer used; instead URLs
are passed on the command-line.
---
src/buildstream/_gitsourcebase.py | 59 ++++++++++-----------------------------
1 file changed, 15 insertions(+), 44 deletions(-)
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 5b7f051..afbf992 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -75,25 +75,16 @@ class _GitMirror(SourceFetcher):
self.primary = primary
self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
- # Ensures that the mirror exists
- def ensure(self, alias_override=None):
-
- # Unfortunately, git does not know how to only clone just a specific ref,
- # so we have to download all of those gigs even if we only need a couple
- # of bytes.
+ # _ensure_repo():
+ #
+ # Ensures that the Git repository exists at the mirror location and is configured
+ # to fetch from the given URL
+ #
+ def _ensure_repo(self):
if not os.path.exists(self.mirror):
-
- # Do the initial clone in a tmpdir just because we want an atomic move
- # after a long standing clone which could fail overtime, for now do
- # this directly in our git directory, eliminating the chances that the
- # system configured tmpdir is not on the same partition.
- #
with self.source.tempdir() as tmpdir:
- url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
self.source.call(
- [self.source.host_git, "clone", "--mirror", "-n", url, tmpdir],
- fail="Failed to clone git repository {}".format(url),
- fail_temporarily=True,
+ [self.source.host_git, "init", "--bare", tmpdir], fail="Failed to initialise repository",
)
try:
@@ -101,39 +92,23 @@ class _GitMirror(SourceFetcher):
except DirectoryExistsError:
# Another process was quicker to download this repository.
# Let's discard our own
- self.source.status("{}: Discarding duplicate clone of {}".format(self.source, url))
+ self.source.status("{}: Discarding duplicate repository".format(self.source))
except OSError as e:
raise SourceError(
- "{}: Failed to move cloned git repository {} from '{}' to '{}': {}".format(
- self.source, url, tmpdir, self.mirror, e
+ "{}: Failed to move created repository from '{}' to '{}': {}".format(
+ self.source, tmpdir, self.mirror, e
)
) from e
- def _fetch(self, alias_override=None):
- url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
-
- if alias_override:
- remote_name = utils.url_directory_name(alias_override)
- _, remotes = self.source.check_output(
- [self.source.host_git, "remote"],
- fail="Failed to retrieve list of remotes in {}".format(self.mirror),
- cwd=self.mirror,
- )
- if remote_name not in remotes:
- self.source.call(
- [self.source.host_git, "remote", "add", remote_name, url],
- fail="Failed to add remote {} with url {}".format(remote_name, url),
- cwd=self.mirror,
- )
- else:
- remote_name = "origin"
+ def _fetch(self, url):
+ self._ensure_repo()
self.source.call(
[
self.source.host_git,
"fetch",
- remote_name,
"--prune",
+ url,
"+refs/heads/*:refs/heads/*",
"+refs/tags/*:refs/tags/*",
],
@@ -143,13 +118,11 @@ class _GitMirror(SourceFetcher):
)
def fetch(self, alias_override=None): # pylint: disable=arguments-differ
- # Resolve the URL for the message
resolved_url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary)
with self.source.timed_activity("Fetching from {}".format(resolved_url), silent_nested=True):
- self.ensure(alias_override)
if not self.has_ref():
- self._fetch(alias_override)
+ self._fetch(resolved_url)
self.assert_ref()
def has_ref(self):
@@ -567,9 +540,7 @@ class _GitSourceBase(Source):
# Resolve the URL for the message
resolved_url = self.translate_url(self.mirror.url)
with self.timed_activity("Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True):
- self.mirror.ensure()
- self.mirror._fetch()
-
+ self.mirror._fetch(resolved_url)
# Update self.mirror.ref and node.ref from the self.tracking branch
ret = self.mirror.latest_commit_with_tags(self.tracking, self.track_tags)
[buildstream] 07/13: _gitsourcebase.py: Add support for tracking
multiple branches
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 946fb13a430914de60cf5db44d3d81524c3ad81e
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Thu Jan 9 14:34:21 2020 +0000
_gitsourcebase.py: Add support for tracking multiple branches
This commit allows 'track' config option to be a list instead of a string. If
a list is given, the overall latest commit across the branches is chosen
for the track.
---
src/buildstream/_gitsourcebase.py | 46 +++++++++++++++++++++++++++------------
1 file changed, 32 insertions(+), 14 deletions(-)
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 409b781..c73e86d 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -31,6 +31,7 @@ from configparser import RawConfigParser
from .source import Source, SourceError, SourceFetcher
from .types import Consistency, CoreWarnings
+from .node import ScalarNode, SequenceNode
from . import utils
from .types import FastEnum
from .utils import move_atomic, DirectoryExistsError
@@ -222,6 +223,18 @@ class _GitMirror(SourceFetcher):
)
return output.strip()
+ # date_of_commit():
+ #
+ # Args:
+ # rev (str): A Git "commit-ish" rev
+ #
+ # Returns:
+ # (int): The commit creation date and time, as seconds since the UNIX epoch
+ #
+ def date_of_commit(self, rev):
+ _, time = self.source.check_output([self.source.host_git, "show", "-s", "--format=%ct", rev], cwd=self.mirror)
+ return int(time.strip())
+
# describe():
#
# Args:
@@ -517,13 +530,20 @@ class _GitSourceBase(Source):
self.original_url = node.get_str("url")
self.mirror = self.BST_MIRROR_CLASS(self, "", self.original_url, ref, tags=tags, primary=True)
- self.tracking = node.get_str("track", None)
+
+ track = node.get_node("track", [ScalarNode, SequenceNode], allow_none=True)
+ if isinstance(track, SequenceNode):
+ self.tracking = track.as_str_list()
+ elif track is not None:
+ self.tracking = [track.as_str()]
+ else:
+ self.tracking = []
self.ref_format = node.get_enum("ref-format", _RefFormat, _RefFormat.SHA1)
# At this point we now know if the source has a ref and/or a track.
# If it is missing both then we will be unable to track or build.
- if self.mirror.ref is None and self.tracking is None:
+ if self.mirror.ref is None and not self.tracking:
raise SourceError(
"{}: Git sources require a ref and/or track".format(self), reason="missing-track-and-ref"
)
@@ -636,8 +656,10 @@ class _GitSourceBase(Source):
with self.timed_activity("Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True):
self.mirror._fetch(resolved_url)
- ref = self.mirror.latest_commit(self.tracking)
+ refs = [self.mirror.latest_commit(branch) for branch in self.tracking]
+ ref = max(refs, key=self.mirror.date_of_commit)
tags = self.mirror.reachable_tags(ref) if self.track_tags else []
+
if self.ref_format == _RefFormat.GIT_DESCRIBE:
ref = self.mirror.describe(ref)
@@ -715,31 +737,27 @@ class _GitSourceBase(Source):
ref_in_track = False
if self.tracking:
_, branch = self.check_output(
- [self.host_git, "branch", "--list", self.tracking, "--contains", self.mirror.ref],
+ [self.host_git, "branch", "--contains", self.mirror.ref, "--list", *self.tracking],
cwd=self.mirror.mirror,
)
if branch:
ref_in_track = True
else:
_, tag = self.check_output(
- [self.host_git, "tag", "--list", self.tracking, "--contains", self.mirror.ref],
+ [self.host_git, "tag", "--contains", self.mirror.ref, "--list", *self.tracking],
cwd=self.mirror.mirror,
)
if tag:
ref_in_track = True
if not ref_in_track:
- detail = (
- "The ref provided for the element does not exist locally "
- + "in the provided track branch / tag '{}'.\n".format(self.tracking)
- + "You may wish to track the element to update the ref from '{}' ".format(self.tracking)
- + "with `bst source track`,\n"
- + "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url)
- )
+ detail = """The ref provided for the element does not exist locally in any of the provided
+ tracking branches or tags. You may wish to track the element to update the ref
+ with `bst source track` or examine the upstream for a specific ref."""
self.warn(
- "{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n".format(
- self, self.mirror.ref, self.tracking, self.mirror.url
+ "{}: expected ref '{}' was not found in given branches/tags for staged repository: '{}'\n".format(
+ self, self.mirror.ref, self.mirror.url
),
detail=detail,
warning_token=CoreWarnings.REF_NOT_IN_TRACK,
[buildstream] 04/13: _gitsourcebase.py: Refactor
latest_commit_with_tags
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4b63d159905313edb76946d998bdda04605237f6
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Wed Jan 8 16:54:45 2020 +0000
_gitsourcebase.py: Refactor latest_commit_with_tags
The functionality has been split into three new functions:
latest_commit, reachable_tags, and describe. No behaviour is changed.
---
src/buildstream/_gitsourcebase.py | 66 ++++++++++++++++++++++++++++-----------
1 file changed, 47 insertions(+), 19 deletions(-)
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 271a08e..409b781 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -206,30 +206,55 @@ class _GitMirror(SourceFetcher):
"{}: expected ref '{}' was not found in git repository: '{}'".format(self.source, self.ref, self.url)
)
- def latest_commit_with_tags(self, tracking, track_tags=False):
+ # latest_commit():
+ #
+ # Args:
+ # branch (str)
+ #
+ # Returns:
+ # (str): The commit rev of the latest commit on the given branch
+ #
+ def latest_commit(self, branch):
_, output = self.source.check_output(
- [self.source.host_git, "rev-parse", tracking],
- fail="Unable to find commit for specified branch name '{}'".format(tracking),
+ [self.source.host_git, "rev-parse", branch],
+ fail="Unable to find commit for specified branch name '{}'".format(branch),
cwd=self.mirror,
)
- ref = output.rstrip("\n")
+ return output.strip()
- if self.source.ref_format == _RefFormat.GIT_DESCRIBE:
- # Prefix the ref with the closest tag, if available,
- # to make the ref human readable
- exit_code, output = self.source.check_output(
- [self.source.host_git, "describe", "--tags", "--abbrev=40", "--long", ref], cwd=self.mirror
- )
- if exit_code == 0:
- ref = output.rstrip("\n")
+ # describe():
+ #
+ # Args:
+ # rev (str): A Git "commit-ish" rev
+ #
+ # Returns:
+ # (str): A human-readable form of the rev as produced by git-describe,
+ # or the rev itself if it cannot be described
+ #
+ def describe(self, rev):
+ exit_code, output = self.source.check_output(
+ [self.source.host_git, "describe", "--tags", "--abbrev=40", "--long", rev], cwd=self.mirror,
+ )
- if not track_tags:
- return ref, []
+ if exit_code == 0:
+ rev = output.strip()
+ return rev
+
+ # reachable_tags():
+ #
+ # Args:
+ # rev (str): A Git "commit-ish" rev
+ #
+ # Returns:
+ # (list): A list of (tag name (str), commit ref (str), annotated (bool))
+ # triples describing a tag, its tagged commit and whether it's annotated
+ #
+ def reachable_tags(self, rev):
tags = set()
for options in [[], ["--first-parent"], ["--tags"], ["--tags", "--first-parent"]]:
exit_code, output = self.source.check_output(
- [self.source.host_git, "describe", "--abbrev=0", ref, *options], cwd=self.mirror
+ [self.source.host_git, "describe", "--abbrev=0", rev, *options], cwd=self.mirror
)
if exit_code == 0:
tag = output.strip()
@@ -243,7 +268,7 @@ class _GitMirror(SourceFetcher):
tags.add((tag, commit_ref.strip(), annotated))
- return ref, list(tags)
+ return list(tags)
def stage(self, directory):
fullpath = os.path.join(directory, self.path)
@@ -610,10 +635,13 @@ class _GitSourceBase(Source):
resolved_url = self.translate_url(self.mirror.url)
with self.timed_activity("Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True):
self.mirror._fetch(resolved_url)
- # Update self.mirror.ref and node.ref from the self.tracking branch
- ret = self.mirror.latest_commit_with_tags(self.tracking, self.track_tags)
- return ret
+ ref = self.mirror.latest_commit(self.tracking)
+ tags = self.mirror.reachable_tags(ref) if self.track_tags else []
+ if self.ref_format == _RefFormat.GIT_DESCRIBE:
+ ref = self.mirror.describe(ref)
+
+ return ref, tags
def init_workspace(self, directory):
with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
[buildstream] 05/13: repo/git.py: Add optional date argument to
add_commit
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c3f1a1f0c6eab12491634a78b9209e5aa0d79155
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Thu Jan 9 14:00:04 2020 +0000
repo/git.py: Add optional date argument to add_commit
---
tests/testutils/repo/git.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index 1deca3f..779eb76 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -42,8 +42,11 @@ class Git(Repo):
def add_annotated_tag(self, tag, message):
self._run_git("tag", "-a", tag, "-m", message)
- def add_commit(self):
- self._run_git("commit", "--allow-empty", "-m", "Additional commit")
+ def add_commit(self, date=None):
+ env = self.env
+ if date is not None:
+ env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = str(date)
+ self._run_git("commit", "--allow-empty", "-m", "Additional commit", env=env)
return self.latest_commit()
def add_file(self, filename):
[buildstream] 10/13: _gitsourcebase.py: Add 'track-latest-tag'
option
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch tmewett/merge-git-tag
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6e34512fa9fb81ba15654c74af4eb240201a14b6
Author: Tom Mewett <to...@codethink.co.uk>
AuthorDate: Tue Jan 14 10:33:05 2020 +0000
_gitsourcebase.py: Add 'track-latest-tag' option
This modifies the behaviour of tracking the source to choosing the latest
tagged commit instead of the branch tip.
---
src/buildstream/_gitsourcebase.py | 98 +++++++++++++++++++++++++++++++++++----
1 file changed, 90 insertions(+), 8 deletions(-)
diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index c73e86d..365a459 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -31,7 +31,7 @@ from configparser import RawConfigParser
from .source import Source, SourceError, SourceFetcher
from .types import Consistency, CoreWarnings
-from .node import ScalarNode, SequenceNode
+from .node import MappingNode, ScalarNode, SequenceNode
from . import utils
from .types import FastEnum
from .utils import move_atomic, DirectoryExistsError
@@ -39,6 +39,7 @@ from .utils import move_atomic, DirectoryExistsError
GIT_MODULES = ".gitmodules"
# Warnings
+WARN_TAG_NOT_FOUND = "tag-not-found"
WARN_INCONSISTENT_SUBMODULE = "inconsistent-submodule"
WARN_UNLISTED_SUBMODULE = "unlisted-submodule"
WARN_INVALID_SUBMODULE = "invalid-submodule"
@@ -64,6 +65,10 @@ def _has_matching_ref(refs, tag, commit):
return any(ref_commit == commit and ref_name in names for ref_commit, ref_name in refs)
+def _undescribe(rev):
+ return rev.split("-g")[-1]
+
+
# This class represents a single Git repository. The Git source needs to account for
# submodules, but we don't want to cache them all under the umbrella of the
# superproject - so we use this class which caches them independently, according
@@ -223,6 +228,35 @@ class _GitMirror(SourceFetcher):
)
return output.strip()
+ # latest_tagged():
+ #
+ # Args:
+ # branch (str)
+ # match (list): Optional list of glob pattern strings
+ # exclude (list): Optional list of glob pattern strings
+ #
+ # For the behaviour of match and exclude, see the corresponding flags of `git describe`.
+ #
+ # Returns:
+ # (str or None): A Git revision of the latest tagged commit on the given branch.
+ # Only tags with names meeting the criteria of `match` and `exclude` are considered.
+ # If no matching tags are found, None is returned.
+ #
+ def latest_tagged(self, branch, match=None, exclude=None):
+ pattern_args = []
+ if match:
+ for pat in match:
+ pattern_args += ["--match", pat]
+ if exclude:
+ for pat in exclude:
+ pattern_args += ["--exclude", pat]
+
+ exit_code, output = self.source.check_output(
+ [self.source.host_git, "describe", "--tags", "--abbrev=0", *pattern_args, branch], cwd=self.mirror
+ )
+
+ return output.strip() + "^{commit}" if exit_code == 0 else None
+
# date_of_commit():
#
# Args:
@@ -518,7 +552,17 @@ class _GitSourceBase(Source):
def configure(self, node):
ref = node.get_str("ref", None)
- config_keys = ["url", "track", "ref", "submodules", "checkout-submodules", "ref-format", "track-tags", "tags"]
+ config_keys = [
+ "url",
+ "track",
+ "ref",
+ "submodules",
+ "checkout-submodules",
+ "ref-format",
+ "track-latest-tag",
+ "track-tags",
+ "tags",
+ ]
node.validate_keys(config_keys + Source.COMMON_CONFIG_KEYS)
tags_node = node.get_sequence("tags", [])
@@ -528,6 +572,20 @@ class _GitSourceBase(Source):
tags = self._load_tags(node)
self.track_tags = node.get_bool("track-tags", default=False)
+ self.match_tags = []
+ self.exclude_tags = []
+ # Allow 'track-latest-tag' to be either a bool or a mapping configuring its sub-options
+ latest_tag = node.get_node("track-latest-tag", [ScalarNode, MappingNode], allow_none=True)
+ if isinstance(latest_tag, MappingNode):
+ latest_tag.validate_keys(["match", "exclude"])
+ self.match_tags += latest_tag.get_str_list("match", [])
+ self.exclude_tags += latest_tag.get_str_list("exclude", [])
+ self.track_latest_tag = True
+ elif latest_tag is not None:
+ self.track_latest_tag = latest_tag.as_bool()
+ else:
+ self.track_latest_tag = False
+
self.original_url = node.get_str("url")
self.mirror = self.BST_MIRROR_CLASS(self, "", self.original_url, ref, tags=tags, primary=True)
@@ -579,7 +637,7 @@ class _GitSourceBase(Source):
if ref is not None:
# If the ref contains "-g" (is in git-describe format),
# only choose the part after, which is the commit ID
- ref = ref.split("-g")[-1]
+ ref = _undescribe(ref)
# Here we want to encode the local name of the repository and
# the ref, if the user changes the alias to fetch the same sources
@@ -656,13 +714,37 @@ class _GitSourceBase(Source):
with self.timed_activity("Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True):
self.mirror._fetch(resolved_url)
- refs = [self.mirror.latest_commit(branch) for branch in self.tracking]
- ref = max(refs, key=self.mirror.date_of_commit)
- tags = self.mirror.reachable_tags(ref) if self.track_tags else []
+ if self.track_latest_tag:
+ revs = []
+ without_tags = []
+ for branch in self.tracking:
+ rev = self.mirror.latest_tagged(branch, match=self.match_tags, exclude=self.exclude_tags)
+ if rev is None:
+ without_tags.append(branch)
+ else:
+ revs.append(rev)
+
+ if without_tags:
+ self.warn(
+ "{}: Cannot find tags on all track targets".format(self),
+ warning_token=WARN_TAG_NOT_FOUND,
+ detail="No matching tags were found on the following track targets:\n\n"
+ + "\n".join(without_tags),
+ )
- if self.ref_format == _RefFormat.GIT_DESCRIBE:
- ref = self.mirror.describe(ref)
+ if not revs:
+ raise SourceError("{}: Failed to find any revisions to track".format(self))
+ else:
+ revs = [self.mirror.latest_commit(branch) for branch in self.tracking]
+
+ ref = max(revs, key=self.mirror.date_of_commit)
+
+ ref = self.mirror.describe(ref) # normalise our potential mixture of rev formats
+ if self.ref_format == _RefFormat.SHA1:
+ ref = _undescribe(ref)
+
+ tags = self.mirror.reachable_tags(ref) if self.track_tags else []
return ref, tags
def init_workspace(self, directory):