You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:16:50 UTC
[buildstream] branch reduce_history_in_cache created (now 6029c7a)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch reduce_history_in_cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 6029c7a Updating .bzr plugin to omit the .bzr dir
This branch includes the following new commits:
new 0fd2306 Limiting git history to reduce cache size
new 6029c7a Updating .bzr plugin to omit the .bzr dir
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/02: Limiting git history to reduce cache size
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch reduce_history_in_cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 0fd230671d0c865b976ff1d9a4f70ae8ff631cfb
Author: Phillip Smyth <ph...@Nexus-x240.dyn.ducie.codethink.co.uk>
AuthorDate: Tue May 29 13:48:19 2018 +0100
Limiting git history to reduce cache size
---
buildstream/plugins/sources/git.py | 80 +++++++++++++++++++++++++++++++++-----
tests/sources/git.py | 1 +
2 files changed, 72 insertions(+), 9 deletions(-)
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 44065ad..4b53c7f 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -70,6 +70,7 @@ git - stage files from a git repository
import os
import re
import shutil
+import shlex
from collections import Mapping
from io import StringIO
@@ -150,15 +151,76 @@ class GitMirror():
def stage(self, directory):
fullpath = os.path.join(directory, self.path)
- # We need to pass '--no-hardlinks' because there's nothing to
- # stop the build from overwriting the files in the .git directory
- # inside the sandbox.
- self.source.call([self.source.host_git, 'clone', '--no-checkout', '--no-hardlinks', self.mirror, fullpath],
- fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath))
-
- self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
- fail="Failed to checkout git ref {}".format(self.ref),
- cwd=fullpath)
+ # Need to get every commit since the last tagged object until the tracking commit
+ if self.has_ref():
+ all_tags = self.source.check_output([self.source.host_git, 'tag'], cwd=self.mirror)[1]
+ all_tags = [x.strip() for x in all_tags.split('\n')]
+ if all_tags:
+ tags_since_sha = self.source.check_output([self.source.host_git,
+ 'tag',
+ '--sort',
+ '--contains',
+ self.ref],
+ cwd=self.mirror)[1]
+
+ tags_since_sha = [x.strip() for x in tags_since_sha.split('\n')]
+ preceeding_tags = [x for x in all_tags if x not in tags_since_sha]
+ if preceeding_tags:
+ last_tag_before_ref = preceeding_tags[-1]
+ else:
+ last_tag_before_ref = 'HEAD'
+
+ # find number of commits since last_tag_before_ref
+ target_depth = self.source.check_output([self.source.host_git,
+ 'rev-list',
+ '--count',
+ 'HEAD...{}'.format(last_tag_before_ref)])[1]
+
+ else:
+ target_depth = self.source.check_output([self.source.host_git,
+ 'rev-list',
+ '--count',
+ 'HEAD...{}'.format(self.ref)], cwd=self.mirror)[1]
+
+ if int(target_depth) == 0:
+ target_depth = 1
+
+ branch = self.source.check_output([self.source.host_git,
+ 'rev-parse',
+ '--abbrev-ref',
+ 'HEAD'], cwd=self.mirror)[1]
+
+ self.source.call([self.source.host_git,
+ 'init',
+ fullpath])
+
+ self.source.call([self.source.host_git,
+ 'fetch',
+ '--depth={}'.format(int(target_depth)),
+ 'ext::git -c uploadpack.allowReachableSHA1InWant=true %s {}'
+ .format(shlex.quote(self.mirror)),
+ self.ref],
+ env=dict(os.environ, GIT_ALLOW_PROTOCOL="ext"), cwd=fullpath)
+
+ self.source.call([self.source.host_git,
+ 'checkout',
+ 'FETCH_HEAD'], cwd=fullpath)
+
+ if "master" not in branch:
+ self.source.call([self.source.host_git,
+ 'branch',
+ '-D',
+ 'master'], cwd=fullpath)
+
+ self.source.call([self.source.host_git,
+ 'reflog',
+ 'expire',
+ '--expire-unreachable=all'
+ '--all'], cwd=fullpath)
+
+ self.source.call([self.source.host_git,
+ 'repack',
+ '-ad'], cwd=fullpath)
def init_workspace(self, directory):
fullpath = os.path.join(directory, self.path)
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 06888c3..495c147 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -288,6 +288,7 @@ def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, dat
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project-override'))
def test_submodule_fetch_project_override(cli, tmpdir, datafiles):
+ print("\n\n\nTemp Directory: {}\n\n\n".format(tmpdir))
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
[buildstream] 02/02: Updating .bzr plugin to omit the .bzr dir
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch reduce_history_in_cache
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 6029c7a3a38223938c606dc292e95d42c68f7cec
Author: Phillip Smyth <ph...@Nexus-x240.dyn.ducie.codethink.co.uk>
AuthorDate: Tue May 29 13:46:38 2018 +0100
Updating .bzr plugin to omit the .bzr dir
---
buildstream/plugins/sources/bzr.py | 2 +
buildstream/plugins/sources/git.py | 87 ++++++++++++++++++++------------------
2 files changed, 47 insertions(+), 42 deletions(-)
diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py
index b499d49..36be982 100644
--- a/buildstream/plugins/sources/bzr.py
+++ b/buildstream/plugins/sources/bzr.py
@@ -121,6 +121,8 @@ class BzrSource(Source):
self._get_branch_dir(), directory],
fail="Failed to checkout revision {} from branch {} to {}"
.format(self.ref, self._get_branch_dir(), directory))
+ # Remove .bzr dir
+ shutil.rmtree(os.path.join(directory, ".bzr"))
def init_workspace(self, directory):
url = os.path.join(self.url, self.tracking)
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 4b53c7f..5d655e4 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -158,7 +158,6 @@ class GitMirror():
if all_tags:
tags_since_sha = self.source.check_output([self.source.host_git,
'tag',
- '--sort',
'--contains',
self.ref],
cwd=self.mirror)[1]
@@ -167,60 +166,64 @@ class GitMirror():
preceeding_tags = [x for x in all_tags if x not in tags_since_sha]
if preceeding_tags:
last_tag_before_ref = preceeding_tags[-1]
+ print("\nWe have a tag!!!!!\n")
else:
- last_tag_before_ref = 'HEAD'
-
- # find number of commits since last_tag_before_ref
- target_depth = self.source.check_output([self.source.host_git,
- 'rev-list',
- '--count',
- 'HEAD...{}'.format(last_tag_before_ref)])[1]
-
+ last_tag_before_ref = self.ref
+ print("\nWe have No tag!!!!!\n")
else:
- target_depth = self.source.check_output([self.source.host_git,
- 'rev-list',
- '--count',
- 'HEAD...{}'.format(self.ref)], cwd=self.mirror)[1]
-
- if int(target_depth) == 0:
- target_depth = 1
-
- branch = self.source.check_output([self.source.host_git,
- 'rev-parse',
- '--abbrev-ref',
- 'HEAD'], cwd=self.mirror)[1]
+ last_tag_before_ref = self.ref
+ print("\nWe have No tag!!!!!\n")
self.source.call([self.source.host_git,
'init',
fullpath])
- self.source.call([self.source.host_git,
- 'fetch',
- '--depth={}'.format(int(target_depth)),
- 'ext::git -c uploadpack.allowReachableSHA1InWant=true %s {}'
- .format(shlex.quote(self.mirror)),
- self.ref],
- env=dict(os.environ, GIT_ALLOW_PROTOCOL="ext"), cwd=fullpath)
+ rev_list = self.source.check_output([self.source.host_git,
+ 'rev-list',
+ '-n1',
+ '--parents',
+ last_tag_before_ref],
+ cwd=self.mirror)[1]
- self.source.call([self.source.host_git,
- 'checkout',
- 'FETCH_HEAD'], cwd=fullpath)
+ rev_list = [x.strip() for x in rev_list.split(' ')]
+ rev_list.pop(0)
+
+ print("ref: {}".format(last_tag_before_ref))
+ print("Rev List: {}".format(rev_list))
- if "master" not in branch:
+ # Adding ext::git line allows bst to fetch a specific commit using its SHA
+ # without changing the remote configuration
+ r = self.source.check_output([self.source.host_git,
+ 'log'], cwd=self.mirror)
+
+ print("r1: {}".format(r))
+ if not rev_list:
+ print("\n\n\nBANANA1\n\n\n")
self.source.call([self.source.host_git,
- 'branch',
- '-D',
- 'master'], cwd=fullpath)
+ 'fetch',
+ 'ext::git -c uploadpack.allowReachableSHA1InWant=true %s {}'
+ .format(shlex.quote(self.mirror)),
+ self.ref],
+ env=dict(os.environ, GIT_ALLOW_PROTOCOL="ext"), cwd=fullpath)
- self.source.call([self.source.host_git,
- 'reflog',
- 'expire',
- '--expire-unreachable=all'
- '--all'], cwd=fullpath)
+ else:
+ print("\n\n\nBANANA2\n\n\n")
+ self.source.call([self.source.host_git,
+ 'fetch'] +
+ ['--shallow-exclude={}'.format(parent) for parent in rev_list] +
+ ['ext::git -c uploadpack.allowReachableSHA1InWant=true %s {}'
+ .format(shlex.quote(self.mirror)),
+ self.ref],
+ env=dict(os.environ, GIT_ALLOW_PROTOCOL="ext"), cwd=fullpath)
self.source.call([self.source.host_git,
- 'repack',
- '-ad'], cwd=fullpath)
+ 'checkout',
+ 'FETCH_HEAD'], cwd=fullpath)
+
+ r = self.source.check_output([self.source.host_git,
+ 'log'], cwd=fullpath)
+
+ print("r2: {}".format(r))
def init_workspace(self, directory):
fullpath = os.path.join(directory, self.path)