You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:24:47 UTC
[buildstream] branch valentindavid/link_files_sort_resolved created
(now 4bd6b9b)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch valentindavid/link_files_sort_resolved
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 4bd6b9b Fix issue with absolute symbolic links in copy_files/link_files
This branch includes the following new commits:
new 96fc331 Resolve paths before ordering them in link_files/copy_files
new 4bd6b9b Fix issue with absolute symbolic links in copy_files/link_files
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/02: Resolve paths before ordering them in
link_files/copy_files
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch valentindavid/link_files_sort_resolved
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 96fc33105c3a14b24fcc4157ae576e21a9ea1070
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Fri Oct 5 16:20:55 2018 +0200
Resolve paths before ordering them in link_files/copy_files
Some paths may not be canonical, and sorting paths solves symlink
issues only when paths are canonical.
Fixes #647.
---
buildstream/utils.py | 11 +++++-
tests/integration/compose-symlink-order.py | 39 ++++++++++++++++++++++
.../project/elements/compose-symlink-order/a.bst | 4 +++
.../project/elements/compose-symlink-order/b.bst | 7 ++++
.../elements/compose-symlink-order/compose.bst | 8 +++++
.../project/files/compose-symlink-order/a/b/.keep | 0
.../project/files/compose-symlink-order/b/a/c/d | 1 +
7 files changed, 69 insertions(+), 1 deletion(-)
diff --git a/buildstream/utils.py b/buildstream/utils.py
index a460031..ae5da34 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -816,7 +816,16 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
# symbolic links which lead to directories before processing files inside
# those directories.
if not presorted:
- filelist = sorted(filelist)
+ resolved = []
+ for f in filelist:
+ dirname = os.path.dirname(f)
+ dirname = os.path.realpath(os.path.join(srcdir, dirname))
+ dirname = os.path.relpath(dirname, srcdir)
+ if dirname == '.':
+ resolved.append(os.path.basename(f))
+ else:
+ resolved.append(os.path.join(dirname, os.path.basename(f)))
+ filelist = sorted(resolved)
# Now walk the list
for path in filelist:
diff --git a/tests/integration/compose-symlink-order.py b/tests/integration/compose-symlink-order.py
new file mode 100644
index 0000000..32a8582
--- /dev/null
+++ b/tests/integration/compose-symlink-order.py
@@ -0,0 +1,39 @@
+import os
+import pytest
+
+from buildstream import _yaml
+
+from tests.testutils import cli_integration as cli
+
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(
+ os.path.dirname(os.path.realpath(__file__)),
+ "project"
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_compose_symlinks_bad_order(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ checkout = os.path.join(cli.directory, 'checkout')
+ element_path = os.path.join(project, 'elements')
+
+ a_files = os.path.join(project, 'files', 'compose-symlink-order', 'a')
+ os.symlink('b', os.path.join(a_files, 'a'),
+ target_is_directory=True)
+
+ result = cli.run(project=project,
+ args=['build', 'compose-symlink-order/compose.bst'])
+ result.assert_success()
+
+ result = cli.run(project=project,
+ args=['checkout', 'compose-symlink-order/compose.bst',
+ checkout])
+ result.assert_success()
+
+ assert os.path.exists(os.path.join(checkout, 'a/c/d'))
+ assert os.path.exists(os.path.join(checkout, 'b/c/d'))
+ assert os.path.islink(os.path.join(checkout, 'a'))
diff --git a/tests/integration/project/elements/compose-symlink-order/a.bst b/tests/integration/project/elements/compose-symlink-order/a.bst
new file mode 100644
index 0000000..16c7b8f
--- /dev/null
+++ b/tests/integration/project/elements/compose-symlink-order/a.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+ path: files/compose-symlink-order/a
diff --git a/tests/integration/project/elements/compose-symlink-order/b.bst b/tests/integration/project/elements/compose-symlink-order/b.bst
new file mode 100644
index 0000000..08bece2
--- /dev/null
+++ b/tests/integration/project/elements/compose-symlink-order/b.bst
@@ -0,0 +1,7 @@
+kind: import
+depends:
+- filename: compose-symlink-order/a.bst
+
+sources:
+- kind: local
+ path: files/compose-symlink-order/b
diff --git a/tests/integration/project/elements/compose-symlink-order/compose.bst b/tests/integration/project/elements/compose-symlink-order/compose.bst
new file mode 100644
index 0000000..3150724
--- /dev/null
+++ b/tests/integration/project/elements/compose-symlink-order/compose.bst
@@ -0,0 +1,8 @@
+kind: compose
+depends:
+- filename: compose-symlink-order/b.bst
+ type: build
+
+config:
+ exclude:
+ - devel
diff --git a/tests/integration/project/files/compose-symlink-order/a/b/.keep b/tests/integration/project/files/compose-symlink-order/a/b/.keep
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integration/project/files/compose-symlink-order/b/a/c/d b/tests/integration/project/files/compose-symlink-order/b/a/c/d
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/integration/project/files/compose-symlink-order/b/a/c/d
@@ -0,0 +1 @@
+test
[buildstream] 02/02: Fix issue with absolute symbolic links in
copy_files/link_files
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch valentindavid/link_files_sort_resolved
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 4bd6b9bb0a0f5af8893761e48d79f949cadf40fa
Author: Valentin David <va...@codethink.co.uk>
AuthorDate: Tue Nov 13 21:24:20 2018 +0100
Fix issue with absolute symbolic links in copy_files/link_files
When paths given by caller for copy_files/link_files used absolute
symbolic links, sorting the paths required to resolve links within the
sysroot.
---
buildstream/utils.py | 46 +++++++++++++++++++++-
tests/integration/compose-symlinks.py | 17 ++++++++
.../elements/compose-symlinks/a-foo-symlink.bst | 11 ++++++
.../compose-symlinks/compose-absolute-symlink.bst | 14 +++++++
.../project/elements/compose-symlinks/foo-dir.bst | 11 ++++++
.../compose-symlinks/integration-move-dir.bst | 14 +++++++
6 files changed, 111 insertions(+), 2 deletions(-)
diff --git a/buildstream/utils.py b/buildstream/utils.py
index ae5da34..fc49615 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -787,6 +787,48 @@ def _ensure_real_directory(root, destpath):
return destpath_resolved
+@functools.lru_cache(maxsize=1)
+def _symloop_max():
+ if hasattr(os, 'sysconf'):
+ try:
+ ret = os.sysconf('_SC_SYMLOOP_MAX')
+ if ret != -1:
+ return ret
+ except ValueError:
+ pass
+ return 8
+
+
+@functools.lru_cache(maxsize=64)
+def _sysroot_realpath(path, sysroot):
+ assert not os.path.isabs(path)
+ assert os.path.isabs(sysroot)
+
+ loop_count = _symloop_max()
+ while True:
+ full_path = os.path.join(sysroot, path)
+ st = os.lstat(full_path)
+ mode = st.st_mode
+ if not stat.S_ISLNK(mode):
+ break
+ loop_count = loop_count - 1
+ if loop_count < 0:
+ raise UtilError("Symlink loop detected: {}".format(os.path.join(sysroot, path)))
+ link_path = os.readlink(full_path)
+ if not os.path.isabs(link_path):
+ link_path = os.path.join('/', os.path.dirname(path), link_path)
+ path = os.path.relpath(os.path.normpath(link_path), '/')
+
+ parent = os.path.dirname(path)
+ if parent != '':
+ parent = _sysroot_realpath(parent, sysroot)
+ full_parent = os.path.join(sysroot, parent)
+ if not os.path.isdir(full_parent):
+ raise UtilError("Path is not a directory: {}".format(full_parent))
+
+ return os.path.join(parent, os.path.basename(path))
+
+
# _process_list()
#
# Internal helper for copying/moving/linking file lists
@@ -817,10 +859,10 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
# those directories.
if not presorted:
resolved = []
+ _sysroot_realpath.cache_clear()
for f in filelist:
dirname = os.path.dirname(f)
- dirname = os.path.realpath(os.path.join(srcdir, dirname))
- dirname = os.path.relpath(dirname, srcdir)
+ dirname = _sysroot_realpath(dirname, srcdir)
if dirname == '.':
resolved.append(os.path.basename(f))
else:
diff --git a/tests/integration/compose-symlinks.py b/tests/integration/compose-symlinks.py
index bf279fa..c72d818 100644
--- a/tests/integration/compose-symlinks.py
+++ b/tests/integration/compose-symlinks.py
@@ -7,6 +7,7 @@ from buildstream import _yaml
from tests.testutils import cli_integration as cli
from tests.testutils.integration import walk_dir
+from tests.testutils.site import IS_LINUX, HAVE_BWRAP
pytestmark = pytest.mark.integration
@@ -41,3 +42,19 @@ def test_compose_symlinks(cli, tmpdir, datafiles):
assert set(walk_dir(checkout)) == set(['/sbin', '/usr', '/usr/sbin',
'/usr/sbin/init', '/usr/sbin/dummy'])
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not IS_LINUX or not HAVE_BWRAP, reason='Only available on linux with bubblewrap')
+def test_compose_absolute_symlinks(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ checkout = os.path.join(cli.directory, 'checkout')
+ element_path = os.path.join(project, 'elements')
+
+ result = cli.run(project=project, args=['build', 'compose-symlinks/compose-absolute-symlink.bst'])
+ result.assert_success()
+
+ result = cli.run(project=project, args=['checkout', 'compose-symlinks/compose-absolute-symlink.bst', checkout])
+ result.assert_success()
+
+ assert os.readlink(os.path.join(checkout, 'foo')) == 'test/foo'
diff --git a/tests/integration/project/elements/compose-symlinks/a-foo-symlink.bst b/tests/integration/project/elements/compose-symlinks/a-foo-symlink.bst
new file mode 100644
index 0000000..3beff33
--- /dev/null
+++ b/tests/integration/project/elements/compose-symlinks/a-foo-symlink.bst
@@ -0,0 +1,11 @@
+kind: script
+
+depends:
+- filename: base.bst
+ type: build
+
+config:
+ commands:
+ - |
+ mkdir -p "%{install-root}/bar"
+ ln -s "/bar" "%{install-root}/foo"
diff --git a/tests/integration/project/elements/compose-symlinks/compose-absolute-symlink.bst b/tests/integration/project/elements/compose-symlinks/compose-absolute-symlink.bst
new file mode 100644
index 0000000..7111cb4
--- /dev/null
+++ b/tests/integration/project/elements/compose-symlinks/compose-absolute-symlink.bst
@@ -0,0 +1,14 @@
+kind: compose
+
+depends:
+- filename: compose-symlinks/foo-dir.bst
+ type: build
+- filename: compose-symlinks/a-foo-symlink.bst
+ type: build
+- filename: compose-symlinks/integration-move-dir.bst
+ type: build
+
+config:
+ include-orphans: true
+ exclude:
+ - dummy
diff --git a/tests/integration/project/elements/compose-symlinks/foo-dir.bst b/tests/integration/project/elements/compose-symlinks/foo-dir.bst
new file mode 100644
index 0000000..e9206cd
--- /dev/null
+++ b/tests/integration/project/elements/compose-symlinks/foo-dir.bst
@@ -0,0 +1,11 @@
+kind: script
+
+depends:
+- filename: base.bst
+ type: build
+
+config:
+ commands:
+ - |
+ mkdir -p "%{install-root}/foo"
+ echo test >"%{install-root}/foo/foo.txt"
diff --git a/tests/integration/project/elements/compose-symlinks/integration-move-dir.bst b/tests/integration/project/elements/compose-symlinks/integration-move-dir.bst
new file mode 100644
index 0000000..851f03f
--- /dev/null
+++ b/tests/integration/project/elements/compose-symlinks/integration-move-dir.bst
@@ -0,0 +1,14 @@
+kind: stack
+
+depends:
+- filename: base.bst
+
+public:
+ bst:
+ integration-commands:
+ - |
+ mkdir test
+ mv foo test/
+ mv bar test/
+ ln -s /test/foo /foo
+ ln -s /test/bar /bar