You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by ro...@apache.org on 2020/12/29 13:44:27 UTC
[buildstream] branch jjardon/tar_plugin_move created (now 1073ae6)
This is an automated email from the ASF dual-hosted git repository.
root pushed a change to branch jjardon/tar_plugin_move
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 1073ae6 Remove "tar" plugin
This branch includes the following new commits:
new 1073ae6 Remove "tar" plugin
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/01: Remove "tar" plugin
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
root pushed a commit to branch jjardon/tar_plugin_move
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 1073ae660ccba5fcc76157f04ff13facc084530d
Author: Javier Jardón <jj...@gnome.org>
AuthorDate: Fri Nov 29 23:40:38 2019 +0900
Remove "tar" plugin
Use the one from bst-plugins-experimental instead
---
.gitlab-ci.yml | 2 +-
doc/examples/autotools/project.conf | 6 +
doc/examples/developing/project.conf | 6 +
doc/examples/integration-commands/project.conf | 6 +
doc/examples/running-commands/project.conf | 6 +
doc/source/core_plugins.rst | 1 -
src/buildstream/plugins/sources/tar.py | 253 -----------
.../testing/_sourcetests/project/project.conf | 6 +
src/buildstream/testing/_utils/site.py | 6 -
tests/cachekey/project/sources/tar1.bst | 5 -
tests/cachekey/project/sources/tar1.expected | 1 -
tests/cachekey/project/sources/tar2.bst | 6 -
tests/cachekey/project/sources/tar2.expected | 1 -
tests/cachekey/project/target.bst | 2 -
tests/conftest.py | 2 -
tests/integration/project/project.conf | 6 +
tests/remoteexecution/project/project.conf | 5 +
tests/sandboxes/project/project.conf | 5 +
tests/sources/tar.py | 471 ---------------------
.../content/base-directory/subdir1/file.txt | 1 -
tests/sources/tar/contains-links/target.bst | 6 -
tests/sources/tar/explicit-basedir/content/a/b/d | 1 -
tests/sources/tar/explicit-basedir/content/a/c | 1 -
tests/sources/tar/explicit-basedir/target.bst | 7 -
tests/sources/tar/fetch/content/a/b/d | 1 -
tests/sources/tar/fetch/content/a/c | 1 -
tests/sources/tar/fetch/target-lz.bst | 6 -
tests/sources/tar/fetch/target.bst | 6 -
tests/sources/tar/no-basedir/content/a/b/d | 1 -
tests/sources/tar/no-basedir/content/a/c | 1 -
tests/sources/tar/no-basedir/target.bst | 7 -
tests/sources/tar/no-ref/a/b/d | 1 -
tests/sources/tar/no-ref/a/c | 1 -
tests/sources/tar/no-ref/target.bst | 5 -
.../out-of-basedir-hardlinks/contents/elsewhere/a | 1 -
.../contents/elsewhere/malicious | 0
.../out-of-basedir-hardlinks/contents/to_extract/a | 1 -
.../out-of-basedir-hardlinks/malicious_target.bst | 5 -
.../tar/out-of-basedir-hardlinks/target.bst | 6 -
tests/sources/tar/read-only/content/a.tar.gz | Bin 10240 -> 0 bytes
tests/sources/tar/read-only/content/b.tar.gz | Bin 148 -> 0 bytes
tests/sources/tar/read-only/content/c.tar.gz | Bin 128 -> 0 bytes
tests/sources/tar/read-only/target.bst | 6 -
tests/testutils/repo/tar.py | 27 --
tox.ini | 4 +-
45 files changed, 49 insertions(+), 843 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 38a5505..1bae0dc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -351,7 +351,7 @@ docs:
image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
variables:
BST_EXT_URL: git+https://gitlab.com/BuildStream/bst-plugins-experimental.git
- BST_EXT_REF: 0.12.0-173-gbe5ac19#egg=bst_plugins_experimental[ostree,cargo]
+ BST_EXT_REF: 1.93.0-0-g4dea6cfc9c5f64e4b8e5fe5bb76f34c77af891a0#egg=bst_plugins_experimental[ostree,cargo,tar]
FD_SDK_REF: freedesktop-sdk-19.08.5-buildstream2-0-g22588e2f41acecbcfc555942eb3086296bc14c6c
before_script:
- |
diff --git a/doc/examples/autotools/project.conf b/doc/examples/autotools/project.conf
index 6c11423..617de5d 100644
--- a/doc/examples/autotools/project.conf
+++ b/doc/examples/autotools/project.conf
@@ -11,3 +11,9 @@ element-path: elements
aliases:
alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
gnu: http://ftpmirror.gnu.org/gnu/automake/
+
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/doc/examples/developing/project.conf b/doc/examples/developing/project.conf
index 3b09209..ec3bae2 100644
--- a/doc/examples/developing/project.conf
+++ b/doc/examples/developing/project.conf
@@ -10,3 +10,9 @@ element-path: elements
# Define an alias for our alpine tarball
aliases:
alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/doc/examples/integration-commands/project.conf b/doc/examples/integration-commands/project.conf
index c6c5990..7fffef0 100644
--- a/doc/examples/integration-commands/project.conf
+++ b/doc/examples/integration-commands/project.conf
@@ -10,3 +10,9 @@ element-path: elements
# Define an alias for our alpine tarball
aliases:
alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/doc/examples/running-commands/project.conf b/doc/examples/running-commands/project.conf
index aafb6a9..b5ef564 100644
--- a/doc/examples/running-commands/project.conf
+++ b/doc/examples/running-commands/project.conf
@@ -10,3 +10,9 @@ element-path: elements
# Define an alias for our alpine tarball
aliases:
alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst
index 7069167..bfd1487 100644
--- a/doc/source/core_plugins.rst
+++ b/doc/source/core_plugins.rst
@@ -44,7 +44,6 @@ Sources
sources/local
sources/remote
- sources/tar
sources/zip
sources/git
sources/bzr
diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py
deleted file mode 100644
index fc4f573..0000000
--- a/src/buildstream/plugins/sources/tar.py
+++ /dev/null
@@ -1,253 +0,0 @@
-#
-# Copyright (C) 2017 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-# Authors:
-# Jonathan Maw <jo...@codethink.co.uk>
-
-"""
-tar - stage files from tar archives
-===================================
-
-**Host dependencies:**
-
- * lzip (for .tar.lz files)
-
-**Usage:**
-
-.. code:: yaml
-
- # Specify the tar source kind
- kind: tar
-
- # Specify the tar url. Using an alias defined in your project
- # configuration is encouraged. 'bst source track' will update the
- # sha256sum in 'ref' to the downloaded file's sha256sum.
- url: upstream:foo.tar
-
- # Specify the ref. It's a sha256sum of the file you download.
- ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
-
- # Specify a glob pattern to indicate the base directory to extract
- # from the tarball. The first matching directory will be used.
- #
- # Note that this is '*' by default since most standard release
- # tarballs contain a self named subdirectory at the root which
- # contains the files one normally wants to extract to build.
- #
- # To extract the root of the tarball directly, this can be set
- # to an empty string.
- base-dir: '*'
-
-See :ref:`built-in functionality doumentation <core_source_builtins>` for
-details on common configuration options for sources.
-"""
-
-import os
-import tarfile
-from contextlib import contextmanager
-from tempfile import TemporaryFile
-
-from buildstream import SourceError
-from buildstream import utils
-
-from ._downloadablefilesource import DownloadableFileSource
-
-
-class ReadableTarInfo(tarfile.TarInfo):
- """
- The goal is to override `TarFile`'s `extractall` semantics by ensuring that on extraction, the
- files are readable by the owner of the file. This is done by overriding the accessor for the
- `mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball,
- so that the owner-read bit is always set.
- """
-
- @property
- def mode(self):
- # Respect umask instead of the file mode stored in the archive.
- # The only bit used from the embedded mode is the executable bit for files.
- umask = utils.get_umask()
- if self.isdir() or bool(self.__permission | 0o100):
- return 0o777 & ~umask
- else:
- return 0o666 & ~umask
-
- @mode.setter
- def mode(self, permission):
- self.__permission = permission # pylint: disable=attribute-defined-outside-init
-
-
-class TarSource(DownloadableFileSource):
- # pylint: disable=attribute-defined-outside-init
-
- def configure(self, node):
- super().configure(node)
-
- self.base_dir = node.get_str("base-dir", "*")
- node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"])
-
- def preflight(self):
- self.host_lzip = None
- if self.url.endswith(".lz"):
- self.host_lzip = utils.get_host_tool("lzip")
-
- def get_unique_key(self):
- return super().get_unique_key() + [self.base_dir]
-
- @contextmanager
- def _run_lzip(self):
- assert self.host_lzip
- with TemporaryFile() as lzip_stdout:
- with open(self._get_mirror_file(), "r") as lzip_file:
- self.call([self.host_lzip, "-d"], stdin=lzip_file, stdout=lzip_stdout)
-
- lzip_stdout.seek(0, 0)
- yield lzip_stdout
-
- @contextmanager
- def _get_tar(self):
- if self.url.endswith(".lz"):
- with self._run_lzip() as lzip_dec:
- with tarfile.open(fileobj=lzip_dec, mode="r:", tarinfo=ReadableTarInfo) as tar:
- yield tar
- else:
- with tarfile.open(self._get_mirror_file(), tarinfo=ReadableTarInfo) as tar:
- yield tar
-
- def stage(self, directory):
- try:
- with self._get_tar() as tar:
- base_dir = None
- if self.base_dir:
- base_dir = self._find_base_dir(tar, self.base_dir)
-
- if base_dir:
- tar.extractall(path=directory, members=self._extract_members(tar, base_dir, directory))
- else:
- tar.extractall(path=directory)
-
- except (tarfile.TarError, OSError) as e:
- raise SourceError("{}: Error staging source: {}".format(self, e)) from e
-
- # Override and translate which filenames to extract
- def _extract_members(self, tar, base_dir, target_dir):
-
- # Assert that a tarfile is safe to extract; specifically, make
- # sure that we don't do anything outside of the target
- # directory (this is possible, if, say, someone engineered a
- # tarfile to contain paths that start with ..).
- def assert_safe(member):
- final_path = os.path.abspath(os.path.join(target_dir, member.path))
- if not final_path.startswith(target_dir):
- raise SourceError(
- "{}: Tarfile attempts to extract outside the staging area: "
- "{} -> {}".format(self, member.path, final_path)
- )
-
- if member.islnk():
- linked_path = os.path.abspath(os.path.join(target_dir, member.linkname))
- if not linked_path.startswith(target_dir):
- raise SourceError(
- "{}: Tarfile attempts to hardlink outside the staging area: "
- "{} -> {}".format(self, member.path, final_path)
- )
-
- # Don't need to worry about symlinks because they're just
- # files here and won't be able to do much harm once we are
- # in a sandbox.
-
- if not base_dir.endswith(os.sep):
- base_dir = base_dir + os.sep
-
- L = len(base_dir)
- for member in tar.getmembers():
-
- # First, ensure that a member never starts with `./`
- if member.path.startswith("./"):
- member.path = member.path[2:]
- if member.islnk() and member.linkname.startswith("./"):
- member.linkname = member.linkname[2:]
-
- # Now extract only the paths which match the normalized path
- if member.path.startswith(base_dir):
- # Hardlinks are smart and collapse into the "original"
- # when their counterpart doesn't exist. This means we
- # only need to modify links to files whose location we
- # change.
- #
- # Since we assert that we're not linking to anything
- # outside the target directory, this should only ever
- # be able to link to things inside the target
- # directory, so we should cover all bases doing this.
- #
- if member.islnk() and member.linkname.startswith(base_dir):
- member.linkname = member.linkname[L:]
-
- member.path = member.path[L:]
-
- assert_safe(member)
- yield member
-
- # We want to iterate over all paths of a tarball, but getmembers()
- # is not enough because some tarballs simply do not contain the leading
- # directory paths for the archived files.
- def _list_tar_paths(self, tar):
-
- visited = set()
- for member in tar.getmembers():
-
- # Remove any possible leading './', offer more consistent behavior
- # across tarballs encoded with or without a leading '.'
- member_name = member.name.lstrip("./")
-
- if not member.isdir():
-
- # Loop over the components of a path, for a path of a/b/c/d
- # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding
- # the final component
- components = member_name.split("/")
- for i in range(len(components) - 1):
- dir_component = "/".join([components[j] for j in range(i + 1)])
- if dir_component not in visited:
- visited.add(dir_component)
- try:
- # Dont yield directory members which actually do
- # exist in the archive
- _ = tar.getmember(dir_component)
- except KeyError:
- if dir_component != ".":
- yield dir_component
-
- continue
-
- # Avoid considering the '.' directory, if any is included in the archive
- # this is to avoid the default 'base-dir: *' value behaving differently
- # depending on whether the tarball was encoded with a leading '.' or not
- if member_name == ".":
- continue
-
- yield member_name
-
- def _find_base_dir(self, tar, pattern):
- paths = self._list_tar_paths(tar)
- matches = sorted(list(utils.glob(paths, pattern)))
- if not matches:
- raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern))
-
- return matches[0]
-
-
-def setup():
- return TarSource
diff --git a/src/buildstream/testing/_sourcetests/project/project.conf b/src/buildstream/testing/_sourcetests/project/project.conf
index 05b68bf..96291e9 100644
--- a/src/buildstream/testing/_sourcetests/project/project.conf
+++ b/src/buildstream/testing/_sourcetests/project/project.conf
@@ -25,3 +25,9 @@ split-rules:
fatal-warnings:
- bad-element-suffix
- bad-characters-in-name
+
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py
index 0dfbce2..a04f652 100644
--- a/src/buildstream/testing/_utils/site.py
+++ b/src/buildstream/testing/_utils/site.py
@@ -53,12 +53,6 @@ except ProgramNotFoundError:
HAVE_BWRAP = False
HAVE_BWRAP_JSON_STATUS = False
-try:
- utils.get_host_tool("lzip")
- HAVE_LZIP = True
-except ProgramNotFoundError:
- HAVE_LZIP = False
-
casd_path = utils.get_host_tool("buildbox-casd")
CASD_SEPARATE_USER = bool(os.stat(casd_path).st_mode & stat.S_ISUID)
del casd_path
diff --git a/tests/cachekey/project/sources/tar1.bst b/tests/cachekey/project/sources/tar1.bst
deleted file mode 100644
index 1dbf27c..0000000
--- a/tests/cachekey/project/sources/tar1.bst
+++ /dev/null
@@ -1,5 +0,0 @@
-kind: import
-sources:
-- kind: tar
- url: https://example.com/releases/1.4/foo-1.4.5.tar.gz
- ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
deleted file mode 100644
index 1531384..0000000
--- a/tests/cachekey/project/sources/tar1.expected
+++ /dev/null
@@ -1 +0,0 @@
-6b550e20ab7b8a11912ca14171e39c76badf7fa161a01c83d817c789b84e45c3
diff --git a/tests/cachekey/project/sources/tar2.bst b/tests/cachekey/project/sources/tar2.bst
deleted file mode 100644
index e357be3..0000000
--- a/tests/cachekey/project/sources/tar2.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-sources:
-- kind: tar
- url: https://example.com/releases/1.4/foo-1.4.5.tar.gz
- ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
- base-dir: src
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
deleted file mode 100644
index 6440dfa..0000000
--- a/tests/cachekey/project/sources/tar2.expected
+++ /dev/null
@@ -1 +0,0 @@
-f890b611cc83036b9c52dddf4eb2a02ccac5a73ae3ddcb34586406d7deba5a11
diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst
index cabf3f7..5d385de 100644
--- a/tests/cachekey/project/target.bst
+++ b/tests/cachekey/project/target.bst
@@ -16,8 +16,6 @@ depends:
- sources/pip1.bst
- sources/remote1.bst
- sources/remote2.bst
-- sources/tar1.bst
-- sources/tar2.bst
- sources/zip1.bst
- sources/zip2.bst
- elements/build1.bst
diff --git a/tests/conftest.py b/tests/conftest.py
index 8d33fa0..4488bb3 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -32,7 +32,6 @@ from buildstream.testing.integration import integration_cache # pylint: disable
from tests.testutils.repo.git import Git
from tests.testutils.repo.bzr import Bzr
-from tests.testutils.repo.tar import Tar
from tests.testutils.repo.zip import Zip
@@ -112,7 +111,6 @@ def remote_services(request):
#################################################
register_repo_kind("git", Git, None)
register_repo_kind("bzr", Bzr, None)
-register_repo_kind("tar", Tar, None)
register_repo_kind("zip", Zip, None)
diff --git a/tests/integration/project/project.conf b/tests/integration/project/project.conf
index ddfe47b..f97b613 100644
--- a/tests/integration/project/project.conf
+++ b/tests/integration/project/project.conf
@@ -21,3 +21,9 @@ split-rules:
/tests
- |
/tests/*
+
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/tests/remoteexecution/project/project.conf b/tests/remoteexecution/project/project.conf
index ddfe47b..63ca319 100644
--- a/tests/remoteexecution/project/project.conf
+++ b/tests/remoteexecution/project/project.conf
@@ -21,3 +21,8 @@ split-rules:
/tests
- |
/tests/*
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/tests/sandboxes/project/project.conf b/tests/sandboxes/project/project.conf
index ddfe47b..63ca319 100644
--- a/tests/sandboxes/project/project.conf
+++ b/tests/sandboxes/project/project.conf
@@ -21,3 +21,8 @@ split-rules:
/tests
- |
/tests/*
+plugins:
+- origin: pip
+ package-name: bst-plugins-experimental
+ sources:
+ tar: 0
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
deleted file mode 100644
index 8ac6274..0000000
--- a/tests/sources/tar.py
+++ /dev/null
@@ -1,471 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-from shutil import copyfile
-import subprocess
-import tarfile
-import tempfile
-import urllib.parse
-
-import pytest
-
-from buildstream import utils
-from buildstream.exceptions import ErrorDomain
-from buildstream.testing import generate_project, generate_element
-from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing._utils.site import HAVE_LZIP
-from tests.testutils.file_server import create_file_server
-from . import list_dir_contents
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tar",)
-
-
-def _assemble_tar(workingdir, srcdir, dstfile):
- old_dir = os.getcwd()
- os.chdir(workingdir)
- with tarfile.open(dstfile, "w:gz") as tar:
- tar.add(srcdir)
- os.chdir(old_dir)
-
-
-def _assemble_tar_lz(workingdir, srcdir, dstfile):
- old_dir = os.getcwd()
- os.chdir(workingdir)
- with tempfile.TemporaryFile() as uncompressed:
- with tarfile.open(fileobj=uncompressed, mode="w:") as tar:
- tar.add(srcdir)
- uncompressed.seek(0, 0)
- with open(dstfile, "wb") as dst:
- subprocess.call(["lzip"], stdin=uncompressed, stdout=dst)
- os.chdir(old_dir)
-
-
-# Test that without ref, consistency is set appropriately.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
-def test_no_ref(cli, tmpdir, datafiles):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
- assert cli.get_element_state(project, "target.bst") == "no reference"
-
-
-# Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-def test_fetch_bad_url(cli, tmpdir, datafiles):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
-
- # Try to fetch it
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- assert "FAILURE Try #" in result.stderr
- result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.SOURCE, None)
-
-
-# Test that when I fetch with an invalid ref, it fails.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-def test_fetch_bad_ref(cli, tmpdir, datafiles):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
-
- # Try to fetch it
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.SOURCE, None)
-
-
-# Test that when tracking with a ref set, there is a warning
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-def test_track_warning(cli, tmpdir, datafiles):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
-
- # Track it
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_success()
- assert "Potential man-in-the-middle attack!" in result.stderr
-
-
-# Test that a staged checkout matches what was tarred up, with the default first subdir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-@pytest.mark.parametrize("srcdir", ["a", "./a"])
-def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
- checkoutdir = os.path.join(str(tmpdir), "checkout")
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
-
- # Track, fetch, build, checkout
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["build", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
- result.assert_success()
-
- # Check that the content of the first directory is checked out (base-dir: '*')
- original_dir = os.path.join(str(datafiles), "content", "a")
- original_contents = list_dir_contents(original_dir)
- checkout_contents = list_dir_contents(checkoutdir)
- assert checkout_contents == original_contents
-
-
-# Test that a staged checkout matches what was tarred up, with an empty base-dir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir"))
-@pytest.mark.parametrize("srcdir", ["a", "./a"])
-def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
- checkoutdir = os.path.join(str(tmpdir), "checkout")
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
-
- # Track, fetch, build, checkout
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["build", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
- result.assert_success()
-
- # Check that the full content of the tarball is checked out (base-dir: '')
- original_dir = os.path.join(str(datafiles), "content")
- original_contents = list_dir_contents(original_dir)
- checkout_contents = list_dir_contents(checkoutdir)
- assert checkout_contents == original_contents
-
-
-# Test that a staged checkout matches what was tarred up, with an explicit basedir
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir"))
-@pytest.mark.parametrize("srcdir", ["a", "./a"])
-def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
- checkoutdir = os.path.join(str(tmpdir), "checkout")
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
-
- # Track, fetch, build, checkout
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["build", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
- result.assert_success()
-
- # Check that the content of the first directory is checked out (base-dir: '*')
- original_dir = os.path.join(str(datafiles), "content", "a")
- original_contents = list_dir_contents(original_dir)
- checkout_contents = list_dir_contents(checkoutdir)
- assert checkout_contents == original_contents
-
-
-# Test that we succeed to extract tarballs with hardlinks when stripping the
-# leading paths
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "contains-links"))
-def test_stage_contains_links(cli, tmpdir, datafiles):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
- checkoutdir = os.path.join(str(tmpdir), "checkout")
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.gz")
-
- # Create a hardlink, we wont trust git to store that info for us
- os.makedirs(os.path.join(str(datafiles), "content", "base-directory", "subdir2"), exist_ok=True)
- file1 = os.path.join(str(datafiles), "content", "base-directory", "subdir1", "file.txt")
- file2 = os.path.join(str(datafiles), "content", "base-directory", "subdir2", "file.txt")
- os.link(file1, file2)
-
- _assemble_tar(os.path.join(str(datafiles), "content"), "base-directory", src_tar)
-
- # Track, fetch, build, checkout
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["build", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
- result.assert_success()
-
- # Check that the content of the first directory is checked out (base-dir: '*')
- original_dir = os.path.join(str(datafiles), "content", "base-directory")
- original_contents = list_dir_contents(original_dir)
- checkout_contents = list_dir_contents(checkoutdir)
- assert checkout_contents == original_contents
-
-
-@pytest.mark.skipif(not HAVE_LZIP, reason="lzip is not available")
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-@pytest.mark.parametrize("srcdir", ["a", "./a"])
-def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
- checkoutdir = os.path.join(str(tmpdir), "checkout")
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.lz")
- _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
-
- # Track, fetch, build, checkout
- result = cli.run(project=project, args=["source", "track", "target-lz.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target-lz.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["build", "target-lz.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["artifact", "checkout", "target-lz.bst", "--directory", checkoutdir])
- result.assert_success()
-
- # Check that the content of the first directory is checked out (base-dir: '*')
- original_dir = os.path.join(str(datafiles), "content", "a")
- original_contents = list_dir_contents(original_dir)
- checkout_contents = list_dir_contents(checkoutdir)
- assert checkout_contents == original_contents
-
-
-# Test that tarballs with read-only files work
-# a - contains read-only files in a writable directory
-# b - root directory has read-only permission
-# c - contains one file that has no read nor write permissions. Base-dir set to '' to extract root of tarball
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "read-only"))
-@pytest.mark.parametrize("tar_name, base_dir", [("a", "*"), ("b", "*"), ("c", "")])
-def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir):
- try:
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
-
- tar_file = "{}.tar.gz".format(tar_name)
-
- generate_element(
- project,
- "target.bst",
- {
- "kind": "import",
- "sources": [{"kind": "tar", "url": "tmpdir:/{}".format(tar_file), "ref": "foo", "base-dir": base_dir}],
- },
- )
-
- # Get the tarball in tests/sources/tar/read-only/content
- #
- # NOTE that we need to do this because tarfile.open and tar.add()
- # are packing the tar up with writeable files and dirs
- tarball = os.path.join(str(datafiles), "content", tar_file)
- if not os.path.exists(tarball):
- raise FileNotFoundError("{} does not exist".format(tarball))
- copyfile(tarball, os.path.join(str(tmpdir), tar_file))
-
- # Because this test can potentially leave directories behind
- # which are difficult to remove, ask buildstream to use
- # our temp directory, so we can clean up.
- tmpdir_str = str(tmpdir)
- if not tmpdir_str.endswith(os.path.sep):
- tmpdir_str += os.path.sep
- env = {"TMP": tmpdir_str}
-
- # Track, fetch, build, checkout
- result = cli.run(project=project, args=["source", "track", "target.bst"], env=env)
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target.bst"], env=env)
- result.assert_success()
- result = cli.run(project=project, args=["build", "target.bst"], env=env)
- result.assert_success()
-
- finally:
- utils._force_rmtree(str(tmpdir))
-
-
-@pytest.mark.parametrize("server_type", ("FTP", "HTTP"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-def test_use_netrc(cli, datafiles, server_type, tmpdir):
- file_server_files = os.path.join(str(tmpdir), "file_server")
- fake_home = os.path.join(str(tmpdir), "fake_home")
- os.makedirs(file_server_files, exist_ok=True)
- os.makedirs(fake_home, exist_ok=True)
- project = str(datafiles)
- checkoutdir = os.path.join(str(tmpdir), "checkout")
-
- os.environ["HOME"] = fake_home
- with open(os.path.join(fake_home, ".netrc"), "wb") as f:
- os.fchmod(f.fileno(), 0o700)
- f.write(b"machine 127.0.0.1\n")
- f.write(b"login testuser\n")
- f.write(b"password 12345\n")
-
- with create_file_server(server_type) as server:
- server.add_user("testuser", "12345", file_server_files)
- generate_project(project, config={"aliases": {"tmpdir": server.base_url()}})
-
- src_tar = os.path.join(file_server_files, "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
-
- server.start()
-
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["build", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
- result.assert_success()
-
- original_dir = os.path.join(str(datafiles), "content", "a")
- original_contents = list_dir_contents(original_dir)
- checkout_contents = list_dir_contents(checkoutdir)
- assert checkout_contents == original_contents
-
-
-@pytest.mark.parametrize("server_type", ("FTP", "HTTP"))
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir):
- file_server_files = os.path.join(str(tmpdir), "file_server")
- fake_home = os.path.join(str(tmpdir), "fake_home")
- os.makedirs(file_server_files, exist_ok=True)
- os.makedirs(fake_home, exist_ok=True)
- project = str(datafiles)
-
- os.environ["HOME"] = fake_home
- with open(os.path.join(fake_home, ".netrc"), "wb") as f:
- os.fchmod(f.fileno(), 0o700)
- f.write(b"machine 127.0.0.1\n")
- f.write(b"login testuser\n")
- f.write(b"password 12345\n")
-
- with create_file_server(server_type) as server:
- server.add_user("otheruser", "12345", file_server_files)
- parts = urllib.parse.urlsplit(server.base_url())
- base_url = urllib.parse.urlunsplit([parts[0], "otheruser@{}".format(parts[1]), *parts[2:]])
- generate_project(project, config={"aliases": {"tmpdir": base_url}})
-
- src_tar = os.path.join(file_server_files, "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
-
- server.start()
-
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_main_error(ErrorDomain.STREAM, None)
- result.assert_task_error(ErrorDomain.SOURCE, None)
-
-
-# Test that BuildStream doesnt crash if HOME is unset while
-# the netrc module is trying to find it's ~/.netrc file.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch"))
-def test_homeless_environment(cli, tmpdir, datafiles):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
-
- # Create a local tar
- src_tar = os.path.join(str(tmpdir), "a.tar.gz")
- _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
-
- # Use a track, make sure the plugin tries to find a ~/.netrc
- result = cli.run(project=project, args=["source", "track", "target.bst"], env={"HOME": None})
- result.assert_success()
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks"))
-def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
- def ensure_link(member):
- # By default, python will simply duplicate files - we want
- # hardlinks!
- if member.path == "contents/to_extract/a":
- member.type = tarfile.LNKTYPE
- member.linkname = "contents/elsewhere/a"
- return member
-
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
- checkoutdir = os.path.join(str(tmpdir), "checkout")
-
- # Create a tarball with an odd hardlink
- src_tar = os.path.join(str(tmpdir), "contents.tar.gz")
- old_dir = os.getcwd()
- os.chdir(str(tmpdir))
- with tarfile.open(src_tar, "w:gz") as tar:
- tar.add("contents", filter=ensure_link)
- os.chdir(old_dir)
-
- # Make sure our tarfile is actually created with the desired
- # attributes set
- with tarfile.open(src_tar, "r:gz") as tar:
- assert any(
- member.islnk() and member.path == "contents/to_extract/a" and member.linkname == "contents/elsewhere/a"
- for member in tar.getmembers()
- )
-
- # Assert that we will actually create a singular copy of the file
- result = cli.run(project=project, args=["source", "track", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["build", "target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir])
- result.assert_success()
-
- original_dir = os.path.join(str(datafiles), "contents", "to_extract")
- original_contents = list_dir_contents(original_dir)
- checkout_contents = list_dir_contents(checkoutdir)
- assert checkout_contents == original_contents
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks"))
-def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles):
- project = str(datafiles)
- generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}})
-
- # Create a maliciously-hardlinked tarball
- def ensure_link(member):
- # By default, python will simply duplicate files - we want
- # hardlinks!
- if member.path == "contents/elsewhere/malicious":
- member.type = tarfile.LNKTYPE
- # This should not be allowed
- member.linkname = "../../../malicious_target.bst"
- return member
-
- src_tar = os.path.join(str(tmpdir), "contents.tar.gz")
- old_dir = os.getcwd()
- os.chdir(str(tmpdir))
- with tarfile.open(src_tar, "w:gz") as tar:
- tar.add("contents", filter=ensure_link)
- os.chdir(old_dir)
-
- # Make sure our tarfile is actually created with the desired
- # attributes set
- with tarfile.open(src_tar, "r:gz") as tar:
- assert any(
- member.islnk()
- and member.path == "contents/elsewhere/malicious"
- and member.linkname == "../../../malicious_target.bst"
- for member in tar.getmembers()
- )
-
- # Try to execute the exploit
- result = cli.run(project=project, args=["source", "track", "malicious_target.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "fetch", "malicious_target.bst"])
- result.assert_main_error(ErrorDomain.STREAM, None)
diff --git a/tests/sources/tar/contains-links/content/base-directory/subdir1/file.txt b/tests/sources/tar/contains-links/content/base-directory/subdir1/file.txt
deleted file mode 100644
index f621448..0000000
--- a/tests/sources/tar/contains-links/content/base-directory/subdir1/file.txt
+++ /dev/null
@@ -1 +0,0 @@
-pony
diff --git a/tests/sources/tar/contains-links/target.bst b/tests/sources/tar/contains-links/target.bst
deleted file mode 100644
index ad413a2..0000000
--- a/tests/sources/tar/contains-links/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/a.tar.gz
- ref: foo
diff --git a/tests/sources/tar/explicit-basedir/content/a/b/d b/tests/sources/tar/explicit-basedir/content/a/b/d
deleted file mode 100644
index 4bcfe98..0000000
--- a/tests/sources/tar/explicit-basedir/content/a/b/d
+++ /dev/null
@@ -1 +0,0 @@
-d
diff --git a/tests/sources/tar/explicit-basedir/content/a/c b/tests/sources/tar/explicit-basedir/content/a/c
deleted file mode 100644
index f2ad6c7..0000000
--- a/tests/sources/tar/explicit-basedir/content/a/c
+++ /dev/null
@@ -1 +0,0 @@
-c
diff --git a/tests/sources/tar/explicit-basedir/target.bst b/tests/sources/tar/explicit-basedir/target.bst
deleted file mode 100644
index fb85bdf..0000000
--- a/tests/sources/tar/explicit-basedir/target.bst
+++ /dev/null
@@ -1,7 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/a.tar.gz
- ref: foo
- base-dir: 'a'
diff --git a/tests/sources/tar/fetch/content/a/b/d b/tests/sources/tar/fetch/content/a/b/d
deleted file mode 100644
index 4bcfe98..0000000
--- a/tests/sources/tar/fetch/content/a/b/d
+++ /dev/null
@@ -1 +0,0 @@
-d
diff --git a/tests/sources/tar/fetch/content/a/c b/tests/sources/tar/fetch/content/a/c
deleted file mode 100644
index f2ad6c7..0000000
--- a/tests/sources/tar/fetch/content/a/c
+++ /dev/null
@@ -1 +0,0 @@
-c
diff --git a/tests/sources/tar/fetch/target-lz.bst b/tests/sources/tar/fetch/target-lz.bst
deleted file mode 100644
index b056912..0000000
--- a/tests/sources/tar/fetch/target-lz.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/a.tar.lz
- ref: foo
diff --git a/tests/sources/tar/fetch/target.bst b/tests/sources/tar/fetch/target.bst
deleted file mode 100644
index ad413a2..0000000
--- a/tests/sources/tar/fetch/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/a.tar.gz
- ref: foo
diff --git a/tests/sources/tar/no-basedir/content/a/b/d b/tests/sources/tar/no-basedir/content/a/b/d
deleted file mode 100644
index 4bcfe98..0000000
--- a/tests/sources/tar/no-basedir/content/a/b/d
+++ /dev/null
@@ -1 +0,0 @@
-d
diff --git a/tests/sources/tar/no-basedir/content/a/c b/tests/sources/tar/no-basedir/content/a/c
deleted file mode 100644
index f2ad6c7..0000000
--- a/tests/sources/tar/no-basedir/content/a/c
+++ /dev/null
@@ -1 +0,0 @@
-c
diff --git a/tests/sources/tar/no-basedir/target.bst b/tests/sources/tar/no-basedir/target.bst
deleted file mode 100644
index 5a34d87..0000000
--- a/tests/sources/tar/no-basedir/target.bst
+++ /dev/null
@@ -1,7 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/a.tar.gz
- ref: foo
- base-dir: ''
diff --git a/tests/sources/tar/no-ref/a/b/d b/tests/sources/tar/no-ref/a/b/d
deleted file mode 100644
index 4bcfe98..0000000
--- a/tests/sources/tar/no-ref/a/b/d
+++ /dev/null
@@ -1 +0,0 @@
-d
diff --git a/tests/sources/tar/no-ref/a/c b/tests/sources/tar/no-ref/a/c
deleted file mode 100644
index f2ad6c7..0000000
--- a/tests/sources/tar/no-ref/a/c
+++ /dev/null
@@ -1 +0,0 @@
-c
diff --git a/tests/sources/tar/no-ref/target.bst b/tests/sources/tar/no-ref/target.bst
deleted file mode 100644
index 5a73fa2..0000000
--- a/tests/sources/tar/no-ref/target.bst
+++ /dev/null
@@ -1,5 +0,0 @@
-kind: autotools
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/a.tar.gz
diff --git a/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/a b/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/a
deleted file mode 100644
index ce01362..0000000
--- a/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/a
+++ /dev/null
@@ -1 +0,0 @@
-hello
diff --git a/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/malicious b/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/malicious
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/a b/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/a
deleted file mode 100644
index ce01362..0000000
--- a/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/a
+++ /dev/null
@@ -1 +0,0 @@
-hello
diff --git a/tests/sources/tar/out-of-basedir-hardlinks/malicious_target.bst b/tests/sources/tar/out-of-basedir-hardlinks/malicious_target.bst
deleted file mode 100644
index b9debe9..0000000
--- a/tests/sources/tar/out-of-basedir-hardlinks/malicious_target.bst
+++ /dev/null
@@ -1,5 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/contents.tar.gz
diff --git a/tests/sources/tar/out-of-basedir-hardlinks/target.bst b/tests/sources/tar/out-of-basedir-hardlinks/target.bst
deleted file mode 100644
index 4f07e2e..0000000
--- a/tests/sources/tar/out-of-basedir-hardlinks/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/contents.tar.gz
- base-dir: contents/to_extract
diff --git a/tests/sources/tar/read-only/content/a.tar.gz b/tests/sources/tar/read-only/content/a.tar.gz
deleted file mode 100644
index 7092262..0000000
Binary files a/tests/sources/tar/read-only/content/a.tar.gz and /dev/null differ
diff --git a/tests/sources/tar/read-only/content/b.tar.gz b/tests/sources/tar/read-only/content/b.tar.gz
deleted file mode 100644
index 79fd1e8..0000000
Binary files a/tests/sources/tar/read-only/content/b.tar.gz and /dev/null differ
diff --git a/tests/sources/tar/read-only/content/c.tar.gz b/tests/sources/tar/read-only/content/c.tar.gz
deleted file mode 100644
index 016a5a2..0000000
Binary files a/tests/sources/tar/read-only/content/c.tar.gz and /dev/null differ
diff --git a/tests/sources/tar/read-only/target.bst b/tests/sources/tar/read-only/target.bst
deleted file mode 100644
index ad413a2..0000000
--- a/tests/sources/tar/read-only/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: tar
- url: tmpdir:/a.tar.gz
- ref: foo
diff --git a/tests/testutils/repo/tar.py b/tests/testutils/repo/tar.py
deleted file mode 100644
index 3eb9d89..0000000
--- a/tests/testutils/repo/tar.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import os
-import tarfile
-
-from buildstream.utils import sha256sum
-
-from buildstream.testing import Repo
-
-
-class Tar(Repo):
- def create(self, directory):
- tarball = os.path.join(self.repo, "file.tar.gz")
-
- old_dir = os.getcwd()
- os.chdir(directory)
- with tarfile.open(tarball, "w:gz") as tar:
- tar.add(".")
- os.chdir(old_dir)
-
- return sha256sum(tarball)
-
- def source_config(self, ref=None):
- tarball = os.path.join(self.repo, "file.tar.gz")
- config = {"kind": "tar", "url": "file://" + tarball, "directory": "", "base-dir": ""}
- if ref is not None:
- config["ref"] = ref
-
- return config
diff --git a/tox.ini b/tox.ini
index 30075ee..5dd17ee 100644
--- a/tox.ini
+++ b/tox.ini
@@ -37,7 +37,7 @@ deps =
py{35,36,37,38}: -rrequirements/dev-requirements.txt
# Install external plugins for plugin tests
- py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree,deb]
+ py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree,deb,tar]
# Only require coverage and pytest-cov when using it
!nocover: -rrequirements/cov-requirements.txt
@@ -173,7 +173,7 @@ deps =
sphinx_rtd_theme >= 0.4.2
pytest
-rrequirements/requirements.txt
- git+https://gitlab.com/BuildStream/bst-plugins-experimental.git@be5ac19e5062bc23a46ed8ce7aa2958a2653c917#egg=bst_plugins_experimental[ostree]
+ git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree,deb,tar]
passenv =
BST_FORCE_SESSION_REBUILD
BST_SOURCE_CACHE