You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:16:29 UTC
[buildstream] branch becky/locally_downloaded_files created (now
60ec787)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch becky/locally_downloaded_files
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at 60ec787 _downloadablefilesource: Tidy up
This branch includes the following new commits:
new 3795f16 Allowing the tar plugin to use a local path
new e1a91ee Adding test for local source download
new b67afa8 Additional test for invalid path
new 60ec787 _downloadablefilesource: Tidy up
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 01/04: Allowing the tar plugin to use a local path
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch becky/locally_downloaded_files
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 3795f16a84407d3c5792d6bcff77bd980d134e4f
Author: Rebecca Grayson <be...@hotmail.co.uk>
AuthorDate: Mon Jun 24 16:09:51 2019 +0100
Allowing the tar plugin to use a local path
_downloadablefilesource.py: Added ability to download from either url or path
Added changes to the code so that now a downloadable file can be sourced
from either a given url or a local path. If both are given, error occurs.
tar.py: Made changes to accept the path sourcing
When a path is given, the tar can be found from the local directory
rather than the mirror directory. Made changes to make the code less
specific to urls so that now it will accept both url and path.
---
.../plugins/sources/_downloadablefilesource.py | 136 +++++++++++++++------
src/buildstream/plugins/sources/tar.py | 43 +++++--
2 files changed, 133 insertions(+), 46 deletions(-)
diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py
index b9b15e2..127047c 100644
--- a/src/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/src/buildstream/plugins/sources/_downloadablefilesource.py
@@ -72,58 +72,100 @@ class _NetrcPasswordManager:
class DownloadableFileSource(Source):
# pylint: disable=attribute-defined-outside-init
- COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
+ COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag', 'path']
__urlopener = None
def configure(self, node):
- self.original_url = self.node_get_member(node, str, 'url')
+ self.original_url = self.node_get_member(node, str, 'url', None)
+ self.original_path = self.node_get_member(node, str, 'path', None)
self.ref = self.node_get_member(node, str, 'ref', None)
- self.url = self.translate_url(self.original_url)
self._warn_deprecated_etag(node)
+ # XXX: Looks like this should be in preflight but child classes are not calling super()
+ if self.original_url is None and self.original_path is None:
+ raise SourceError("Please specify either a 'path' or a 'url'.")
+ if self.original_url is not None and self.original_path is not None:
+ self.url_provenance = self.node_provenance(node, member_name='url')
+ self.path_provenance = self.node_provenance(node, member_name='path')
+
+ raise SourceError("You cannot specify both 'path' ({}) and 'url' ({})"
+ .format(self.path_provenance, self.url_provenance))
+
+ # If using a url, use appropriate checks
+ if self.original_url is not None:
+ self.url = self.translate_url(self.original_url)
+
+ # If using path, do checks
+ if self.original_path is not None:
+ self.path = self.node_get_project_path(node, 'path')
+ self.fullpath = os.path.join(self.get_project_directory(), self.path)
+ self.sha = unique_key(self.fullpath)
+ else:
+ self.sha = self.ref
+
def preflight(self):
return
def get_unique_key(self):
- return [self.original_url, self.ref]
+ self.__unique_key = None
+ if self.original_url is not None:
+ return [self.original_url, self.ref]
+ else:
+ return [(os.path.basename(self.original_path)), self.sha]
def get_consistency(self):
- if self.ref is None:
- return Consistency.INCONSISTENT
+ if self.original_url is not None:
+ if self.ref is None:
+ return Consistency.INCONSISTENT
- if os.path.isfile(self._get_mirror_file()):
- return Consistency.CACHED
+ if os.path.isfile(self._get_mirror_file()):
+ return Consistency.CACHED
+
+ else:
+ return Consistency.RESOLVED
else:
- return Consistency.RESOLVED
+ return Consistency.CACHED
def load_ref(self, node):
- self.ref = self.node_get_member(node, str, 'ref', None)
- self._warn_deprecated_etag(node)
+ if self.original_url is not None:
+ self.ref = self.node_get_member(node, str, 'ref', None)
+ self._warn_deprecated_etag(node)
+ else:
+ pass
def get_ref(self):
- return self.ref
+ if self.original_url is not None:
+ return self.ref
+ else:
+ return None
def set_ref(self, ref, node):
- node['ref'] = self.ref = ref
+ if self.original_url is not None:
+ node['ref'] = self.ref = self.sha = ref
+ else:
+ pass
def track(self):
# there is no 'track' field in the source to determine what/whether
# or not to update refs, because tracking a ref is always a conscious
# decision by the user.
- with self.timed_activity("Tracking {}".format(self.url),
- silent_nested=True):
- new_ref = self._ensure_mirror()
-
- if self.ref and self.ref != new_ref:
- detail = "When tracking, new ref differs from current ref:\n" \
- + " Tracked URL: {}\n".format(self.url) \
- + " Current ref: {}\n".format(self.ref) \
- + " New ref: {}\n".format(new_ref)
- self.warn("Potential man-in-the-middle attack!", detail=detail)
-
- return new_ref
+ if self.original_url is not None:
+ with self.timed_activity("Tracking {}".format(self.url),
+ silent_nested=True):
+ new_ref = self._ensure_mirror()
+
+ if self.ref and self.ref != new_ref:
+ detail = "When tracking, new ref differs from current ref:\n" \
+ + " Tracked URL: {}\n".format(self.url) \
+ + " Current ref: {}\n".format(self.ref) \
+ + " New ref: {}\n".format(new_ref)
+ self.warn("Potential man-in-the-middle attack!", detail=detail)
+
+ return new_ref
+ else:
+ pass
def fetch(self):
@@ -131,16 +173,22 @@ class DownloadableFileSource(Source):
# file to be already cached because Source.fetch() will
# not be called if the source is already Consistency.CACHED.
#
- if os.path.isfile(self._get_mirror_file()):
- return # pragma: nocover
+ if self.original_url is not None:
+ if os.path.isfile(self._get_mirror_file()):
+ return # pragma: nocover
+
+ # Download the file, raise hell if the sha256sums don't match,
+ # and mirror the file otherwise.
+ with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
+ sha256 = self._ensure_mirror()
+ if sha256 != self.ref:
+ raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
+ .format(self.url, sha256, self.ref))
+ else:
+ pass
- # Download the file, raise hell if the sha256sums don't match,
- # and mirror the file otherwise.
- with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
- sha256 = self._ensure_mirror()
- if sha256 != self.ref:
- raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
- .format(self.url, sha256, self.ref))
+ def _get_local_path(self):
+ return self.path
def _warn_deprecated_etag(self, node):
etag = self.node_get_member(node, str, 'etag', None)
@@ -221,8 +269,12 @@ class DownloadableFileSource(Source):
.format(self, self.url, e), temporary=True) from e
def _get_mirror_dir(self):
+ if self.original_url is not None:
+ directory_name = utils.url_directory_name(self.original_url)
+ else:
+ directory_name = self.original_path
return os.path.join(self.get_mirror_directory(),
- utils.url_directory_name(self.original_url))
+ directory_name)
def _get_mirror_file(self, sha=None):
return os.path.join(self._get_mirror_dir(), sha or self.ref)
@@ -248,3 +300,17 @@ class DownloadableFileSource(Source):
ftp_handler = _NetrcFTPOpener(netrc_config)
DownloadableFileSource.__urlopener = urllib.request.build_opener(http_auth, ftp_handler)
return DownloadableFileSource.__urlopener
+
+
+# Create a unique key for a file
+def unique_key(filename):
+
+ # Return some hard coded things for files which
+ # have no content to calculate a key for
+ if os.path.islink(filename):
+ # For a symbolic link, use the link target as its unique identifier
+ return os.readlink(filename)
+ elif os.path.isdir(filename):
+ return "0"
+
+ return utils.sha256sum(filename)
diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py
index c90de74..0ba9e72 100644
--- a/src/buildstream/plugins/sources/tar.py
+++ b/src/buildstream/plugins/sources/tar.py
@@ -77,8 +77,12 @@ class TarSource(DownloadableFileSource):
self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir'])
def preflight(self):
+ if self.original_url is not None:
+ file_source = self.url
+ else:
+ file_source = self.path
self.host_lzip = None
- if self.url.endswith('.lz'):
+ if file_source.endswith('.lz'):
self.host_lzip = utils.get_host_tool('lzip')
def get_unique_key(self):
@@ -88,23 +92,40 @@ class TarSource(DownloadableFileSource):
def _run_lzip(self):
assert self.host_lzip
with TemporaryFile() as lzip_stdout:
- with open(self._get_mirror_file(), 'r') as lzip_file:
- self.call([self.host_lzip, '-d'],
- stdin=lzip_file,
- stdout=lzip_stdout)
-
- lzip_stdout.seek(0, 0)
- yield lzip_stdout
+ if self.original_url is not None:
+ with open(self._get_mirror_file(), 'r') as lzip_file:
+ self.call([self.host_lzip, '-d'],
+ stdin=lzip_file,
+ stdout=lzip_stdout)
+
+ lzip_stdout.seek(0, 0)
+ yield lzip_stdout
+ else:
+ with open(self.fullpath, 'r') as lzip_file:
+ self.call([self.host_lzip, '-d'],
+ stdin=lzip_file,
+ stdout=lzip_stdout)
+
+ lzip_stdout.seek(0, 0)
+ yield lzip_stdout
@contextmanager
def _get_tar(self):
- if self.url.endswith('.lz'):
+ if self.original_url is not None:
+ file_source = self.url
+ else:
+ file_source = self.path
+ if file_source.endswith('.lz'):
with self._run_lzip() as lzip_dec:
with tarfile.open(fileobj=lzip_dec, mode='r:') as tar:
yield tar
else:
- with tarfile.open(self._get_mirror_file()) as tar:
- yield tar
+ if self.original_url is not None:
+ with tarfile.open(self._get_mirror_file()) as tar:
+ yield tar
+ else:
+ with tarfile.open(self.fullpath) as tar:
+ yield tar
def stage(self, directory):
try:
[buildstream] 03/04: Additional test for invalid path
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch becky/locally_downloaded_files
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit b67afa85dc6a1a59eae751128deebaa55b250484
Author: Rebecca Grayson <be...@hotmail.co.uk>
AuthorDate: Thu Jun 27 15:08:54 2019 +0100
Additional test for invalid path
A test has been added to ensure if path is equal to anything other
than None, errors are handled correctly.
---
tests/sources/tar.py | 13 +++++++++++++
tests/sources/tar/fetch/target-no-path.bst | 5 +++++
2 files changed, 18 insertions(+)
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index 68791cb..553b6e2 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -86,6 +86,19 @@ def test_fetch_bad_url(cli, tmpdir, datafiles):
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
+# Test that when I fetch an invalid path, errors are handled gracefully.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_invalid_path(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target-no-path.bst'
+ ])
+ result.assert_main_error(ErrorDomain.STREAM, None)
+ result.assert_task_error(ErrorDomain.SOURCE, None)
+
# Test that when I fetch a nonexistent path, errors are handled gracefully.
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
def test_fetch_bad_path(cli, tmpdir, datafiles):
diff --git a/tests/sources/tar/fetch/target-no-path.bst b/tests/sources/tar/fetch/target-no-path.bst
new file mode 100644
index 0000000..56e4e05
--- /dev/null
+++ b/tests/sources/tar/fetch/target-no-path.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: ""
[buildstream] 04/04: _downloadablefilesource: Tidy up
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch becky/locally_downloaded_files
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 60ec787fc5773e27accbace5a4aab80162b0f63d
Author: Rebecca Grayson <be...@hotmail.co.uk>
AuthorDate: Fri Jun 28 11:35:32 2019 +0100
_downloadablefilesource: Tidy up
---
.../plugins/sources/_downloadablefilesource.py | 123 ++++++++++-----------
1 file changed, 58 insertions(+), 65 deletions(-)
diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py
index 127047c..f1f2549 100644
--- a/src/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/src/buildstream/plugins/sources/_downloadablefilesource.py
@@ -92,80 +92,76 @@ class DownloadableFileSource(Source):
raise SourceError("You cannot specify both 'path' ({}) and 'url' ({})"
.format(self.path_provenance, self.url_provenance))
- # If using a url, use appropriate checks
- if self.original_url is not None:
- self.url = self.translate_url(self.original_url)
-
- # If using path, do checks
- if self.original_path is not None:
+ # If using path, use appropriate checks
+ if self.original_url is None:
self.path = self.node_get_project_path(node, 'path')
self.fullpath = os.path.join(self.get_project_directory(), self.path)
self.sha = unique_key(self.fullpath)
- else:
- self.sha = self.ref
+
+ # If using url, do checks
+ if self.original_path is None:
+ self.url = self.translate_url(self.original_url)
def preflight(self):
return
def get_unique_key(self):
- self.__unique_key = None
- if self.original_url is not None:
- return [self.original_url, self.ref]
- else:
- return [(os.path.basename(self.original_path)), self.sha]
+ if self.original_url is None:
+ return [os.path.basename(self.original_path), self.sha]
+
+ return [self.original_url, self.ref]
def get_consistency(self):
- if self.original_url is not None:
- if self.ref is None:
- return Consistency.INCONSISTENT
+ if self.original_url is None:
+ return Consistency.CACHED
- if os.path.isfile(self._get_mirror_file()):
- return Consistency.CACHED
+ if self.ref is None:
+ return Consistency.INCONSISTENT
- else:
- return Consistency.RESOLVED
+ if os.path.isfile(self._get_mirror_file()):
+ return Consistency.CACHED
else:
- return Consistency.CACHED
+ return Consistency.RESOLVED
def load_ref(self, node):
- if self.original_url is not None:
- self.ref = self.node_get_member(node, str, 'ref', None)
- self._warn_deprecated_etag(node)
- else:
- pass
+ if self.original_url is None:
+ return
+
+ self.ref = self.node_get_member(node, str, 'ref', None)
+ self._warn_deprecated_etag(node)
def get_ref(self):
- if self.original_url is not None:
- return self.ref
- else:
+ if self.original_url is None:
return None
+ return self.ref
+
def set_ref(self, ref, node):
- if self.original_url is not None:
- node['ref'] = self.ref = self.sha = ref
- else:
- pass
+ if self.original_url is None:
+ return
+
+ node['ref'] = self.ref = ref
def track(self):
# there is no 'track' field in the source to determine what/whether
# or not to update refs, because tracking a ref is always a conscious
# decision by the user.
- if self.original_url is not None:
- with self.timed_activity("Tracking {}".format(self.url),
- silent_nested=True):
- new_ref = self._ensure_mirror()
-
- if self.ref and self.ref != new_ref:
- detail = "When tracking, new ref differs from current ref:\n" \
- + " Tracked URL: {}\n".format(self.url) \
- + " Current ref: {}\n".format(self.ref) \
- + " New ref: {}\n".format(new_ref)
- self.warn("Potential man-in-the-middle attack!", detail=detail)
-
- return new_ref
- else:
- pass
+ if self.original_url is None:
+ return
+
+ with self.timed_activity("Tracking {}".format(self.url),
+ silent_nested=True):
+ new_ref = self._ensure_mirror()
+
+ if self.ref != new_ref:
+ detail = "When tracking, new ref differs from current ref:\n" \
+ + " Tracked URL: {}\n".format(self.url) \
+ + " Current ref: {}\n".format(self.ref) \
+ + " New ref: {}\n".format(new_ref)
+ self.warn("Potential man-in-the-middle attack!", detail=detail)
+
+ return new_ref
def fetch(self):
@@ -173,19 +169,19 @@ class DownloadableFileSource(Source):
# file to be already cached because Source.fetch() will
# not be called if the source is already Consistency.CACHED.
#
- if self.original_url is not None:
- if os.path.isfile(self._get_mirror_file()):
- return # pragma: nocover
-
- # Download the file, raise hell if the sha256sums don't match,
- # and mirror the file otherwise.
- with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
- sha256 = self._ensure_mirror()
- if sha256 != self.ref:
- raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
- .format(self.url, sha256, self.ref))
- else:
- pass
+ if self.original_url is None:
+ return
+
+ if os.path.isfile(self._get_mirror_file()):
+ return # pragma: nocover
+
+ # Download the file, raise hell if the sha256sums don't match,
+ # and mirror the file otherwise.
+ with self.timed_activity("Fetching {}".format(self.url), silent_nested=True):
+ sha256 = self._ensure_mirror()
+ if sha256 != self.ref:
+ raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!"
+ .format(self.url, sha256, self.ref))
def _get_local_path(self):
return self.path
@@ -269,10 +265,7 @@ class DownloadableFileSource(Source):
.format(self, self.url, e), temporary=True) from e
def _get_mirror_dir(self):
- if self.original_url is not None:
- directory_name = utils.url_directory_name(self.original_url)
- else:
- directory_name = self.original_path
+ directory_name = utils.url_directory_name(self.original_url)
return os.path.join(self.get_mirror_directory(),
directory_name)
[buildstream] 02/04: Adding test for local source download
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch becky/locally_downloaded_files
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e1a91ee79b48771a9109d21bac7d9e082a63e4d7
Author: Rebecca Grayson <be...@hotmail.co.uk>
AuthorDate: Tue Jun 25 15:21:32 2019 +0100
Adding test for local source download
test added to tar.py, local-source contains project.
---
tests/sources/tar.py | 163 ++++++++++++++++++++++-
tests/sources/tar/fetch-local/content/a/b/d | 1 +
tests/sources/tar/fetch-local/content/a/c | 1 +
tests/sources/tar/fetch-local/target-lz.bst | 5 +
tests/sources/tar/fetch-local/target.bst | 5 +
tests/sources/tar/fetch/file.txt | 1 +
tests/sources/tar/fetch/target-file.bst | 6 +
tests/sources/tar/fetch/target-path.bst | 5 +
tests/sources/tar/invalid-rel-path/content/a/b/d | 1 +
tests/sources/tar/invalid-rel-path/content/a/c | 1 +
tests/sources/tar/invalid-rel-path/file.txt | 1 +
tests/sources/tar/invalid-rel-path/target.bst | 6 +
tests/sources/tar/local-source/content/a/b/d | 1 +
tests/sources/tar/local-source/content/a/c | 1 +
tests/sources/tar/local-source/target.bst | 6 +
tests/sources/tar/no-url-and-path/content/a/b/d | 1 +
tests/sources/tar/no-url-and-path/content/a/c | 1 +
tests/sources/tar/no-url-and-path/target.bst | 6 +
tests/sources/tar/url-and-path/content/a/b/d | 1 +
tests/sources/tar/url-and-path/content/a/c | 1 +
tests/sources/tar/url-and-path/target.bst | 7 +
21 files changed, 219 insertions(+), 2 deletions(-)
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index a6c1a4d..68791cb 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -10,7 +10,7 @@ import urllib.parse
import pytest
-from buildstream._exceptions import ErrorDomain
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream import _yaml
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing._utils.site import HAVE_LZIP
@@ -64,7 +64,6 @@ def generate_project_file_server(base_url, project_dir):
}
}, project_file)
-
# Test that without ref, consistency is set appropriately.
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
def test_no_ref(cli, tmpdir, datafiles):
@@ -87,6 +86,65 @@ def test_fetch_bad_url(cli, tmpdir, datafiles):
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
+# Test that when I fetch a nonexistent path, errors are handled gracefully.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_bad_path(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target-path.bst'
+ ])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
+
+# Test that when I fetch a non regular path or directory, errors are handled.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_non_regular_file_or_directory(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+ localfile = os.path.join(project, 'file.txt')
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target-file.bst'
+ ])
+ if os.path.isdir(localfile) and not os.path.islink(localfile):
+ result.assert_success()
+ elif os.path.isfile(localfile) and not os.path.islink(localfile):
+ result.assert_success()
+ else:
+ result.assert_main_error(ErrorDomain.LOAD,
+ LoadErrorReason.PROJ_PATH_INVALID_KIND)
+
+# Test that when I fetch an invalid absolute path, errors are handled.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_invalid_absolute_path(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ with open(os.path.join(project, "target-file.bst"), 'r') as f:
+ old_yaml = f.read()
+
+ new_yaml = old_yaml.replace("file.txt", os.path.join(project, "file.txt"))
+ assert old_yaml != new_yaml
+
+ with open(os.path.join(project, "target-file.bst"), 'w') as f:
+ f.write(new_yaml)
+
+ result = cli.run(project=project, args=['show', 'target-file.bst'])
+ result.assert_main_error(ErrorDomain.LOAD,
+ LoadErrorReason.PROJ_PATH_INVALID)
+
+# Test that when I fetch an invalid relative path, it fails.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'invalid-rel-path'))
+def test_fetch_invalid_relative_path(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ result = cli.run(project=project, args=['show', 'target.bst'])
+ result.assert_main_error(ErrorDomain.LOAD,
+ LoadErrorReason.PROJ_PATH_INVALID)
# Test that when I fetch with an invalid ref, it fails.
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
@@ -105,6 +163,46 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
+# Test that when neither url or path are provided, it fails
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-url-and-path'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_no_url_and_path(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ # Create a local tar
+ src_tar = os.path.join(str(project), "file", "a.tar.lz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target.bst'
+ ])
+
+ failed_message = "Please specify either a 'path' or a 'url'."
+ result.assert_main_error(ErrorDomain.SOURCE, None, fail_message=failed_message)
+
+# Test that when both url and path are provided, it fails
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'url-and-path'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_url_and_path(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ # Create a local tar
+ src_tar = os.path.join(str(project), "file", "a.tar.lz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target.bst'
+ ])
+ failed_message = "You cannot specify both 'path' (target.bst[line 5 column 8])\
+ and 'url' (target.bst[line 6 column 7])"
+ result.assert_main_error(ErrorDomain.SOURCE, None, fail_message=failed_message)
+
# Test that when tracking with a ref set, there is a warning
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
@@ -210,6 +308,37 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir):
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
+# Test that a staged checkout matches what was tarred up, with an explicit basedir
+# for a local source
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'local-source'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_download_local_source(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+ # Create a local tar
+ print(project)
+ src_tar = os.path.join(str(project), "file", "a.tar.gz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Track, fetch, build, checkout
+ result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['build', 'target.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result.assert_success()
+
+ # Check that the content of the first directory is checked out (base-dir: '*')
+ original_dir = os.path.join(str(project), "content", "a")
+ original_contents = list_dir_contents(original_dir)
+ checkout_contents = list_dir_contents(checkoutdir)
+ assert checkout_contents == original_contents
+
# Test that we succeed to extract tarballs with hardlinks when stripping the
# leading paths
@@ -276,6 +405,36 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
assert checkout_contents == original_contents
+@pytest.mark.skipif(not HAVE_LZIP, reason='lzip is not available')
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch-local'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_stage_default_basedir_lzip_local(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+ # Create a local tar
+ src_tar = os.path.join(str(project), "file", "a.tar.lz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Track, fetch, build, checkout
+ result = cli.run(project=project, args=['source', 'track', 'target-lz.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['source', 'fetch', 'target-lz.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['build', 'target-lz.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['artifact', 'checkout', 'target-lz.bst', '--directory', checkoutdir])
+ result.assert_success()
+
+ # Check that the content of the first directory is checked out (base-dir: '*')
+ original_dir = os.path.join(str(project), "content", "a")
+ original_contents = list_dir_contents(original_dir)
+ checkout_contents = list_dir_contents(checkoutdir)
+ assert checkout_contents == original_contents
+
+
# Test that a tarball that contains a read only dir works
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'read-only'))
def test_read_only_dir(cli, tmpdir, datafiles):
diff --git a/tests/sources/tar/fetch-local/content/a/b/d b/tests/sources/tar/fetch-local/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/fetch-local/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/fetch-local/content/a/c b/tests/sources/tar/fetch-local/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/fetch-local/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/fetch-local/target-lz.bst b/tests/sources/tar/fetch-local/target-lz.bst
new file mode 100644
index 0000000..ecd1f9e
--- /dev/null
+++ b/tests/sources/tar/fetch-local/target-lz.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.lz
diff --git a/tests/sources/tar/fetch-local/target.bst b/tests/sources/tar/fetch-local/target.bst
new file mode 100644
index 0000000..cc5f15c
--- /dev/null
+++ b/tests/sources/tar/fetch-local/target.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
diff --git a/tests/sources/tar/fetch/file.txt b/tests/sources/tar/fetch/file.txt
new file mode 100644
index 0000000..5adcd3d
--- /dev/null
+++ b/tests/sources/tar/fetch/file.txt
@@ -0,0 +1 @@
+This is a text file.
diff --git a/tests/sources/tar/fetch/target-file.bst b/tests/sources/tar/fetch/target-file.bst
new file mode 100644
index 0000000..63430a5
--- /dev/null
+++ b/tests/sources/tar/fetch/target-file.bst
@@ -0,0 +1,6 @@
+kind: import
+description: This is the pony
+sources:
+- kind: local
+ path: file.txt
+
diff --git a/tests/sources/tar/fetch/target-path.bst b/tests/sources/tar/fetch/target-path.bst
new file mode 100644
index 0000000..cc5f15c
--- /dev/null
+++ b/tests/sources/tar/fetch/target-path.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
diff --git a/tests/sources/tar/invalid-rel-path/content/a/b/d b/tests/sources/tar/invalid-rel-path/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/invalid-rel-path/content/a/c b/tests/sources/tar/invalid-rel-path/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/invalid-rel-path/file.txt b/tests/sources/tar/invalid-rel-path/file.txt
new file mode 100644
index 0000000..5adcd3d
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/file.txt
@@ -0,0 +1 @@
+This is a text file.
diff --git a/tests/sources/tar/invalid-rel-path/target.bst b/tests/sources/tar/invalid-rel-path/target.bst
new file mode 100644
index 0000000..6145831
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: ../invalidrelpath/file/a.tar.gz
+
diff --git a/tests/sources/tar/local-source/content/a/b/d b/tests/sources/tar/local-source/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/local-source/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/local-source/content/a/c b/tests/sources/tar/local-source/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/local-source/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/local-source/target.bst b/tests/sources/tar/local-source/target.bst
new file mode 100644
index 0000000..8a300d4
--- /dev/null
+++ b/tests/sources/tar/local-source/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
+
diff --git a/tests/sources/tar/no-url-and-path/content/a/b/d b/tests/sources/tar/no-url-and-path/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/no-url-and-path/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/no-url-and-path/content/a/c b/tests/sources/tar/no-url-and-path/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/no-url-and-path/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/no-url-and-path/target.bst b/tests/sources/tar/no-url-and-path/target.bst
new file mode 100644
index 0000000..0bf4f93
--- /dev/null
+++ b/tests/sources/tar/no-url-and-path/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+
+ ref: foo
diff --git a/tests/sources/tar/url-and-path/content/a/b/d b/tests/sources/tar/url-and-path/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/url-and-path/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/url-and-path/content/a/c b/tests/sources/tar/url-and-path/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/url-and-path/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/url-and-path/target.bst b/tests/sources/tar/url-and-path/target.bst
new file mode 100644
index 0000000..c62556e
--- /dev/null
+++ b/tests/sources/tar/url-and-path/target.bst
@@ -0,0 +1,7 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
+ url: tmpdir:/a.tar.gz
+ ref: foo