You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by gi...@apache.org on 2020/12/29 13:16:31 UTC
[buildstream] 02/04: Adding test for local source download
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch becky/locally_downloaded_files
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e1a91ee79b48771a9109d21bac7d9e082a63e4d7
Author: Rebecca Grayson <be...@hotmail.co.uk>
AuthorDate: Tue Jun 25 15:21:32 2019 +0100
Adding test for local source download
test added to tar.py, local-source contains project.
---
tests/sources/tar.py | 163 ++++++++++++++++++++++-
tests/sources/tar/fetch-local/content/a/b/d | 1 +
tests/sources/tar/fetch-local/content/a/c | 1 +
tests/sources/tar/fetch-local/target-lz.bst | 5 +
tests/sources/tar/fetch-local/target.bst | 5 +
tests/sources/tar/fetch/file.txt | 1 +
tests/sources/tar/fetch/target-file.bst | 6 +
tests/sources/tar/fetch/target-path.bst | 5 +
tests/sources/tar/invalid-rel-path/content/a/b/d | 1 +
tests/sources/tar/invalid-rel-path/content/a/c | 1 +
tests/sources/tar/invalid-rel-path/file.txt | 1 +
tests/sources/tar/invalid-rel-path/target.bst | 6 +
tests/sources/tar/local-source/content/a/b/d | 1 +
tests/sources/tar/local-source/content/a/c | 1 +
tests/sources/tar/local-source/target.bst | 6 +
tests/sources/tar/no-url-and-path/content/a/b/d | 1 +
tests/sources/tar/no-url-and-path/content/a/c | 1 +
tests/sources/tar/no-url-and-path/target.bst | 6 +
tests/sources/tar/url-and-path/content/a/b/d | 1 +
tests/sources/tar/url-and-path/content/a/c | 1 +
tests/sources/tar/url-and-path/target.bst | 7 +
21 files changed, 219 insertions(+), 2 deletions(-)
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index a6c1a4d..68791cb 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -10,7 +10,7 @@ import urllib.parse
import pytest
-from buildstream._exceptions import ErrorDomain
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
from buildstream import _yaml
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing._utils.site import HAVE_LZIP
@@ -64,7 +64,6 @@ def generate_project_file_server(base_url, project_dir):
}
}, project_file)
-
# Test that without ref, consistency is set appropriately.
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
def test_no_ref(cli, tmpdir, datafiles):
@@ -87,6 +86,65 @@ def test_fetch_bad_url(cli, tmpdir, datafiles):
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
+# Test that when I fetch a nonexistent path, errors are handled gracefully.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_bad_path(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target-path.bst'
+ ])
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
+
+# Test that when I fetch a non regular path or directory, errors are handled.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_non_regular_file_or_directory(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+ localfile = os.path.join(project, 'file.txt')
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target-file.bst'
+ ])
+ if os.path.isdir(localfile) and not os.path.islink(localfile):
+ result.assert_success()
+ elif os.path.isfile(localfile) and not os.path.islink(localfile):
+ result.assert_success()
+ else:
+ result.assert_main_error(ErrorDomain.LOAD,
+ LoadErrorReason.PROJ_PATH_INVALID_KIND)
+
+# Test that when I fetch an invalid absolute path, errors are handled.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
+def test_fetch_invalid_absolute_path(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ with open(os.path.join(project, "target-file.bst"), 'r') as f:
+ old_yaml = f.read()
+
+ new_yaml = old_yaml.replace("file.txt", os.path.join(project, "file.txt"))
+ assert old_yaml != new_yaml
+
+ with open(os.path.join(project, "target-file.bst"), 'w') as f:
+ f.write(new_yaml)
+
+ result = cli.run(project=project, args=['show', 'target-file.bst'])
+ result.assert_main_error(ErrorDomain.LOAD,
+ LoadErrorReason.PROJ_PATH_INVALID)
+
+# Test that when I fetch an invalid relative path, it fails.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'invalid-rel-path'))
+def test_fetch_invalid_relative_path(cli, tmpdir, datafiles):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ result = cli.run(project=project, args=['show', 'target.bst'])
+ result.assert_main_error(ErrorDomain.LOAD,
+ LoadErrorReason.PROJ_PATH_INVALID)
# Test that when I fetch with an invalid ref, it fails.
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
@@ -105,6 +163,46 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
+# Test that when neither url or path are provided, it fails
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-url-and-path'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_no_url_and_path(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ # Create a local tar
+ src_tar = os.path.join(str(project), "file", "a.tar.lz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target.bst'
+ ])
+
+ failed_message = "Please specify either a 'path' or a 'url'."
+ result.assert_main_error(ErrorDomain.SOURCE, None, fail_message=failed_message)
+
+# Test that when both url and path are provided, it fails
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'url-and-path'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_url_and_path(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+
+ # Create a local tar
+ src_tar = os.path.join(str(project), "file", "a.tar.lz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Try to fetch it
+ result = cli.run(project=project, args=[
+ 'source', 'fetch', 'target.bst'
+ ])
+ failed_message = "You cannot specify both 'path' (target.bst[line 5 column 8])\
+ and 'url' (target.bst[line 6 column 7])"
+ result.assert_main_error(ErrorDomain.SOURCE, None, fail_message=failed_message)
+
# Test that when tracking with a ref set, there is a warning
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
@@ -210,6 +308,37 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir):
checkout_contents = list_dir_contents(checkoutdir)
assert checkout_contents == original_contents
+# Test that a staged checkout matches what was tarred up, with an explicit basedir
+# for a local source
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'local-source'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_download_local_source(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+ # Create a local tar
+ print(project)
+ src_tar = os.path.join(str(project), "file", "a.tar.gz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Track, fetch, build, checkout
+ result = cli.run(project=project, args=['source', 'track', 'target.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['source', 'fetch', 'target.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['build', 'target.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir])
+ result.assert_success()
+
+ # Check that the content of the first directory is checked out (base-dir: '*')
+ original_dir = os.path.join(str(project), "content", "a")
+ original_contents = list_dir_contents(original_dir)
+ checkout_contents = list_dir_contents(checkoutdir)
+ assert checkout_contents == original_contents
+
# Test that we succeed to extract tarballs with hardlinks when stripping the
# leading paths
@@ -276,6 +405,36 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
assert checkout_contents == original_contents
+@pytest.mark.skipif(not HAVE_LZIP, reason='lzip is not available')
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch-local'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_stage_default_basedir_lzip_local(cli, tmpdir, datafiles, srcdir):
+ project = str(datafiles)
+ generate_project(project, tmpdir)
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+ # Create a local tar
+ src_tar = os.path.join(str(project), "file", "a.tar.lz")
+ os.mkdir(os.path.join(str(project), "file"))
+ _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+ # Track, fetch, build, checkout
+ result = cli.run(project=project, args=['source', 'track', 'target-lz.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['source', 'fetch', 'target-lz.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['build', 'target-lz.bst'])
+ result.assert_success()
+ result = cli.run(project=project, args=['artifact', 'checkout', 'target-lz.bst', '--directory', checkoutdir])
+ result.assert_success()
+
+ # Check that the content of the first directory is checked out (base-dir: '*')
+ original_dir = os.path.join(str(project), "content", "a")
+ original_contents = list_dir_contents(original_dir)
+ checkout_contents = list_dir_contents(checkoutdir)
+ assert checkout_contents == original_contents
+
+
# Test that a tarball that contains a read only dir works
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'read-only'))
def test_read_only_dir(cli, tmpdir, datafiles):
diff --git a/tests/sources/tar/fetch-local/content/a/b/d b/tests/sources/tar/fetch-local/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/fetch-local/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/fetch-local/content/a/c b/tests/sources/tar/fetch-local/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/fetch-local/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/fetch-local/target-lz.bst b/tests/sources/tar/fetch-local/target-lz.bst
new file mode 100644
index 0000000..ecd1f9e
--- /dev/null
+++ b/tests/sources/tar/fetch-local/target-lz.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.lz
diff --git a/tests/sources/tar/fetch-local/target.bst b/tests/sources/tar/fetch-local/target.bst
new file mode 100644
index 0000000..cc5f15c
--- /dev/null
+++ b/tests/sources/tar/fetch-local/target.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
diff --git a/tests/sources/tar/fetch/file.txt b/tests/sources/tar/fetch/file.txt
new file mode 100644
index 0000000..5adcd3d
--- /dev/null
+++ b/tests/sources/tar/fetch/file.txt
@@ -0,0 +1 @@
+This is a text file.
diff --git a/tests/sources/tar/fetch/target-file.bst b/tests/sources/tar/fetch/target-file.bst
new file mode 100644
index 0000000..63430a5
--- /dev/null
+++ b/tests/sources/tar/fetch/target-file.bst
@@ -0,0 +1,6 @@
+kind: import
+description: This is the pony
+sources:
+- kind: local
+ path: file.txt
+
diff --git a/tests/sources/tar/fetch/target-path.bst b/tests/sources/tar/fetch/target-path.bst
new file mode 100644
index 0000000..cc5f15c
--- /dev/null
+++ b/tests/sources/tar/fetch/target-path.bst
@@ -0,0 +1,5 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
diff --git a/tests/sources/tar/invalid-rel-path/content/a/b/d b/tests/sources/tar/invalid-rel-path/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/invalid-rel-path/content/a/c b/tests/sources/tar/invalid-rel-path/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/invalid-rel-path/file.txt b/tests/sources/tar/invalid-rel-path/file.txt
new file mode 100644
index 0000000..5adcd3d
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/file.txt
@@ -0,0 +1 @@
+This is a text file.
diff --git a/tests/sources/tar/invalid-rel-path/target.bst b/tests/sources/tar/invalid-rel-path/target.bst
new file mode 100644
index 0000000..6145831
--- /dev/null
+++ b/tests/sources/tar/invalid-rel-path/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: ../invalidrelpath/file/a.tar.gz
+
diff --git a/tests/sources/tar/local-source/content/a/b/d b/tests/sources/tar/local-source/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/local-source/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/local-source/content/a/c b/tests/sources/tar/local-source/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/local-source/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/local-source/target.bst b/tests/sources/tar/local-source/target.bst
new file mode 100644
index 0000000..8a300d4
--- /dev/null
+++ b/tests/sources/tar/local-source/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
+
diff --git a/tests/sources/tar/no-url-and-path/content/a/b/d b/tests/sources/tar/no-url-and-path/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/no-url-and-path/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/no-url-and-path/content/a/c b/tests/sources/tar/no-url-and-path/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/no-url-and-path/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/no-url-and-path/target.bst b/tests/sources/tar/no-url-and-path/target.bst
new file mode 100644
index 0000000..0bf4f93
--- /dev/null
+++ b/tests/sources/tar/no-url-and-path/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+
+ ref: foo
diff --git a/tests/sources/tar/url-and-path/content/a/b/d b/tests/sources/tar/url-and-path/content/a/b/d
new file mode 100644
index 0000000..4bcfe98
--- /dev/null
+++ b/tests/sources/tar/url-and-path/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/url-and-path/content/a/c b/tests/sources/tar/url-and-path/content/a/c
new file mode 100644
index 0000000..f2ad6c7
--- /dev/null
+++ b/tests/sources/tar/url-and-path/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/url-and-path/target.bst b/tests/sources/tar/url-and-path/target.bst
new file mode 100644
index 0000000..c62556e
--- /dev/null
+++ b/tests/sources/tar/url-and-path/target.bst
@@ -0,0 +1,7 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+ path: file/a.tar.gz
+ url: tmpdir:/a.tar.gz
+ ref: foo