You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by no...@apache.org on 2020/12/29 12:35:32 UTC
[buildstream] branch bschubert/standardize-source-tests created
(now c958d4c)
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a change to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git.
at c958d4c tests: Stop using the implicit source tests and explicitely run them
This branch includes the following new commits:
new 301c164 workspace.py: Simplify test by only allowing opening a single workspace
new 14a136e workspace.py: Remove arguments that are unused during the test
new 22b6591 workspace.py: Remove some abstractions and write the whole test in place
new a5e6540 workspace.py: Simplify invocation of the cli command
new e8280a9 workspace.py: Don't use the same directory for everything
new 09fca56 testing: Fix type annotation error that mypy didn't catch
new 22d6093 tests: Stop using the 'create_repo' utility, create them directly
new ad99234 sourcetests: Create a new SourceTests class
new 984b44e build_checkout.py: Adapt to use the standard source tests
new 43b4da3 mirror.py: Adapt to use the standard source tests
new 118b4b8 source_determinism.py: Adapt to use the standard source tests
new e24a133 track_cross_junction.py: Adapt to use the standard source tests
new a2d1e16 track.py: Adapt to use the standard source tests
new 567a9be workspace.py: Adapt to use the standard source tests
new c958d4c tests: Stop using the implicit source tests and explicitely run them
The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[buildstream] 13/15: track.py: Adapt to use the standard source
tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a2d1e16921df4e1382f601aa4fac8644308b7c9a
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Thu Oct 8 20:58:50 2020 +0000
track.py: Adapt to use the standard source tests
---
src/buildstream/testing/_sourcetests/__init__.py | 8 +-
src/buildstream/testing/_sourcetests/track.py | 551 +++++++++++------------
2 files changed, 282 insertions(+), 277 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/__init__.py b/src/buildstream/testing/_sourcetests/__init__.py
index 2a1f26a..e0b2e41 100644
--- a/src/buildstream/testing/_sourcetests/__init__.py
+++ b/src/buildstream/testing/_sourcetests/__init__.py
@@ -24,11 +24,17 @@ from .fetch import FetchSourceTests
from .mirror import MirrorSourceTests
from .source_determinism import SourceDeterminismTests
from .track_cross_junction import TrackCrossJunctionTests
+from .track import TrackSourceTests
__all__ = ["SourceTests"]
class SourceTests(
- BuildCheckoutSourceTests, FetchSourceTests, MirrorSourceTests, SourceDeterminismTests, TrackCrossJunctionTests
+ BuildCheckoutSourceTests,
+ FetchSourceTests,
+ MirrorSourceTests,
+ SourceDeterminismTests,
+ TrackCrossJunctionTests,
+ TrackSourceTests,
):
"""Definition of standardized tests that each source should pass."""
diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py
index 38ef217..c804518 100644
--- a/src/buildstream/testing/_sourcetests/track.py
+++ b/src/buildstream/testing/_sourcetests/track.py
@@ -25,10 +25,9 @@ import pytest
from buildstream import _yaml
from buildstream.exceptions import ErrorDomain
from .._utils import generate_junction
-from .. import create_repo
from .. import cli # pylint: disable=unused-import
from .utils import update_project_configuration
-from .utils import kind # pylint: disable=unused-import
+from .base import BaseSourceTests
# Project directory
@@ -44,304 +43,226 @@ def generate_element(repo, element_path, dep_name=None):
_yaml.roundtrip_dump(element, element_path)
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
-def test_track(cli, tmpdir, datafiles, ref_storage, kind):
- project = str(datafiles)
- dev_files_path = os.path.join(project, "files", "dev-files")
- element_path = os.path.join(project, "elements")
- element_name = "track-test-{}.bst".format(kind)
+class TrackSourceTests(BaseSourceTests):
+ @pytest.mark.datafiles(DATA_DIR)
+ @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
+ def test_track(self, cli, tmpdir, datafiles, ref_storage):
+ project = str(datafiles)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "track-test-{}.bst".format(self.KIND)
- update_project_configuration(project, {"ref-storage": ref_storage})
+ update_project_configuration(project, {"ref-storage": ref_storage})
- # Create our repo object of the given source type with
- # the dev files, and then collect the initial ref.
- #
- repo = create_repo(kind, str(tmpdir))
- repo.create(dev_files_path)
-
- # Generate the element
- generate_element(repo, os.path.join(element_path, element_name))
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = self.REPO(str(tmpdir))
+ repo.create(dev_files_path)
- # Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == "no reference"
+ # Generate the element
+ generate_element(repo, os.path.join(element_path, element_name))
- # Now first try to track it
- result = cli.run(project=project, args=["source", "track", element_name])
- result.assert_success()
+ # Assert that a fetch is needed
+ assert cli.get_element_state(project, element_name) == "no reference"
- # And now fetch it: The Source has probably already cached the
- # latest ref locally, but it is not required to have cached
- # the associated content of the latest ref at track time, that
- # is the job of fetch.
- result = cli.run(project=project, args=["source", "fetch", element_name])
- result.assert_success()
+ # Now first try to track it
+ result = cli.run(project=project, args=["source", "track", element_name])
+ result.assert_success()
- # Assert that we are now buildable because the source is
- # now cached.
- assert cli.get_element_state(project, element_name) == "buildable"
+ # And now fetch it: The Source has probably already cached the
+ # latest ref locally, but it is not required to have cached
+ # the associated content of the latest ref at track time, that
+ # is the job of fetch.
+ result = cli.run(project=project, args=["source", "fetch", element_name])
+ result.assert_success()
- # Assert there was a project.refs created, depending on the configuration
- if ref_storage == "project.refs":
- assert os.path.exists(os.path.join(project, "project.refs"))
- else:
- assert not os.path.exists(os.path.join(project, "project.refs"))
+ # Assert that we are now buildable because the source is
+ # now cached.
+ assert cli.get_element_state(project, element_name) == "buildable"
+ # Assert there was a project.refs created, depending on the configuration
+ if ref_storage == "project.refs":
+ assert os.path.exists(os.path.join(project, "project.refs"))
+ else:
+ assert not os.path.exists(os.path.join(project, "project.refs"))
-# NOTE:
-#
-# This test checks that recursive tracking works by observing
-# element states after running a recursive tracking operation.
-#
-# However, this test is ALSO valuable as it stresses the source
-# plugins in a situation where many source plugins are operating
-# at once on the same backing repository.
-#
-# Do not change this test to use a separate 'Repo' per element
-# as that would defeat the purpose of the stress test, otherwise
-# please refactor that aspect into another test.
-#
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("amount", [1, 10])
-def test_track_recurse(cli, tmpdir, datafiles, kind, amount):
- project = str(datafiles)
- dev_files_path = os.path.join(project, "files", "dev-files")
- element_path = os.path.join(project, "elements")
-
- # Try to actually launch as many fetch jobs as possible at the same time
+ # NOTE:
#
- # This stresses the Source plugins and helps to ensure that
- # they handle concurrent access to the store correctly.
- cli.configure({"scheduler": {"fetchers": amount,}})
-
- # Create our repo object of the given source type with
- # the dev files, and then collect the initial ref.
+ # This test checks that recursive tracking works by observing
+ # element states after running a recursive tracking operation.
#
- repo = create_repo(kind, str(tmpdir))
- repo.create(dev_files_path)
-
- # Write out our test targets
- element_names = []
- last_element_name = None
- for i in range(amount + 1):
- element_name = "track-test-{}-{}.bst".format(kind, i + 1)
- filename = os.path.join(element_path, element_name)
-
- element_names.append(element_name)
-
- generate_element(repo, filename, dep_name=last_element_name)
- last_element_name = element_name
-
- # Assert that a fetch is needed
- states = cli.get_element_states(project, [last_element_name])
- for element_name in element_names:
- assert states[element_name] == "no reference"
-
- # Now first try to track it
- result = cli.run(project=project, args=["source", "track", "--deps", "all", last_element_name])
- result.assert_success()
-
- # And now fetch it: The Source has probably already cached the
- # latest ref locally, but it is not required to have cached
- # the associated content of the latest ref at track time, that
- # is the job of fetch.
- result = cli.run(project=project, args=["source", "fetch", "--deps", "all", last_element_name])
- result.assert_success()
-
- # Assert that the base is buildable and the rest are waiting
- states = cli.get_element_states(project, [last_element_name])
- for element_name in element_names:
- if element_name == element_names[0]:
- assert states[element_name] == "buildable"
- else:
- assert states[element_name] == "waiting"
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_track_recurse_except(cli, tmpdir, datafiles, kind):
- project = str(datafiles)
- dev_files_path = os.path.join(project, "files", "dev-files")
- element_path = os.path.join(project, "elements")
- element_dep_name = "track-test-dep-{}.bst".format(kind)
- element_target_name = "track-test-target-{}.bst".format(kind)
-
- # Create our repo object of the given source type with
- # the dev files, and then collect the initial ref.
+ # However, this test is ALSO valuable as it stresses the source
+ # plugins in a situation where many source plugins are operating
+ # at once on the same backing repository.
#
- repo = create_repo(kind, str(tmpdir))
- repo.create(dev_files_path)
-
- # Write out our test targets
- generate_element(repo, os.path.join(element_path, element_dep_name))
- generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name)
-
- # Assert that a fetch is needed
- states = cli.get_element_states(project, [element_target_name])
- assert states[element_dep_name] == "no reference"
- assert states[element_target_name] == "no reference"
-
- # Now first try to track it
- result = cli.run(
- project=project, args=["source", "track", "--deps", "all", "--except", element_dep_name, element_target_name]
- )
- result.assert_success()
-
- # And now fetch it: The Source has probably already cached the
- # latest ref locally, but it is not required to have cached
- # the associated content of the latest ref at track time, that
- # is the job of fetch.
- result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name])
- result.assert_success()
-
- # Assert that the dependency is buildable and the target is waiting
- states = cli.get_element_states(project, [element_target_name])
- assert states[element_dep_name] == "no reference"
- assert states[element_target_name] == "waiting"
+ # Do not change this test to use a separate 'Repo' per element
+ # as that would defeat the purpose of the stress test, otherwise
+ # please refactor that aspect into another test.
+ #
+ @pytest.mark.datafiles(DATA_DIR)
+ @pytest.mark.parametrize("amount", [1, 10])
+ def test_track_recurse(self, cli, tmpdir, datafiles, amount):
+ project = str(datafiles)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+
+ # Try to actually launch as many fetch jobs as possible at the same time
+ #
+ # This stresses the Source plugins and helps to ensure that
+ # they handle concurrent access to the store correctly.
+ cli.configure({"scheduler": {"fetchers": amount,}})
+
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = self.REPO(str(tmpdir))
+ repo.create(dev_files_path)
+
+ # Write out our test targets
+ element_names = []
+ last_element_name = None
+ for i in range(amount + 1):
+ element_name = "track-test-{}-{}.bst".format(self.KIND, i + 1)
+ filename = os.path.join(element_path, element_name)
+
+ element_names.append(element_name)
+
+ generate_element(repo, filename, dep_name=last_element_name)
+ last_element_name = element_name
+
+ # Assert that a fetch is needed
+ states = cli.get_element_states(project, [last_element_name])
+ for element_name in element_names:
+ assert states[element_name] == "no reference"
+
+ # Now first try to track it
+ result = cli.run(project=project, args=["source", "track", "--deps", "all", last_element_name])
+ result.assert_success()
+ # And now fetch it: The Source has probably already cached the
+ # latest ref locally, but it is not required to have cached
+ # the associated content of the latest ref at track time, that
+ # is the job of fetch.
+ result = cli.run(project=project, args=["source", "fetch", "--deps", "all", last_element_name])
+ result.assert_success()
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
-def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
- project = str(datafiles)
- subproject_path = os.path.join(project, "files", "sub-project")
- junction_path = os.path.join(project, "elements", "junction.bst")
- etc_files = os.path.join(subproject_path, "files", "etc-files")
- repo_element_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst")
+ # Assert that the base is buildable and the rest are waiting
+ states = cli.get_element_states(project, [last_element_name])
+ for element_name in element_names:
+ if element_name == element_names[0]:
+ assert states[element_name] == "buildable"
+ else:
+ assert states[element_name] == "waiting"
+
+ @pytest.mark.datafiles(DATA_DIR)
+ def test_track_recurse_except(self, cli, tmpdir, datafiles):
+ project = str(datafiles)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_dep_name = "track-test-dep-{}.bst".format(self.KIND)
+ element_target_name = "track-test-target-{}.bst".format(self.KIND)
+
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = self.REPO(str(tmpdir))
+ repo.create(dev_files_path)
+
+ # Write out our test targets
+ generate_element(repo, os.path.join(element_path, element_dep_name))
+ generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name)
+
+ # Assert that a fetch is needed
+ states = cli.get_element_states(project, [element_target_name])
+ assert states[element_dep_name] == "no reference"
+ assert states[element_target_name] == "no reference"
+
+ # Now first try to track it
+ result = cli.run(
+ project=project,
+ args=["source", "track", "--deps", "all", "--except", element_dep_name, element_target_name],
+ )
+ result.assert_success()
- update_project_configuration(project, {"ref-storage": ref_storage})
+ # And now fetch it: The Source has probably already cached the
+ # latest ref locally, but it is not required to have cached
+ # the associated content of the latest ref at track time, that
+ # is the job of fetch.
+ result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name])
+ result.assert_success()
- repo = create_repo(kind, str(tmpdir.join("element_repo")))
- repo.create(etc_files)
+ # Assert that the dependency is buildable and the target is waiting
+ states = cli.get_element_states(project, [element_target_name])
+ assert states[element_dep_name] == "no reference"
+ assert states[element_target_name] == "waiting"
- generate_element(repo, repo_element_path)
+ @pytest.mark.datafiles(DATA_DIR)
+ @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
+ def test_cross_junction(self, cli, tmpdir, datafiles, ref_storage):
+ project = str(datafiles)
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
+ etc_files = os.path.join(subproject_path, "files", "etc-files")
+ repo_element_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst")
- generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False)
+ update_project_configuration(project, {"ref-storage": ref_storage})
- # Track the junction itself first.
- result = cli.run(project=project, args=["source", "track", "junction.bst"])
- result.assert_success()
+ repo = self.REPO(str(tmpdir.join("element_repo")))
+ repo.create(etc_files)
- assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "no reference"
+ generate_element(repo, repo_element_path)
- # Track the cross junction element. -J is not given, it is implied.
- result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc-repo.bst"])
+ generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False)
- if ref_storage == "inline":
- # This is not allowed to track cross junction without project.refs.
- result.assert_main_error(ErrorDomain.PIPELINE, "untrackable-sources")
- else:
+ # Track the junction itself first.
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
result.assert_success()
- assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "buildable"
-
- assert os.path.exists(os.path.join(project, "project.refs"))
-
+ assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "no reference"
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
-def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
- project = str(datafiles)
- dev_files_path = os.path.join(project, "files", "dev-files")
- element_path = os.path.join(project, "elements")
- element_name = "track-test-{}.bst".format(kind)
+ # Track the cross junction element. -J is not given, it is implied.
+ result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc-repo.bst"])
- update_project_configuration(project, {"ref-storage": ref_storage})
+ if ref_storage == "inline":
+ # This is not allowed to track cross junction without project.refs.
+ result.assert_main_error(ErrorDomain.PIPELINE, "untrackable-sources")
+ else:
+ result.assert_success()
- # Create our repo object of the given source type with
- # the dev files, and then collect the initial ref.
- #
- repo = create_repo(kind, str(tmpdir))
- ref = repo.create(dev_files_path)
-
- # Generate the element
- element = {"kind": "import", "(@)": ["elements/sources.yml"]}
- sources = {"sources": [repo.source_config()]}
-
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- _yaml.roundtrip_dump(sources, os.path.join(element_path, "sources.yml"))
-
- # Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == "no reference"
-
- # Now first try to track it
- result = cli.run(project=project, args=["source", "track", element_name])
- result.assert_success()
-
- # And now fetch it: The Source has probably already cached the
- # latest ref locally, but it is not required to have cached
- # the associated content of the latest ref at track time, that
- # is the job of fetch.
- result = cli.run(project=project, args=["source", "fetch", element_name])
- result.assert_success()
-
- # Assert that we are now buildable because the source is
- # now cached.
- assert cli.get_element_state(project, element_name) == "buildable"
-
- # Assert there was a project.refs created, depending on the configuration
- if ref_storage == "project.refs":
- assert os.path.exists(os.path.join(project, "project.refs"))
- else:
- assert not os.path.exists(os.path.join(project, "project.refs"))
-
- new_sources = _yaml.load(os.path.join(element_path, "sources.yml"), shortname="sources.yml")
-
- # Get all of the sources
- assert "sources" in new_sources
- sources_list = new_sources.get_sequence("sources")
- assert len(sources_list) == 1
-
- # Get the first source from the sources list
- new_source = sources_list.mapping_at(0)
- assert "ref" in new_source
- assert ref == new_source.get_str("ref")
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
-def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
- project = str(datafiles)
- dev_files_path = os.path.join(project, "files", "dev-files")
- element_path = os.path.join(project, "elements")
- element_name = "track-test-{}.bst".format(kind)
- subproject_path = os.path.join(project, "files", "sub-project")
- sub_element_path = os.path.join(subproject_path, "elements")
- junction_path = os.path.join(element_path, "junction.bst")
-
- update_project_configuration(project, {"ref-storage": ref_storage})
-
- # Create our repo object of the given source type with
- # the dev files, and then collect the initial ref.
- #
- repo = create_repo(kind, str(tmpdir.join("element_repo")))
- repo.create(dev_files_path)
+ assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "buildable"
- # Generate the element
- element = {"kind": "import", "(@)": ["junction.bst:elements/sources.yml"]}
- sources = {"sources": [repo.source_config()]}
+ assert os.path.exists(os.path.join(project, "project.refs"))
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, "sources.yml"))
+ @pytest.mark.datafiles(DATA_DIR)
+ @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
+ def test_track_include(self, cli, tmpdir, datafiles, ref_storage):
+ project = str(datafiles)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "track-test-{}.bst".format(self.KIND)
- generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=True)
+ update_project_configuration(project, {"ref-storage": ref_storage})
- result = cli.run(project=project, args=["source", "track", "junction.bst"])
- result.assert_success()
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = self.REPO(str(tmpdir))
+ ref = repo.create(dev_files_path)
- # Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == "no reference"
+ # Generate the element
+ element = {"kind": "import", "(@)": ["elements/sources.yml"]}
+ sources = {"sources": [repo.source_config()]}
- # Now first try to track it
- result = cli.run(project=project, args=["source", "track", element_name])
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
+ _yaml.roundtrip_dump(sources, os.path.join(element_path, "sources.yml"))
- # Assert there was a project.refs created, depending on the configuration
- if ref_storage == "inline":
- # FIXME: We should expect an error. But only a warning is emitted
- # result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment')
+ # Assert that a fetch is needed
+ assert cli.get_element_state(project, element_name) == "no reference"
- assert "junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction" in result.stderr
- else:
- assert os.path.exists(os.path.join(project, "project.refs"))
+ # Now first try to track it
+ result = cli.run(project=project, args=["source", "track", element_name])
+ result.assert_success()
# And now fetch it: The Source has probably already cached the
# latest ref locally, but it is not required to have cached
@@ -354,18 +275,96 @@ def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
# now cached.
assert cli.get_element_state(project, element_name) == "buildable"
+ # Assert there was a project.refs created, depending on the configuration
+ if ref_storage == "project.refs":
+ assert os.path.exists(os.path.join(project, "project.refs"))
+ else:
+ assert not os.path.exists(os.path.join(project, "project.refs"))
+
+ new_sources = _yaml.load(os.path.join(element_path, "sources.yml"), shortname="sources.yml")
+
+ # Get all of the sources
+ assert "sources" in new_sources
+ sources_list = new_sources.get_sequence("sources")
+ assert len(sources_list) == 1
+
+ # Get the first source from the sources list
+ new_source = sources_list.mapping_at(0)
+ assert "ref" in new_source
+ assert ref == new_source.get_str("ref")
+
+ @pytest.mark.datafiles(DATA_DIR)
+ @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
+ def test_track_include_junction(self, cli, tmpdir, datafiles, ref_storage):
+ project = str(datafiles)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "track-test-{}.bst".format(self.KIND)
+ subproject_path = os.path.join(project, "files", "sub-project")
+ sub_element_path = os.path.join(subproject_path, "elements")
+ junction_path = os.path.join(element_path, "junction.bst")
+
+ update_project_configuration(project, {"ref-storage": ref_storage})
+
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = self.REPO(str(tmpdir.join("element_repo")))
+ repo.create(dev_files_path)
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
-def test_track_junction_included(cli, tmpdir, datafiles, ref_storage, kind):
- project = str(datafiles)
- element_path = os.path.join(project, "elements")
- subproject_path = os.path.join(project, "files", "sub-project")
- junction_path = os.path.join(element_path, "junction.bst")
+ # Generate the element
+ element = {"kind": "import", "(@)": ["junction.bst:elements/sources.yml"]}
+ sources = {"sources": [repo.source_config()]}
- update_project_configuration(project, {"ref-storage": ref_storage, "(@)": ["junction.bst:test.yml"]})
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
+ _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, "sources.yml"))
- generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False)
+ generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=True)
- result = cli.run(project=project, args=["source", "track", "junction.bst"])
- result.assert_success()
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
+ result.assert_success()
+
+ # Assert that a fetch is needed
+ assert cli.get_element_state(project, element_name) == "no reference"
+
+ # Now first try to track it
+ result = cli.run(project=project, args=["source", "track", element_name])
+
+ # Assert there was a project.refs created, depending on the configuration
+ if ref_storage == "inline":
+ # FIXME: We should expect an error. But only a warning is emitted
+ # result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment')
+
+ assert (
+ "junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction" in result.stderr
+ )
+ else:
+ assert os.path.exists(os.path.join(project, "project.refs"))
+
+ # And now fetch it: The Source has probably already cached the
+ # latest ref locally, but it is not required to have cached
+ # the associated content of the latest ref at track time, that
+ # is the job of fetch.
+ result = cli.run(project=project, args=["source", "fetch", element_name])
+ result.assert_success()
+
+ # Assert that we are now buildable because the source is
+ # now cached.
+ assert cli.get_element_state(project, element_name) == "buildable"
+
+ @pytest.mark.datafiles(DATA_DIR)
+ @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
+ def test_track_junction_included(self, cli, tmpdir, datafiles, ref_storage):
+ # XXX: Not really sure what this is testing, an it doesn't seem to use
+ # the kind at any point?
+ project = str(datafiles)
+ element_path = os.path.join(project, "elements")
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(element_path, "junction.bst")
+
+ update_project_configuration(project, {"ref-storage": ref_storage, "(@)": ["junction.bst:test.yml"]})
+
+ generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False)
+
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
+ result.assert_success()
[buildstream] 06/15: testing: Fix type annotation error that mypy
didn't catch
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 09fca56c4f216c1845f90155d216534582f3bf0a
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Sun Oct 11 12:30:43 2020 +0000
testing: Fix type annotation error that mypy didn't catch
---
src/buildstream/testing/__init__.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/buildstream/testing/__init__.py b/src/buildstream/testing/__init__.py
index 19c19a6..7e9f009 100644
--- a/src/buildstream/testing/__init__.py
+++ b/src/buildstream/testing/__init__.py
@@ -21,6 +21,7 @@ This package contains various utilities which make it easier to test plugins.
import os
from collections import OrderedDict
+from typing import Tuple
from buildstream.exceptions import ErrorDomain, LoadErrorReason
from ._yaml import generate_project, generate_element, load_yaml
from .repo import Repo
@@ -47,7 +48,7 @@ except ImportError:
# Of the form plugin_name -> (repo_class, plugin_package)
-ALL_REPO_KINDS = OrderedDict() # type: OrderedDict[Repo, str]
+ALL_REPO_KINDS = OrderedDict() # type: OrderedDict[str, Tuple[Repo, str]]
def create_repo(kind, directory, subdir="repo"):
[buildstream] 07/15: tests: Stop using the 'create_repo' utility,
create them directly
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 22d6093611ca15cb354d4751b1c97268df2be237
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Wed Oct 14 07:41:40 2020 +0000
tests: Stop using the 'create_repo' utility, create them directly
It is more context to know about and the 'create_repo' fixture doesn't
do anything more than instanciating the class in our cases
---
tests/elements/filter.py | 13 ++---
tests/format/include.py | 6 +--
tests/format/junctions.py | 11 +++--
tests/frontend/buildcheckout.py | 4 +-
tests/frontend/cross_junction_workspace.py | 7 +--
tests/frontend/default_target.py | 7 +--
tests/frontend/logging.py | 9 ++--
tests/frontend/mirror.py | 24 ++++-----
tests/frontend/order.py | 6 ++-
tests/frontend/pull.py | 5 +-
tests/frontend/track.py | 16 +++---
tests/sourcecache/fetch.py | 4 +-
tests/sourcecache/push.py | 12 ++---
tests/sources/bzr.py | 5 +-
tests/sources/git.py | 79 +++++++++++++++---------------
tests/sources/no_fetch_cached.py | 5 +-
tests/testutils/element_generators.py | 5 +-
tests/testutils/junction.py | 4 +-
18 files changed, 118 insertions(+), 104 deletions(-)
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index 443f64d..39c9431 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -6,11 +6,12 @@ import shutil
import pytest
-from buildstream.testing import create_repo
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.exceptions import ErrorDomain
from buildstream import _yaml
+from tests.testutils.repo.git import Git
+
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "filter",)
@@ -189,7 +190,7 @@ def test_filter_workspace_reset(datafiles, cli, tmpdir):
@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track(datafiles, cli, tmpdir):
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -235,7 +236,7 @@ def test_filter_track(datafiles, cli, tmpdir):
@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_excepted(datafiles, cli, tmpdir):
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -280,7 +281,7 @@ def test_filter_track_excepted(datafiles, cli, tmpdir):
@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_multi_to_one(datafiles, cli, tmpdir):
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -326,7 +327,7 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir):
@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_multi(datafiles, cli, tmpdir):
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
@@ -387,7 +388,7 @@ def test_filter_track_multi(datafiles, cli, tmpdir):
@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic"))
def test_filter_track_multi_exclude(datafiles, cli, tmpdir):
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(str(datafiles), "files"))
elements_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
diff --git a/tests/format/include.py b/tests/format/include.py
index 11659d4..e56a4f6 100644
--- a/tests/format/include.py
+++ b/tests/format/include.py
@@ -7,8 +7,8 @@ import pytest
from buildstream import _yaml
from buildstream.exceptions import ErrorDomain, LoadErrorReason
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from tests.testutils import generate_junction
+from tests.testutils.repo.git import Git
# Project directory
@@ -109,7 +109,7 @@ def test_junction_element_partial_project_project(cli, tmpdir, datafiles):
subproject_path = os.path.join(project, "subproject")
junction_path = os.path.join(project, "junction.bst")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(subproject_path)
@@ -133,7 +133,7 @@ def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles):
subproject_path = os.path.join(project, "subproject")
junction_path = os.path.join(project, "junction.bst")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(subproject_path)
diff --git a/tests/format/junctions.py b/tests/format/junctions.py
index c4eea2d..2a34068 100644
--- a/tests/format/junctions.py
+++ b/tests/format/junctions.py
@@ -8,7 +8,8 @@ import pytest
from buildstream import _yaml
from buildstream.exceptions import ErrorDomain, LoadErrorReason
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
+
+from tests.testutils.repo.tar import Tar
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "junctions",)
@@ -260,7 +261,7 @@ def test_tar_show(cli, tmpdir, datafiles):
project = os.path.join(str(datafiles), "use-repo")
# Create the repo from 'baserepo' subdir
- repo = create_repo("tar", str(tmpdir))
+ repo = Tar(str(tmpdir))
ref = repo.create(os.path.join(project, "baserepo"))
# Write out junction element with tar source
@@ -279,7 +280,7 @@ def test_tar_build(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the repo from 'baserepo' subdir
- repo = create_repo("tar", str(tmpdir))
+ repo = Tar(str(tmpdir))
ref = repo.create(os.path.join(project, "baserepo"))
# Write out junction element with tar source
@@ -304,7 +305,7 @@ def test_tar_missing_project_conf(cli, tmpdir, datafiles):
os.remove(datafiles / "use-repo" / "baserepo" / "project.conf")
# Create the repo from 'base' subdir
- repo = create_repo("tar", str(tmpdir))
+ repo = Tar(str(tmpdir))
ref = repo.create(os.path.join(project, "baserepo"))
# Write out junction element with tar source
@@ -324,7 +325,7 @@ def test_build_tar_cross_junction_names(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the repo from 'base' subdir
- repo = create_repo("tar", str(tmpdir))
+ repo = Tar(str(tmpdir))
ref = repo.create(os.path.join(project, "baserepo"))
# Write out junction element with tar source
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index 5afa521..48b649c 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -9,13 +9,13 @@ import re
import pytest
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from buildstream.testing._utils.site import IS_WINDOWS, CASD_SEPARATE_USER
from buildstream import _yaml
from buildstream.exceptions import ErrorDomain, LoadErrorReason
from buildstream import utils
from tests.testutils import generate_junction, create_artifact_share
+from tests.testutils.repo.git import Git
from . import configure_project
@@ -1069,7 +1069,7 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
project = str(datafiles)
checkout_dir = os.path.join(str(tmpdir), "checkout")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
index c3b80ae..f1d344f 100644
--- a/tests/frontend/cross_junction_workspace.py
+++ b/tests/frontend/cross_junction_workspace.py
@@ -3,9 +3,10 @@
import os
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from buildstream import _yaml
+from tests.testutils.repo.git import Git
+
def prepare_junction_project(cli, tmpdir):
main_project = tmpdir.join("main")
@@ -23,7 +24,7 @@ def prepare_junction_project(cli, tmpdir):
import_repo_dir = tmpdir.join("import_repo")
os.makedirs(str(import_repo_dir))
- import_repo = create_repo("git", str(import_repo_dir))
+ import_repo = Git(str(import_repo_dir))
import_ref = import_repo.create(str(import_dir))
_yaml.roundtrip_dump(
@@ -32,7 +33,7 @@ def prepare_junction_project(cli, tmpdir):
sub_repo_dir = tmpdir.join("sub_repo")
os.makedirs(str(sub_repo_dir))
- sub_repo = create_repo("git", str(sub_repo_dir))
+ sub_repo = Git(str(sub_repo_dir))
sub_ref = sub_repo.create(str(sub_project))
_yaml.roundtrip_dump(
diff --git a/tests/frontend/default_target.py b/tests/frontend/default_target.py
index bb7a495..828a25f 100644
--- a/tests/frontend/default_target.py
+++ b/tests/frontend/default_target.py
@@ -6,8 +6,9 @@ import os
import pytest
from buildstream import _yaml
-from buildstream.testing import cli, create_repo # pylint: disable=unused-import
+from buildstream.testing import cli # pylint: disable=unused-import
from tests.testutils import create_artifact_share
+from tests.testutils.repo.git import Git
# project directory
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "default-target",)
@@ -95,7 +96,7 @@ def test_default_target_track(cli, tmpdir, datafiles):
target = "track-fetch-test.bst"
# First, create an element with trackable sources
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(project)
element_conf = {"kind": "import", "sources": [repo.source_config()]}
_yaml.roundtrip_dump(element_conf, os.path.join(project, "elements", target))
@@ -125,7 +126,7 @@ def test_default_target_fetch(cli, tmpdir, datafiles):
target = "track-fetch-test.bst"
# First, create an element with trackable sources
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(project)
element_conf = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
_yaml.roundtrip_dump(element_conf, os.path.join(project, "elements", target))
diff --git a/tests/frontend/logging.py b/tests/frontend/logging.py
index 6eb0589..5811e0f 100644
--- a/tests/frontend/logging.py
+++ b/tests/frontend/logging.py
@@ -6,12 +6,13 @@ import re
import pytest
-from buildstream.testing import create_repo
-
from buildstream import _yaml
from buildstream.exceptions import ErrorDomain
from buildstream.testing import cli # pylint: disable=unused-import
+from tests.testutils.repo.git import Git
+
+
# Project directory
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@@ -26,7 +27,7 @@ def test_default_logging(cli, tmpdir, datafiles):
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
#
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(bin_files_path)
# Write out our test target
@@ -55,7 +56,7 @@ def test_custom_logging(cli, tmpdir, datafiles):
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
#
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(bin_files_path)
# Write out our test target
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index bffc754..4884ef3 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -5,9 +5,11 @@ import os
import pytest
from buildstream import _yaml
-from buildstream.testing import create_repo
from buildstream.testing import cli # pylint: disable=unused-import
+from tests.testutils.repo.git import Git
+from tests.testutils.repo.tar import Tar
+
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -67,7 +69,7 @@ def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror):
element_dir = os.path.join(project_dir, "elements")
# Create repo objects of the upstream and mirror
- upstream_repo = create_repo("tar", upstream_repodir)
+ upstream_repo = Tar(upstream_repodir)
upstream_repo.create(bin_files_path)
mirror_repo = upstream_repo.copy(mirror_repodir)
upstream_ref = upstream_repo.create(dev_files_path)
@@ -247,15 +249,15 @@ def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr")
mirror_dir = os.path.join(str(datafiles), "mirror")
- defined_subrepo = create_repo("git", str(tmpdir), "defined_subrepo")
+ defined_subrepo = Git(str(tmpdir), "defined_subrepo")
defined_subrepo.create(bin_files_path)
defined_subrepo.copy(mirror_dir)
defined_subrepo.add_file(foo_file)
- found_subrepo = create_repo("git", str(tmpdir), "found_subrepo")
+ found_subrepo = Git(str(tmpdir), "found_subrepo")
found_subrepo.create(dev_files_path)
- main_repo = create_repo("git", str(tmpdir))
+ main_repo = Git(str(tmpdir))
main_mirror_ref = main_repo.create(bin_files_path)
main_repo.add_submodule("defined", "file://" + defined_subrepo.repo)
main_repo.add_submodule("found", "file://" + found_subrepo.repo)
@@ -320,12 +322,12 @@ def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
upstream_bin_repodir = os.path.join(str(tmpdir), "bin-upstream")
mirror_bin_repodir = os.path.join(str(tmpdir), "bin-mirror")
- upstream_bin_repo = create_repo("git", upstream_bin_repodir)
+ upstream_bin_repo = Git(upstream_bin_repodir)
upstream_bin_repo.create(bin_files_path)
mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
dev_repodir = os.path.join(str(tmpdir), "dev-upstream")
- dev_repo = create_repo("git", dev_repodir)
+ dev_repo = Git(dev_repodir)
dev_repo.create(dev_files_path)
main_files = os.path.join(str(tmpdir), "main-files")
@@ -333,7 +335,7 @@ def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
with open(os.path.join(main_files, "README"), "w") as f:
f.write("TEST\n")
main_repodir = os.path.join(str(tmpdir), "main-upstream")
- main_repo = create_repo("git", main_repodir)
+ main_repo = Git(main_repodir)
main_repo.create(main_files)
upstream_url = "file://{}".format(upstream_bin_repo.repo)
@@ -399,11 +401,11 @@ def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr")
bin_repodir = os.path.join(str(tmpdir), "bin-repo")
- bin_repo = create_repo("git", bin_repodir)
+ bin_repo = Git(bin_repodir)
bin_repo.create(bin_files_path)
dev_repodir = os.path.join(str(tmpdir), "dev-repo")
- dev_repo = create_repo("git", dev_repodir)
+ dev_repo = Git(dev_repodir)
dev_repo.create(dev_files_path)
main_files = os.path.join(str(tmpdir), "main-files")
@@ -411,7 +413,7 @@ def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
with open(os.path.join(main_files, "README"), "w") as f:
f.write("TEST\n")
upstream_main_repodir = os.path.join(str(tmpdir), "main-upstream")
- upstream_main_repo = create_repo("git", upstream_main_repodir)
+ upstream_main_repo = Git(upstream_main_repodir)
upstream_main_repo.create(main_files)
upstream_main_repo.add_submodule("bin", url="file://{}".format(bin_repo.repo))
diff --git a/tests/frontend/order.py b/tests/frontend/order.py
index fbeb7c3..47b6303 100644
--- a/tests/frontend/order.py
+++ b/tests/frontend/order.py
@@ -4,10 +4,12 @@
import os
import pytest
-from buildstream.testing import create_repo
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream import _yaml
+from tests.testutils.repo.git import Git
+
+
# Project directory
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@@ -24,7 +26,7 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
def create_element(project, name, dependencies):
dev_files_path = os.path.join(project, "files", "dev-files")
element_path = os.path.join(project, "elements")
- repo = create_repo("git", project, "{}-repo".format(name))
+ repo = Git(project, "{}-repo".format(name))
ref = repo.create(dev_files_path)
element = {"kind": "import", "sources": [repo.source_config(ref=ref)], "depends": dependencies}
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index c1fa76a..dca3916 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -7,7 +7,6 @@ import stat
import pytest
from buildstream import utils, _yaml
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from tests.testutils import (
create_artifact_share,
create_split_share,
@@ -15,7 +14,7 @@ from tests.testutils import (
assert_shared,
assert_not_shared,
)
-
+from tests.testutils.repo.git import Git
# Project directory
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
@@ -318,7 +317,7 @@ def test_pull_missing_blob_split_share(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_pull_missing_local_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index 682ee0e..7d7fcbd 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -6,11 +6,13 @@ import os
import re
import pytest
-from buildstream.testing import create_repo, generate_project
+from buildstream.testing import generate_project
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.exceptions import ErrorDomain, LoadErrorReason
from buildstream import _yaml
from tests.testutils import generate_junction
+from tests.testutils.repo.git import Git
+from tests.testutils.repo.tar import Tar
from . import configure_project
# Project directory
@@ -37,7 +39,7 @@ def test_track_single(cli, tmpdir, datafiles):
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(dev_files_path)
# Write out our test targets
@@ -77,7 +79,7 @@ def test_track_optional(cli, tmpdir, datafiles, ref_storage):
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(dev_files_path)
# Now create an optional test branch and add a commit to that,
@@ -174,7 +176,7 @@ def test_track_cross_junction(cli, tmpdir, datafiles, cross_junction, ref_storag
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(dev_files_path)
# Generate two elements using the git source, one in
@@ -341,7 +343,7 @@ def test_track_error_cannot_write_file(cli, tmpdir, datafiles):
configure_project(project, {"ref-storage": "inline"})
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(dev_files_path)
element_full_path = os.path.join(element_path, element_name)
@@ -369,7 +371,7 @@ def test_no_needless_overwrite(cli, tmpdir, datafiles):
# Create our repo object of the given source type with
# the dev files, and then collect the initial ref.
#
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(dev_files_path)
# Write out our test target and assert it exists
@@ -456,7 +458,7 @@ def test_track_skip(cli, tmpdir, datafiles):
# Generate a regular import element which will have a workspace open
#
- repo = create_repo("tar", str(tmpdir))
+ repo = Tar(str(tmpdir))
repo.create(dev_files_path)
generate_element(repo, os.path.join(element_path, element_workspace_name))
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index 76f5508..0eef6b7 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -28,15 +28,15 @@ from buildstream.exceptions import ErrorDomain
from buildstream._project import Project
from buildstream import _yaml
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from tests.testutils import create_artifact_share, dummy_context
+from tests.testutils.repo.git import Git
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
def create_test_element(tmpdir, project_dir):
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project_dir, "files"))
element_path = os.path.join(project_dir, "elements")
element_name = "fetch.bst"
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 25a4309..eaae9e7 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -29,8 +29,8 @@ from buildstream.exceptions import ErrorDomain
from buildstream._project import Project
from buildstream import _yaml
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from tests.testutils import create_artifact_share, dummy_context
+from tests.testutils.repo.git import Git
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
@@ -71,7 +71,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project_dir, "files"))
element_path = os.path.join(project_dir, "elements")
element_name = "push.bst"
@@ -121,7 +121,7 @@ def test_source_push(cli, tmpdir, datafiles):
_yaml.roundtrip_dump(user_config, file=user_config_file)
cli.configure(user_config)
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project_dir, "files"))
element_path = os.path.join(project_dir, "elements")
element_name = "push.bst"
@@ -172,7 +172,7 @@ def test_push_pull(cli, datafiles, tmpdir):
cli.configure(user_config)
# create repo to pull from
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project_dir, "files"))
element_path = os.path.join(project_dir, "elements")
element_name = "push.bst"
@@ -210,7 +210,7 @@ def test_push_fail(cli, tmpdir, datafiles):
cli.configure(user_config)
# create repo to pull from
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project_dir, "files"))
element_path = os.path.join(project_dir, "elements")
element_name = "push.bst"
@@ -239,7 +239,7 @@ def test_source_push_build_fail(cli, tmpdir, datafiles):
}
cli.configure(user_config)
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project_dir, "files"))
element_path = os.path.join(project_dir, "elements")
diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py
index 2dcacfe..37e6dc3 100644
--- a/tests/sources/bzr.py
+++ b/tests/sources/bzr.py
@@ -5,10 +5,11 @@ import os
import pytest
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from buildstream.testing import generate_element
from buildstream.testing._utils.site import HAVE_BZR
+from tests.testutils.repo.bzr import Bzr
+
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bzr")
@@ -18,7 +19,7 @@ def test_fetch_checkout(cli, tmpdir, datafiles):
project = str(datafiles)
checkoutdir = os.path.join(str(tmpdir), "checkout")
- repo = create_repo("bzr", str(tmpdir))
+ repo = Bzr(str(tmpdir))
ref = repo.create(os.path.join(project, "basic"))
# Write out our test target
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 30657d8..22b87ea 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -34,9 +34,10 @@ from buildstream.exceptions import ErrorDomain
from buildstream.plugin import CoreWarnings
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing import generate_project, generate_element, load_yaml
-from buildstream.testing import create_repo
from buildstream.testing._utils.site import HAVE_GIT, HAVE_OLD_GIT
+from tests.testutils.repo.git import Git
+
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "git",)
@@ -46,7 +47,7 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Write out our test target with a bad ref
@@ -67,7 +68,7 @@ def test_fetch_shallow(cli, tmpdir, datafiles):
workspacedir = os.path.join(str(tmpdir), "workspace")
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
first_commit = repo.latest_commit()
repo.add_commit()
@@ -94,11 +95,11 @@ def test_submodule_fetch_checkout(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -128,15 +129,15 @@ def test_recursive_submodule_fetch_checkout(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create a submodule from the 'othersubrepofiles' subdir
- subsubrepo = create_repo("git", str(tmpdir), "subsubrepo")
+ subsubrepo = Git(str(tmpdir), "subsubrepo")
subsubrepo.create(os.path.join(project, "othersubrepofiles"))
# Create another submodule from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Configure submodules
@@ -168,11 +169,11 @@ def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -202,11 +203,11 @@ def test_submodule_fetch_source_disable(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -236,11 +237,11 @@ def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -270,15 +271,15 @@ def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create another submodule from the 'othersubrepofiles' subdir
- other_subrepo = create_repo("git", str(tmpdir), "othersubrepo")
+ other_subrepo = Git(str(tmpdir), "othersubrepo")
other_subrepo.create(os.path.join(project, "othersubrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -310,15 +311,15 @@ def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, dat
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create another submodule from the 'othersubrepofiles' subdir
- other_subrepo = create_repo("git", str(tmpdir), "othersubrepo")
+ other_subrepo = Git(str(tmpdir), "othersubrepo")
other_subrepo.create(os.path.join(project, "othersubrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -350,11 +351,11 @@ def test_submodule_fetch_project_override(cli, tmpdir, datafiles):
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -383,7 +384,7 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project, "repofiles"))
# Write out our test target
@@ -415,7 +416,7 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Write out our test target
@@ -441,7 +442,7 @@ def test_ref_not_in_track(cli, tmpdir, datafiles, fail):
generate_project(project, config={"fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]})
# Create the repo from 'repofiles', create a branch without latest commit
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project, "repofiles"))
gitsource = repo.source_config(ref=ref)
@@ -475,11 +476,11 @@ def test_unlisted_submodule(cli, tmpdir, datafiles, fail):
generate_project(project, config={"fatal-warnings": ["git:unlisted-submodule"]})
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -533,11 +534,11 @@ def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail):
generate_project(project, config={"fatal-warnings": ["git:unlisted-submodule"]})
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created, but use
@@ -587,7 +588,7 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail):
generate_project(project, config={"fatal-warnings": ["git:invalid-submodule"]})
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project, "repofiles"))
# Create the source without any submodules, and add
@@ -640,11 +641,11 @@ def test_track_invalid_submodule(cli, tmpdir, datafiles, fail):
generate_project(project, config={"fatal-warnings": ["git:invalid-submodule"]})
# Create the submodule first from the 'subrepofiles' subdir
- subrepo = create_repo("git", str(tmpdir), "subrepo")
+ subrepo = Git(str(tmpdir), "subrepo")
subrepo.create(os.path.join(project, "subrepofiles"))
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
# Add a submodule pointing to the one we created
@@ -691,7 +692,7 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
project = str(datafiles)
# Create the repo from 'repofiles' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(os.path.join(project, "repofiles"))
if tag:
repo.add_tag("tag")
@@ -743,7 +744,7 @@ def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
with open(file0, "w") as f:
f.write("test\n")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
def tag(name):
if tag_type == "annotated":
@@ -853,7 +854,7 @@ def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_ty
with open(file0, "w") as f:
f.write("test\n")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
def tag(name):
if tag_type == "annotated":
@@ -960,7 +961,7 @@ def test_git_describe_relevant_history(cli, tmpdir, datafiles):
with open(file0, "w") as f:
f.write("test\n")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(repofiles)
file1 = os.path.join(str(tmpdir), "file1")
@@ -1033,7 +1034,7 @@ def test_default_do_not_track_tags(cli, tmpdir, datafiles):
with open(file0, "w") as f:
f.write("test\n")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
repo.create(repofiles)
repo.add_tag("tag")
@@ -1073,7 +1074,7 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles):
with open(file0, "w") as f:
f.write("test\n")
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
top_commit = repo.create(repofiles)
@@ -1119,7 +1120,7 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles):
otherpath = os.path.join(str(tmpdir), "other_path")
shutil.copytree(repo.repo, os.path.join(otherpath, "repo"))
- create_repo("git", otherpath)
+ Git(otherpath)
repodir, reponame = os.path.split(repo.repo)
diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py
index 4984f53..ece0ca3 100644
--- a/tests/sources/no_fetch_cached.py
+++ b/tests/sources/no_fetch_cached.py
@@ -5,10 +5,11 @@ import os
import pytest
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import create_repo
from buildstream.testing import generate_element
from buildstream.testing._utils.site import HAVE_GIT
+from tests.testutils.repo.git import Git
+
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "no-fetch-cached")
@@ -22,7 +23,7 @@ def test_no_fetch_cached(cli, tmpdir, datafiles):
project = str(datafiles)
# Create the repo from 'files' subdir
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
ref = repo.create(os.path.join(project, "files"))
# Write out test target with a cached and a non-cached source
diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py
index 6da465a..5b0a922 100644
--- a/tests/testutils/element_generators.py
+++ b/tests/testutils/element_generators.py
@@ -2,7 +2,8 @@ import os
from buildstream import _yaml
from buildstream import utils
-from buildstream.testing import create_repo
+
+from .repo.git import Git
# create_element_size()
@@ -29,7 +30,7 @@ def create_element_size(name, project_dir, elements_path, dependencies, size):
# Create a git repo
repodir = os.path.join(project_dir, "repos")
- repo = create_repo("git", repodir, subdir=name)
+ repo = Git(repodir, subdir=name)
with utils._tempdir(dir=project_dir) as tmp:
diff --git a/tests/testutils/junction.py b/tests/testutils/junction.py
index 4889fdb..856ea5c 100644
--- a/tests/testutils/junction.py
+++ b/tests/testutils/junction.py
@@ -1,6 +1,6 @@
from buildstream import _yaml
-from buildstream.testing import create_repo
+from tests.testutils.repo.git import Git
# generate_junction()
#
@@ -19,7 +19,7 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True,
# Create a repo to hold the subproject and generate
# a junction element for it
#
- repo = create_repo("git", str(tmpdir))
+ repo = Git(str(tmpdir))
source_ref = ref = repo.create(subproject_path)
if not store_ref:
source_ref = None
[buildstream] 12/15: track_cross_junction.py: Adapt to use the
standard source tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e24a133b9dbe80faa28769c51e5f7fe99109d7d1
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Tue Oct 6 20:03:51 2020 +0000
track_cross_junction.py: Adapt to use the standard source tests
---
src/buildstream/testing/_sourcetests/__init__.py | 5 +-
src/buildstream/testing/_sourcetests/base.py | 22 +++
.../testing/_sourcetests/track_cross_junction.py | 203 ++++++++++-----------
3 files changed, 124 insertions(+), 106 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/__init__.py b/src/buildstream/testing/_sourcetests/__init__.py
index c21d185..2a1f26a 100644
--- a/src/buildstream/testing/_sourcetests/__init__.py
+++ b/src/buildstream/testing/_sourcetests/__init__.py
@@ -23,9 +23,12 @@ from .build_checkout import BuildCheckoutSourceTests
from .fetch import FetchSourceTests
from .mirror import MirrorSourceTests
from .source_determinism import SourceDeterminismTests
+from .track_cross_junction import TrackCrossJunctionTests
__all__ = ["SourceTests"]
-class SourceTests(BuildCheckoutSourceTests, FetchSourceTests, MirrorSourceTests, SourceDeterminismTests):
+class SourceTests(
+ BuildCheckoutSourceTests, FetchSourceTests, MirrorSourceTests, SourceDeterminismTests, TrackCrossJunctionTests
+):
"""Definition of standardized tests that each source should pass."""
diff --git a/src/buildstream/testing/_sourcetests/base.py b/src/buildstream/testing/_sourcetests/base.py
index 6ee361d..c7eabfb 100644
--- a/src/buildstream/testing/_sourcetests/base.py
+++ b/src/buildstream/testing/_sourcetests/base.py
@@ -19,10 +19,14 @@ import os
from abc import ABC, abstractmethod
from typing import Type
+from buildstream import _yaml
from ..repo import Repo
class BaseSourceTests(ABC):
+ PACKAGE = None
+ """The package from which the plugin is coming, if not coming from BuildStream core."""
+
@property
@classmethod
@abstractmethod
@@ -34,3 +38,21 @@ class BaseSourceTests(ABC):
@abstractmethod
def REPO(cls) -> Type[Repo]:
"""Get the repo implementation for the currently tested source."""
+
+ def add_plugins_conf(self, project):
+ """
+ Add the given plugin to the configuration of the given project.
+
+ Args:
+ project (str): path to the project on which to register the plugin
+ plugin_kind (str): name of the plugin kind to register
+ """
+ project_conf_file = os.path.join(project, "project.conf")
+ project_conf = _yaml.roundtrip_load(project_conf_file)
+
+ if self.PACKAGE is not None:
+ project_conf["plugins"] = [
+ {"origin": "pip", "package-name": self.PACKAGE, "sources": [self.KIND],},
+ ]
+
+ _yaml.roundtrip_dump(project_conf, project_conf_file)
diff --git a/src/buildstream/testing/_sourcetests/track_cross_junction.py b/src/buildstream/testing/_sourcetests/track_cross_junction.py
index 2c41419..adb643a 100644
--- a/src/buildstream/testing/_sourcetests/track_cross_junction.py
+++ b/src/buildstream/testing/_sourcetests/track_cross_junction.py
@@ -21,13 +21,10 @@
import os
-import pytest
-
from buildstream import _yaml
+from .base import BaseSourceTests
from .._utils import generate_junction
-from .. import create_repo, ALL_REPO_KINDS
from .. import cli # pylint: disable=unused-import
-from .utils import add_plugins_conf
# Project directory
@@ -43,39 +40,6 @@ def generate_element(repo, element_path, dep_name=None):
_yaml.roundtrip_dump(element, element_path)
-def generate_import_element(tmpdir, kind, project, name):
- element_name = "import-{}.bst".format(name)
- repo_element_path = os.path.join(project, "elements", element_name)
- files = str(tmpdir.join("imported_files_{}".format(name)))
- os.makedirs(files)
-
- with open(os.path.join(files, "{}.txt".format(name)), "w") as f:
- f.write(name)
-
- repo = create_repo(kind, str(tmpdir.join("element_{}_repo".format(name))))
- repo.create(files)
-
- generate_element(repo, repo_element_path)
-
- return element_name
-
-
-def generate_project(tmpdir, name, kind, config=None):
- if config is None:
- config = {}
-
- project_name = "project-{}".format(name)
- subproject_path = os.path.join(str(tmpdir.join(project_name)))
- os.makedirs(os.path.join(subproject_path, "elements"))
-
- project_conf = {"name": name, "min-version": "2.0", "element-path": "elements"}
- project_conf.update(config)
- _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, "project.conf"))
- add_plugins_conf(subproject_path, kind)
-
- return project_name, subproject_path
-
-
def generate_simple_stack(project, name, dependencies):
element_name = "{}.bst".format(name)
element_path = os.path.join(project, "elements", element_name)
@@ -94,91 +58,120 @@ def generate_cross_element(project, subproject_name, import_name):
)
-@pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys())
-def test_cross_junction_multiple_projects(cli, tmpdir, kind):
- tmpdir = tmpdir.join(kind)
+class TrackCrossJunctionTests(BaseSourceTests):
+ def generate_project(self, tmpdir, name, config=None):
+ if config is None:
+ config = {}
- # Generate 3 projects: main, a, b
- _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"})
- project_a, project_a_path = generate_project(tmpdir, "a", kind)
- project_b, project_b_path = generate_project(tmpdir, "b", kind)
+ project_name = "project-{}".format(name)
+ subproject_path = os.path.join(str(tmpdir.join(project_name)))
+ os.makedirs(os.path.join(subproject_path, "elements"))
- # Generate an element with a trackable source for each project
- element_a = generate_import_element(tmpdir, kind, project_a_path, "a")
- element_b = generate_import_element(tmpdir, kind, project_b_path, "b")
- element_c = generate_import_element(tmpdir, kind, project, "c")
+ project_conf = {"name": name, "min-version": "2.0", "element-path": "elements"}
+ project_conf.update(config)
+ _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, "project.conf"))
+ self.add_plugins_conf(subproject_path)
- # Create some indirections to the elements with dependencies to test --deps
- stack_a = generate_simple_stack(project_a_path, "stack-a", [element_a])
- stack_b = generate_simple_stack(project_b_path, "stack-b", [element_b])
+ return project_name, subproject_path
- # Create junctions for projects a and b in main.
- junction_a = "{}.bst".format(project_a)
- junction_a_path = os.path.join(project, "elements", junction_a)
- generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False)
+ def generate_import_element(self, tmpdir, project, name):
+ element_name = "import-{}.bst".format(name)
+ repo_element_path = os.path.join(project, "elements", element_name)
+ files = str(tmpdir.join("imported_files_{}".format(name)))
+ os.makedirs(files)
- junction_b = "{}.bst".format(project_b)
- junction_b_path = os.path.join(project, "elements", junction_b)
- generate_junction(tmpdir.join("repo_b"), project_b_path, junction_b_path, store_ref=False)
+ with open(os.path.join(files, "{}.txt".format(name)), "w") as f:
+ f.write(name)
- # Track the junctions.
- result = cli.run(project=project, args=["source", "track", junction_a, junction_b])
- result.assert_success()
+ repo = self.REPO(str(tmpdir.join("element_{}_repo".format(name))))
+ repo.create(files)
- # Import elements from a and b in to main.
- imported_a = generate_cross_element(project, project_a, stack_a)
- imported_b = generate_cross_element(project, project_b, stack_b)
+ generate_element(repo, repo_element_path)
- # Generate a top level stack depending on everything
- all_bst = generate_simple_stack(project, "all", [imported_a, imported_b, element_c])
+ return element_name
- # Track without following junctions. But explicitly also track the elements in project a.
- result = cli.run(
- project=project, args=["source", "track", "--deps", "all", all_bst, "{}:{}".format(junction_a, stack_a)]
- )
- result.assert_success()
+ def test_cross_junction_multiple_projects(self, cli, tmpdir):
+ tmpdir = tmpdir.join(self.KIND)
- # Elements in project b should not be tracked. But elements in project a and main should.
- expected = [element_c, "{}:{}".format(junction_a, element_a)]
- assert set(result.get_tracked_elements()) == set(expected)
+ # Generate 3 projects: main, a, b
+ _, project = self.generate_project(tmpdir, "main", {"ref-storage": "project.refs"})
+ project_a, project_a_path = self.generate_project(tmpdir, "a")
+ project_b, project_b_path = self.generate_project(tmpdir, "b",)
+ # Generate an element with a trackable source for each project
+ element_a = self.generate_import_element(tmpdir, project_a_path, "a")
+ element_b = self.generate_import_element(tmpdir, project_b_path, "b")
+ element_c = self.generate_import_element(tmpdir, project, "c")
-@pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys())
-def test_track_exceptions(cli, tmpdir, kind):
- tmpdir = tmpdir.join(kind)
+ # Create some indirections to the elements with dependencies to test --deps
+ stack_a = generate_simple_stack(project_a_path, "stack-a", [element_a])
+ stack_b = generate_simple_stack(project_b_path, "stack-b", [element_b])
- _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"})
- project_a, project_a_path = generate_project(tmpdir, "a", kind)
+ # Create junctions for projects a and b in main.
+ junction_a = "{}.bst".format(project_a)
+ junction_a_path = os.path.join(project, "elements", junction_a)
+ generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False)
- element_a = generate_import_element(tmpdir, kind, project_a_path, "a")
- element_b = generate_import_element(tmpdir, kind, project_a_path, "b")
+ junction_b = "{}.bst".format(project_b)
+ junction_b_path = os.path.join(project, "elements", junction_b)
+ generate_junction(tmpdir.join("repo_b"), project_b_path, junction_b_path, store_ref=False)
- all_bst = generate_simple_stack(project_a_path, "all", [element_a, element_b])
+ # Track the junctions.
+ result = cli.run(project=project, args=["source", "track", junction_a, junction_b])
+ result.assert_success()
- junction_a = "{}.bst".format(project_a)
- junction_a_path = os.path.join(project, "elements", junction_a)
- generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False)
+ # Import elements from a and b in to main.
+ imported_a = generate_cross_element(project, project_a, stack_a)
+ imported_b = generate_cross_element(project, project_b, stack_b)
- result = cli.run(project=project, args=["source", "track", junction_a])
- result.assert_success()
+ # Generate a top level stack depending on everything
+ all_bst = generate_simple_stack(project, "all", [imported_a, imported_b, element_c])
- imported_b = generate_cross_element(project, project_a, element_b)
- indirection = generate_simple_stack(project, "indirection", [imported_b])
+ # Track without following junctions. But explicitly also track the elements in project a.
+ result = cli.run(
+ project=project, args=["source", "track", "--deps", "all", all_bst, "{}:{}".format(junction_a, stack_a)]
+ )
+ result.assert_success()
- result = cli.run(
- project=project,
- args=[
- "source",
- "track",
- "--deps",
- "all",
- "--except",
- indirection,
- "{}:{}".format(junction_a, all_bst),
- imported_b,
- ],
- )
- result.assert_success()
+ # Elements in project b should not be tracked. But elements in project a and main should.
+ expected = [element_c, "{}:{}".format(junction_a, element_a)]
+ assert set(result.get_tracked_elements()) == set(expected)
+
+ def test_track_exceptions(self, cli, tmpdir):
+ tmpdir = tmpdir.join(self.KIND)
+
+ _, project = self.generate_project(tmpdir, "main", {"ref-storage": "project.refs"})
+ project_a, project_a_path = self.generate_project(tmpdir, "a")
+
+ element_a = self.generate_import_element(tmpdir, project_a_path, "a")
+ element_b = self.generate_import_element(tmpdir, project_a_path, "b")
+
+ all_bst = generate_simple_stack(project_a_path, "all", [element_a, element_b])
+
+ junction_a = "{}.bst".format(project_a)
+ junction_a_path = os.path.join(project, "elements", junction_a)
+ generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False)
+
+ result = cli.run(project=project, args=["source", "track", junction_a])
+ result.assert_success()
+
+ imported_b = generate_cross_element(project, project_a, element_b)
+ indirection = generate_simple_stack(project, "indirection", [imported_b])
+
+ result = cli.run(
+ project=project,
+ args=[
+ "source",
+ "track",
+ "--deps",
+ "all",
+ "--except",
+ indirection,
+ "{}:{}".format(junction_a, all_bst),
+ imported_b,
+ ],
+ )
+ result.assert_success()
- expected = ["{}:{}".format(junction_a, element_a), "{}:{}".format(junction_a, element_b)]
- assert set(result.get_tracked_elements()) == set(expected)
+ expected = ["{}:{}".format(junction_a, element_a), "{}:{}".format(junction_a, element_b)]
+ assert set(result.get_tracked_elements()) == set(expected)
[buildstream] 15/15: tests: Stop using the implicit source tests
and explicitely run them
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit c958d4c9bb15e689b13408599245e360f215df21
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Sun Oct 11 12:46:25 2020 +0000
tests: Stop using the implicit source tests and explicitely run them
This makes use of the new helper classes to generate the source tests,
making them explicit to run
---
tests/conftest.py | 29 -----------------------------
tests/sources/bzr.py | 7 ++++++-
tests/sources/git.py | 7 +++++++
tests/sources/tar.py | 8 +++++++-
tests/sources/zip.py | 8 +++++++-
5 files changed, 27 insertions(+), 32 deletions(-)
diff --git a/tests/conftest.py b/tests/conftest.py
index d79ad40..30888a3 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -21,10 +21,8 @@
#
import os
-import pkg_resources
import pytest
-from buildstream.testing import register_repo_kind, sourcetests_collection_hook
from buildstream.testing._fixtures import ( # pylint: disable=unused-import
default_thread_number,
reset_global_node_state,
@@ -33,12 +31,6 @@ from buildstream.testing._fixtures import ( # pylint: disable=unused-import
from buildstream.testing.integration import integration_cache # pylint: disable=unused-import
-from tests.testutils.repo.git import Git
-from tests.testutils.repo.bzr import Bzr
-from tests.testutils.repo.tar import Tar
-from tests.testutils.repo.zip import Zip
-
-
#
# This file is loaded by pytest, we use it to add a custom
# `--integration` option to our test suite, and to install
@@ -130,27 +122,6 @@ def remote_services(request):
#################################################
-# Setup for templated source tests #
-#################################################
-register_repo_kind("git", Git, None)
-register_repo_kind("bzr", Bzr, None)
-register_repo_kind("tar", Tar, None)
-register_repo_kind("zip", Zip, None)
-
-
-# This hook enables pytest to collect the templated source tests from
-# buildstream.testing
-def pytest_sessionstart(session):
- if session.config.getvalue("plugins"):
- # Enable all plugins that implement the 'buildstream.tests.source_plugins' hook
- for entrypoint in pkg_resources.iter_entry_points("buildstream.tests.source_plugins"):
- module = entrypoint.load()
- module.register_sources()
-
- sourcetests_collection_hook(session)
-
-
-#################################################
# Isolated environment #
#################################################
@pytest.fixture(scope="session", autouse=True)
diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py
index 37e6dc3..52def66 100644
--- a/tests/sources/bzr.py
+++ b/tests/sources/bzr.py
@@ -5,7 +5,7 @@ import os
import pytest
from buildstream.testing import cli # pylint: disable=unused-import
-from buildstream.testing import generate_element
+from buildstream.testing import generate_element, SourceTests
from buildstream.testing._utils.site import HAVE_BZR
from tests.testutils.repo.bzr import Bzr
@@ -13,6 +13,11 @@ from tests.testutils.repo.bzr import Bzr
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bzr")
+class TestBzrSource(SourceTests):
+ KIND = "bzr"
+ REPO = Bzr
+
+
@pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR))
def test_fetch_checkout(cli, tmpdir, datafiles):
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 22b87ea..79d9886 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -34,13 +34,20 @@ from buildstream.exceptions import ErrorDomain
from buildstream.plugin import CoreWarnings
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing import generate_project, generate_element, load_yaml
+from buildstream.testing import SourceTests
from buildstream.testing._utils.site import HAVE_GIT, HAVE_OLD_GIT
from tests.testutils.repo.git import Git
+
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "git",)
+class TestGitSource(SourceTests):
+ KIND = "git"
+ REPO = Git
+
+
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, "template"))
def test_fetch_bad_ref(cli, tmpdir, datafiles):
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index ed662dc..48d6eea 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -12,10 +12,11 @@ import pytest
from buildstream import utils
from buildstream.exceptions import ErrorDomain
-from buildstream.testing import generate_project, generate_element
+from buildstream.testing import generate_project, generate_element, SourceTests
from buildstream.testing import cli # pylint: disable=unused-import
from buildstream.testing._utils.site import HAVE_LZIP
from tests.testutils.file_server import create_file_server
+from tests.testutils.repo.tar import Tar
from . import list_dir_contents
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tar",)
@@ -41,6 +42,11 @@ def _assemble_tar_lz(workingdir, srcdir, dstfile):
os.chdir(old_dir)
+class TestTarSource(SourceTests):
+ KIND = "tar"
+ REPO = Tar
+
+
# Test that without ref, consistency is set appropriately.
@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
def test_no_ref(cli, tmpdir, datafiles):
diff --git a/tests/sources/zip.py b/tests/sources/zip.py
index 201271f..9bcc009 100644
--- a/tests/sources/zip.py
+++ b/tests/sources/zip.py
@@ -7,9 +7,10 @@ import zipfile
import pytest
from buildstream.exceptions import ErrorDomain
-from buildstream.testing import generate_project
+from buildstream.testing import generate_project, SourceTests
from buildstream.testing import cli # pylint: disable=unused-import
from tests.testutils.file_server import create_file_server
+from tests.testutils.repo.zip import Zip
from . import list_dir_contents
DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "zip",)
@@ -27,6 +28,11 @@ def _assemble_zip(workingdir, dstfile):
os.chdir(old_dir)
+class TestZipSource(SourceTests):
+ KIND = "zip"
+ REPO = Zip
+
+
# Test that without ref, consistency is set appropriately.
@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
def test_no_ref(cli, tmpdir, datafiles):
[buildstream] 01/15: workspace.py: Simplify test by only allowing
opening a single workspace
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 301c1649d8ecead56dfa89cde956e8a74c35d771
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Thu Oct 8 21:23:50 2020 +0000
workspace.py: Simplify test by only allowing opening a single workspace
This test seems to be copy pasted from our tests around workspaces and
doesn't need to be so complicated
---
src/buildstream/testing/_sourcetests/workspace.py | 62 ++++-------------------
1 file changed, 10 insertions(+), 52 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index 3520a8c..7f3c5de 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -67,33 +67,14 @@ class WorkspaceCreator:
if element_attrs:
element = {**element, **element_attrs}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- return element_name, element_path, workspace_dir
-
- def create_workspace_elements(self, kinds, suffixs=None, workspace_dir_usr=None, element_attrs=None):
-
- element_tuples = []
-
- if suffixs is None:
- suffixs = ["",] * len(kinds)
- else:
- if len(suffixs) != len(kinds):
- raise "terable error"
-
- for suffix, kind in zip(suffixs, kinds):
- element_name, _, workspace_dir = self.create_workspace_element(
- kind, suffix, workspace_dir_usr, element_attrs
- )
- element_tuples.append((element_name, workspace_dir))
# Assert that there is no reference, a fetch is needed
- states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples])
- assert not any(states[e] != "fetch needed" for e, _ in element_tuples)
-
- return element_tuples
+ assert self.cli.get_element_state(self.project_path, element_name) == "fetch needed"
+ return element_name, workspace_dir
- def open_workspaces(self, kinds, suffixs=None, workspace_dir=None, element_attrs=None, no_checkout=False):
+ def open_workspace(self, kind, suffix=None, workspace_dir=None, element_attrs=None, no_checkout=False):
- element_tuples = self.create_workspace_elements(kinds, suffixs, workspace_dir, element_attrs)
+ element_name, workspace_dir = self.create_workspace_element(kind, suffix, workspace_dir, element_attrs)
os.makedirs(self.workspace_cmd, exist_ok=True)
# Now open the workspace, this should have the effect of automatically
@@ -103,46 +84,23 @@ class WorkspaceCreator:
if no_checkout:
args.append("--no-checkout")
if workspace_dir is not None:
- assert len(element_tuples) == 1, "test logic error"
- _, workspace_dir = element_tuples[0]
args.extend(["--directory", workspace_dir])
- args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
+ args.append(element_name)
result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
result.assert_success()
if not no_checkout:
# Assert that we are now buildable because the source is now cached.
- states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples])
- assert not any(states[e] != "buildable" for e, _ in element_tuples)
+ assert self.cli.get_element_state(self.project_path, element_name) == "buildable"
# Check that the executable hello file is found in each workspace
- for _, workspace in element_tuples:
- filename = os.path.join(workspace, "usr", "bin", "hello")
- assert os.path.exists(filename)
-
- return element_tuples
-
-
-def open_workspace(
- cli,
- tmpdir,
- datafiles,
- kind,
- suffix="",
- workspace_dir=None,
- project_path=None,
- element_attrs=None,
- no_checkout=False,
-):
- workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path)
- workspaces = workspace_object.open_workspaces((kind,), (suffix,), workspace_dir, element_attrs, no_checkout)
- assert len(workspaces) == 1
- element_name, workspace = workspaces[0]
- return element_name, workspace_object.project_path, workspace
+ filename = os.path.join(workspace_dir, "usr", "bin", "hello")
+ assert os.path.exists(filename)
@pytest.mark.datafiles(DATA_DIR)
def test_open(cli, tmpdir, datafiles, kind):
- open_workspace(cli, tmpdir, datafiles, kind)
+ workspace_object = WorkspaceCreator(cli, tmpdir, datafiles)
+ workspace_object.open_workspace(kind)
[buildstream] 11/15: source_determinism.py: Adapt to use the
standard source tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 118b4b88f5275fe0aed2223128dccd3c4712c808
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Tue Oct 6 19:48:27 2020 +0000
source_determinism.py: Adapt to use the standard source tests
---
src/buildstream/testing/_sourcetests/__init__.py | 3 +-
.../testing/_sourcetests/source_determinism.py | 136 +++++++++++----------
2 files changed, 71 insertions(+), 68 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/__init__.py b/src/buildstream/testing/_sourcetests/__init__.py
index f046b02..c21d185 100644
--- a/src/buildstream/testing/_sourcetests/__init__.py
+++ b/src/buildstream/testing/_sourcetests/__init__.py
@@ -22,9 +22,10 @@
from .build_checkout import BuildCheckoutSourceTests
from .fetch import FetchSourceTests
from .mirror import MirrorSourceTests
+from .source_determinism import SourceDeterminismTests
__all__ = ["SourceTests"]
-class SourceTests(BuildCheckoutSourceTests, FetchSourceTests, MirrorSourceTests):
+class SourceTests(BuildCheckoutSourceTests, FetchSourceTests, MirrorSourceTests, SourceDeterminismTests):
"""Definition of standardized tests that each source should pass."""
diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py
index b834f32..af5ea63 100644
--- a/src/buildstream/testing/_sourcetests/source_determinism.py
+++ b/src/buildstream/testing/_sourcetests/source_determinism.py
@@ -23,10 +23,9 @@ import os
import pytest
from buildstream import _yaml
+from .base import BaseSourceTests
from .._utils.site import HAVE_SANDBOX, CASD_SEPARATE_USER
-from .. import create_repo
from .. import cli # pylint: disable=unused-import
-from .utils import kind # pylint: disable=unused-import
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -47,68 +46,71 @@ def create_test_directory(*path, mode=0o644):
os.chmod(path, mode)
-@pytest.mark.integration
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
- if CASD_SEPARATE_USER and kind == "ostree":
- pytest.xfail("The ostree plugin ignores the umask")
-
- project = str(datafiles)
- element_name = "list.bst"
- element_path = os.path.join(project, "elements", element_name)
- repodir = os.path.join(str(tmpdir), "repo")
- sourcedir = os.path.join(project, "source")
-
- create_test_file(sourcedir, "a.txt", mode=0o700)
- create_test_file(sourcedir, "b.txt", mode=0o755)
- create_test_file(sourcedir, "c.txt", mode=0o600)
- create_test_file(sourcedir, "d.txt", mode=0o400)
- create_test_file(sourcedir, "e.txt", mode=0o644)
- create_test_file(sourcedir, "f.txt", mode=0o4755)
- create_test_file(sourcedir, "g.txt", mode=0o2755)
- create_test_file(sourcedir, "h.txt", mode=0o1755)
- create_test_directory(sourcedir, "dir-a", mode=0o0700)
- create_test_directory(sourcedir, "dir-c", mode=0o0755)
- create_test_directory(sourcedir, "dir-d", mode=0o4755)
- create_test_directory(sourcedir, "dir-e", mode=0o2755)
- create_test_directory(sourcedir, "dir-f", mode=0o1755)
-
- repo = create_repo(kind, repodir)
- ref = repo.create(sourcedir)
- source = repo.source_config(ref=ref)
- element = {
- "kind": "manual",
- "depends": [{"filename": "base.bst", "type": "build"}],
- "sources": [source],
- "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']},
- }
- _yaml.roundtrip_dump(element, element_path)
-
- def get_value_for_umask(umask):
- checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(umask))
-
- old_umask = os.umask(umask)
-
- try:
- test_values = []
- result = cli.run(project=project, args=["build", element_name])
- result.assert_success()
-
- result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir])
- result.assert_success()
-
- with open(os.path.join(checkoutdir, "ls-l"), "r") as f:
- for line in f.readlines():
- test_values.append(line.split()[0] + " " + line.split()[-1])
- return test_values
- finally:
- os.umask(old_umask)
- cli.remove_artifact_from_cache(project, element_name)
-
- if CASD_SEPARATE_USER:
- # buildbox-casd running as separate user of the same group can't
- # function in a test environment with a too restrictive umask.
- assert get_value_for_umask(0o002) == get_value_for_umask(0o007)
- else:
- assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
+class SourceDeterminismTests(BaseSourceTests):
+ @pytest.mark.integration
+ @pytest.mark.datafiles(DATA_DIR)
+ @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+ def test_deterministic_source_umask(self, cli, tmpdir, datafiles):
+ if CASD_SEPARATE_USER and self.KIND == "ostree":
+ pytest.xfail("The ostree plugin ignores the umask")
+
+ project = str(datafiles)
+ element_name = "list.bst"
+ element_path = os.path.join(project, "elements", element_name)
+ repodir = os.path.join(str(tmpdir), "repo")
+ sourcedir = os.path.join(project, "source")
+
+ create_test_file(sourcedir, "a.txt", mode=0o700)
+ create_test_file(sourcedir, "b.txt", mode=0o755)
+ create_test_file(sourcedir, "c.txt", mode=0o600)
+ create_test_file(sourcedir, "d.txt", mode=0o400)
+ create_test_file(sourcedir, "e.txt", mode=0o644)
+ create_test_file(sourcedir, "f.txt", mode=0o4755)
+ create_test_file(sourcedir, "g.txt", mode=0o2755)
+ create_test_file(sourcedir, "h.txt", mode=0o1755)
+ create_test_directory(sourcedir, "dir-a", mode=0o0700)
+ create_test_directory(sourcedir, "dir-c", mode=0o0755)
+ create_test_directory(sourcedir, "dir-d", mode=0o4755)
+ create_test_directory(sourcedir, "dir-e", mode=0o2755)
+ create_test_directory(sourcedir, "dir-f", mode=0o1755)
+
+ repo = self.REPO(repodir)
+ ref = repo.create(sourcedir)
+ source = repo.source_config(ref=ref)
+ element = {
+ "kind": "manual",
+ "depends": [{"filename": "base.bst", "type": "build"}],
+ "sources": [source],
+ "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']},
+ }
+ _yaml.roundtrip_dump(element, element_path)
+
+ def get_value_for_umask(umask):
+ checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(umask))
+
+ old_umask = os.umask(umask)
+
+ try:
+ test_values = []
+ result = cli.run(project=project, args=["build", element_name])
+ result.assert_success()
+
+ result = cli.run(
+ project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir]
+ )
+ result.assert_success()
+
+ with open(os.path.join(checkoutdir, "ls-l"), "r") as f:
+ for line in f.readlines():
+ test_values.append(line.split()[0] + " " + line.split()[-1])
+ return test_values
+ finally:
+ os.umask(old_umask)
+ cli.remove_artifact_from_cache(project, element_name)
+
+ if CASD_SEPARATE_USER:
+ # buildbox-casd running as separate user of the same group can't
+ # function in a test environment with a too restrictive umask.
+ assert get_value_for_umask(0o002) == get_value_for_umask(0o007)
+ else:
+ assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
[buildstream] 08/15: sourcetests: Create a new SourceTests class
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit ad99234e3a853ad8a64dca18193ac6eaa776222c
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Sat Aug 22 13:34:56 2020 +0000
sourcetests: Create a new SourceTests class
This adds a new 'SourceTests' class that will contain all
standardised tests that authors of sources would want to pass.
This lets the users only inherit this class and then implement the
missing parts and all the tests would be able to run explicitely
---
src/buildstream/testing/__init__.py | 8 ++
src/buildstream/testing/_sourcetests/__init__.py | 28 +++++++
src/buildstream/testing/_sourcetests/base.py | 36 +++++++++
src/buildstream/testing/_sourcetests/fetch.py | 98 +++++++++++-------------
src/buildstream/testing/_sourcetests/generic.py | 25 ++++++
tests/testutils/repo/bzr.py | 2 +-
tests/testutils/repo/git.py | 2 +-
7 files changed, 142 insertions(+), 57 deletions(-)
diff --git a/src/buildstream/testing/__init__.py b/src/buildstream/testing/__init__.py
index 7e9f009..09606c3 100644
--- a/src/buildstream/testing/__init__.py
+++ b/src/buildstream/testing/__init__.py
@@ -30,6 +30,7 @@ from .integration import integration_cache
from ._cachekeys import check_cache_key_stability
__all__ = [
+ "SourceTests",
"check_cache_key_stability",
"create_repo",
"register_repo_kind",
@@ -133,3 +134,10 @@ def sourcetests_collection_hook(session):
# automatically collect templated tests.
if should_collect_tests(session.config):
session.config.args.append(source_test_path)
+
+
+# XXX: this needs to be imported last, as it depends on things from
+# this module. We should move that back up once we've finished the
+# refactor of the source tests
+# pylint: disable=wrong-import-position
+from ._sourcetests import SourceTests
diff --git a/src/buildstream/testing/_sourcetests/__init__.py b/src/buildstream/testing/_sourcetests/__init__.py
index e69de29..3a8a506 100644
--- a/src/buildstream/testing/_sourcetests/__init__.py
+++ b/src/buildstream/testing/_sourcetests/__init__.py
@@ -0,0 +1,28 @@
+#
+# Copyright (C) 2020 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# XXX: we should reorganize those modules after the source tests have
+# been refactored
+# pylint: disable=cyclic-import
+
+from .fetch import FetchSourceTests
+
+__all__ = ["SourceTests"]
+
+
+class SourceTests(FetchSourceTests):
+ """Definition of standardized tests that each source should pass."""
diff --git a/src/buildstream/testing/_sourcetests/base.py b/src/buildstream/testing/_sourcetests/base.py
new file mode 100644
index 0000000..6ee361d
--- /dev/null
+++ b/src/buildstream/testing/_sourcetests/base.py
@@ -0,0 +1,36 @@
+#
+# Copyright (C) 2020 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+from abc import ABC, abstractmethod
+from typing import Type
+
+from ..repo import Repo
+
+
+class BaseSourceTests(ABC):
+ @property
+ @classmethod
+ @abstractmethod
+ def KIND(cls) -> str:
+ """Human readable name of the source currently being tested."""
+
+ @property
+ @classmethod
+ @abstractmethod
+ def REPO(cls) -> Type[Repo]:
+ """Get the repo implementation for the currently tested source."""
diff --git a/src/buildstream/testing/_sourcetests/fetch.py b/src/buildstream/testing/_sourcetests/fetch.py
index 05b43d7..62bbd31 100644
--- a/src/buildstream/testing/_sourcetests/fetch.py
+++ b/src/buildstream/testing/_sourcetests/fetch.py
@@ -1,6 +1,6 @@
#
# Copyright (C) 2018 Codethink Limited
-# Copyright (C) 2019 Bloomberg Finance LP
+# Copyright (C) 2020 Bloomberg Finance LP
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -16,79 +16,67 @@
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
import os
import pytest
from buildstream import _yaml
from .._utils import generate_junction
-from .. import create_repo
-from .. import cli # pylint: disable=unused-import
from .utils import update_project_configuration
-from .utils import kind # pylint: disable=unused-import
-
-
-# Project directory
-TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, "project")
+from .base import BaseSourceTests
-@pytest.mark.datafiles(DATA_DIR)
-def test_fetch(cli, tmpdir, datafiles, kind):
- project = str(datafiles)
- bin_files_path = os.path.join(project, "files", "bin-files")
- element_path = os.path.join(project, "elements")
- element_name = "fetch-test-{}.bst".format(kind)
+class FetchSourceTests(BaseSourceTests):
+ def test_fetch(self, cli, tmpdir, datafiles):
+ project = str(datafiles)
+ bin_files_path = os.path.join(project, "files", "bin-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "fetch-test-{}.bst".format(self.KIND)
- # Create our repo object of the given source type with
- # the bin files, and then collect the initial ref.
- #
- repo = create_repo(kind, str(tmpdir))
- ref = repo.create(bin_files_path)
+ # Create our repo object of the given source type with
+ # the bin files, and then collect the initial ref.
+ #
+ repo = self.REPO(str(tmpdir))
+ ref = repo.create(bin_files_path)
- # Write out our test target
- element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
+ # Write out our test target
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- # Assert that a fetch is needed
- assert cli.get_element_state(project, element_name) == "fetch needed"
+ # Assert that a fetch is needed
+ assert cli.get_element_state(project, element_name) == "fetch needed"
- # Now try to fetch it
- result = cli.run(project=project, args=["source", "fetch", element_name])
- result.assert_success()
+ # Now try to fetch it
+ result = cli.run(project=project, args=["source", "fetch", element_name])
+ result.assert_success()
- # Assert that we are now buildable because the source is
- # now cached.
- assert cli.get_element_state(project, element_name) == "buildable"
+ # Assert that we are now buildable because the source is
+ # now cached.
+ assert cli.get_element_state(project, element_name) == "buildable"
+ @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
+ def test_fetch_cross_junction(self, cli, tmpdir, datafiles, ref_storage):
+ project = str(datafiles)
+ subproject_path = os.path.join(project, "files", "sub-project")
+ junction_path = os.path.join(project, "elements", "junction.bst")
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"])
-def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
- project = str(datafiles)
- subproject_path = os.path.join(project, "files", "sub-project")
- junction_path = os.path.join(project, "elements", "junction.bst")
+ import_etc_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst")
+ etc_files_path = os.path.join(subproject_path, "files", "etc-files")
- import_etc_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst")
- etc_files_path = os.path.join(subproject_path, "files", "etc-files")
+ repo = self.REPO(str(tmpdir.join("import-etc")))
+ ref = repo.create(etc_files_path)
- repo = create_repo(kind, str(tmpdir.join("import-etc")))
- ref = repo.create(etc_files_path)
+ element = {"kind": "import", "sources": [repo.source_config(ref=(ref if ref_storage == "inline" else None))]}
+ _yaml.roundtrip_dump(element, import_etc_path)
- element = {"kind": "import", "sources": [repo.source_config(ref=(ref if ref_storage == "inline" else None))]}
- _yaml.roundtrip_dump(element, import_etc_path)
+ update_project_configuration(project, {"ref-storage": ref_storage})
- update_project_configuration(project, {"ref-storage": ref_storage})
+ generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
- generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline"))
+ if ref_storage == "project.refs":
+ result = cli.run(project=project, args=["source", "track", "junction.bst"])
+ result.assert_success()
+ result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc.bst"])
+ result.assert_success()
- if ref_storage == "project.refs":
- result = cli.run(project=project, args=["source", "track", "junction.bst"])
- result.assert_success()
- result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc.bst"])
+ result = cli.run(project=project, args=["source", "fetch", "junction.bst:import-etc.bst"])
result.assert_success()
-
- result = cli.run(project=project, args=["source", "fetch", "junction.bst:import-etc.bst"])
- result.assert_success()
diff --git a/src/buildstream/testing/_sourcetests/generic.py b/src/buildstream/testing/_sourcetests/generic.py
new file mode 100644
index 0000000..9308e6e
--- /dev/null
+++ b/src/buildstream/testing/_sourcetests/generic.py
@@ -0,0 +1,25 @@
+#
+# Copyright (C) 2020 Bloomberg Finance LP
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+from . import SourceTests
+from .. import ALL_REPO_KINDS
+
+
+for kind, (repo_cls, _) in ALL_REPO_KINDS.items():
+ cls_name = "Test{}Source".format(kind.upper())
+ globals()[cls_name] = type(cls_name, (SourceTests,), {"KIND": cls_name, "REPO": repo_cls})
diff --git a/tests/testutils/repo/bzr.py b/tests/testutils/repo/bzr.py
index b698341..7421be8 100644
--- a/tests/testutils/repo/bzr.py
+++ b/tests/testutils/repo/bzr.py
@@ -7,7 +7,7 @@ from buildstream.testing._utils.site import BZR, BZR_ENV, HAVE_BZR
class Bzr(Repo):
- def __init__(self, directory, subdir):
+ def __init__(self, directory, subdir="repo"):
if not HAVE_BZR:
pytest.skip("bzr is not available")
super().__init__(directory, subdir)
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index 1deca3f..02fe878 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -9,7 +9,7 @@ from buildstream.testing._utils.site import GIT, GIT_ENV, HAVE_GIT
class Git(Repo):
- def __init__(self, directory, subdir):
+ def __init__(self, directory, subdir="repo"):
if not HAVE_GIT:
pytest.skip("git is not available")
[buildstream] 04/15: workspace.py: Simplify invocation of the cli
command
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit a5e6540c23c6004cd6ef8031120afa5b15c480e2
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Fri Oct 9 08:10:28 2020 +0000
workspace.py: Simplify invocation of the cli command
---
src/buildstream/testing/_sourcetests/workspace.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index f99f91a..40dcb6a 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -59,11 +59,7 @@ def test_open(cli, tmpdir, datafiles, kind):
# Now open the workspace, this should have the effect of automatically
# fetching the source from the repo.
- args = ["workspace", "open"]
- args.extend(["--directory", workspace_dir])
-
- args.append(element_name)
- result = cli.run(cwd=workspace_cmd, project=project_path, args=args)
+ result = cli.run(project=project_path, args=["workspace", "open", "--directory", workspace_dir, element_name])
result.assert_success()
[buildstream] 10/15: mirror.py: Adapt to use the standard source
tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 43b4da3f71c7884c7c6689cc0c5f7744c27f92e7
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Sun Aug 23 11:09:06 2020 +0000
mirror.py: Adapt to use the standard source tests
---
src/buildstream/testing/_sourcetests/__init__.py | 3 +-
src/buildstream/testing/_sourcetests/mirror.py | 499 +++++++++++------------
2 files changed, 243 insertions(+), 259 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/__init__.py b/src/buildstream/testing/_sourcetests/__init__.py
index f033c57..f046b02 100644
--- a/src/buildstream/testing/_sourcetests/__init__.py
+++ b/src/buildstream/testing/_sourcetests/__init__.py
@@ -21,9 +21,10 @@
from .build_checkout import BuildCheckoutSourceTests
from .fetch import FetchSourceTests
+from .mirror import MirrorSourceTests
__all__ = ["SourceTests"]
-class SourceTests(BuildCheckoutSourceTests, FetchSourceTests):
+class SourceTests(BuildCheckoutSourceTests, FetchSourceTests, MirrorSourceTests):
"""Definition of standardized tests that each source should pass."""
diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py
index 836ef05..cf69ec9 100644
--- a/src/buildstream/testing/_sourcetests/mirror.py
+++ b/src/buildstream/testing/_sourcetests/mirror.py
@@ -20,18 +20,11 @@
# pylint: disable=redefined-outer-name
import os
-import pytest
from buildstream import _yaml
from buildstream.exceptions import ErrorDomain
from .._utils import generate_junction
-from .. import create_repo
-from .. import cli # pylint: disable=unused-import
-from .utils import kind # pylint: disable=unused-import
-
-# Project directory
-TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, "project")
+from .base import BaseSourceTests
def _set_project_mirrors_and_aliases(project_path, mirrors, aliases):
@@ -54,253 +47,243 @@ def _set_project_includes_and_aliases(project_path, includes, aliases):
_yaml.roundtrip_dump(project_conf, project_conf_path)
-@pytest.mark.datafiles(DATA_DIR)
-def test_mirror_fetch(cli, tmpdir, datafiles, kind):
- project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
- dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
- upstream_repodir = os.path.join(str(tmpdir), "upstream")
- mirror_repodir = os.path.join(str(tmpdir), "mirror")
- element_dir = os.path.join(project_dir, "elements")
-
- # Create repo objects of the upstream and mirror
- upstream_repo = create_repo(kind, upstream_repodir)
- upstream_repo.create(bin_files_path)
- mirror_repo = upstream_repo.copy(mirror_repodir)
- upstream_ref = upstream_repo.create(dev_files_path)
-
- element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
- element_name = "test.bst"
- element_path = os.path.join(element_dir, element_name)
- full_repo = element["sources"][0]["url"]
- upstream_map, repo_name = os.path.split(full_repo)
- alias = "foo-" + kind
- aliased_repo = alias + ":" + repo_name
- element["sources"][0]["url"] = aliased_repo
- full_mirror = mirror_repo.source_config()["url"]
- mirror_map, _ = os.path.split(full_mirror)
- _yaml.roundtrip_dump(element, element_path)
-
- _set_project_mirrors_and_aliases(
- project_dir,
- [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
- {alias: upstream_map + "/"},
- )
-
- # No obvious ways of checking that the mirror has been fetched
- # But at least we can be sure it succeeds
- result = cli.run(project=project_dir, args=["source", "fetch", element_name])
- result.assert_success()
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
- project_dir = str(datafiles)
- dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
- upstream_repodir = os.path.join(project_dir, "upstream")
- mirror_repodir = os.path.join(str(tmpdir), "mirror")
- element_dir = os.path.join(project_dir, "elements")
-
- # Create repo objects of the upstream and mirror
- upstream_repo = create_repo(kind, upstream_repodir)
- ref = upstream_repo.create(dev_files_path)
- mirror_repo = upstream_repo.copy(mirror_repodir)
-
- element = {"kind": "import", "sources": [upstream_repo.source_config(ref=ref)]}
-
- element_name = "test.bst"
- element_path = os.path.join(element_dir, element_name)
- full_repo = element["sources"][0]["url"]
- _, repo_name = os.path.split(full_repo)
- alias = "foo-" + kind
- aliased_repo = alias + ":" + repo_name
- element["sources"][0]["url"] = aliased_repo
- full_mirror = mirror_repo.source_config()["url"]
- mirror_map, _ = os.path.split(full_mirror)
- _yaml.roundtrip_dump(element, element_path)
-
- _set_project_mirrors_and_aliases(
- project_dir,
- [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"]},},],
- {alias: "http://www.example.com"},
- )
-
- result = cli.run(project=project_dir, args=["source", "fetch", element_name])
- result.assert_success()
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
- project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
- upstream_repodir = os.path.join(str(tmpdir), "upstream")
- mirror_repodir = os.path.join(str(tmpdir), "mirror")
- element_dir = os.path.join(project_dir, "elements")
-
- # Create repo objects of the upstream and mirror
- upstream_repo = create_repo(kind, upstream_repodir)
- upstream_ref = upstream_repo.create(bin_files_path)
- mirror_repo = upstream_repo.copy(mirror_repodir)
-
- element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
- element_name = "test.bst"
- element_path = os.path.join(element_dir, element_name)
- full_repo = element["sources"][0]["url"]
- upstream_map, repo_name = os.path.split(full_repo)
- alias = "foo-" + kind
- aliased_repo = alias + ":" + repo_name
- element["sources"][0]["url"] = aliased_repo
- full_mirror = mirror_repo.source_config()["url"]
- mirror_map, _ = os.path.split(full_mirror)
- _yaml.roundtrip_dump(element, element_path)
-
- config_project_dir = str(tmpdir.join("config"))
- os.makedirs(config_project_dir, exist_ok=True)
- config_project = {"name": "config", "min-version": "2.0"}
- _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf"))
- extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]}
- _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml"))
- generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst"))
-
- _set_project_includes_and_aliases(
- project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"},
- )
-
- # Now make the upstream unavailable.
- os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo))
- result = cli.run(project=project_dir, args=["source", "fetch", element_name])
- result.assert_success()
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
- project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
- upstream_repodir = os.path.join(str(tmpdir), "upstream")
- mirror_repodir = os.path.join(str(tmpdir), "mirror")
- element_dir = os.path.join(project_dir, "elements")
-
- # Create repo objects of the upstream and mirror
- upstream_repo = create_repo(kind, upstream_repodir)
- upstream_ref = upstream_repo.create(bin_files_path)
- mirror_repo = upstream_repo.copy(mirror_repodir)
-
- element = {"kind": "junction", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
- element_name = "test.bst"
- element_path = os.path.join(element_dir, element_name)
- full_repo = element["sources"][0]["url"]
- upstream_map, repo_name = os.path.split(full_repo)
- alias = "foo-" + kind
- aliased_repo = alias + ":" + repo_name
- element["sources"][0]["url"] = aliased_repo
- full_mirror = mirror_repo.source_config()["url"]
- mirror_map, _ = os.path.split(full_mirror)
- _yaml.roundtrip_dump(element, element_path)
-
- config_project_dir = str(tmpdir.join("config"))
- os.makedirs(config_project_dir, exist_ok=True)
- config_project = {"name": "config", "min-version": "2.0"}
- _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf"))
- extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]}
- _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml"))
- generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst"))
-
- _set_project_includes_and_aliases(project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"})
-
- # Now make the upstream unavailable.
- os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo))
- result = cli.run(project=project_dir, args=["source", "fetch", element_name])
- result.assert_main_error(ErrorDomain.STREAM, None)
- # Now make the upstream available again.
- os.rename("{}.bak".format(upstream_repo.repo), upstream_repo.repo)
- result = cli.run(project=project_dir, args=["source", "fetch", element_name])
- result.assert_success()
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
- project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
- dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
- upstream_repodir = os.path.join(str(tmpdir), "upstream")
- mirror_repodir = os.path.join(str(tmpdir), "mirror")
- element_dir = os.path.join(project_dir, "elements")
-
- # Create repo objects of the upstream and mirror
- upstream_repo = create_repo(kind, upstream_repodir)
- upstream_repo.create(bin_files_path)
- mirror_repo = upstream_repo.copy(mirror_repodir)
- upstream_ref = upstream_repo.create(dev_files_path)
-
- element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
-
- element_name = "test.bst"
- element_path = os.path.join(element_dir, element_name)
- full_repo = element["sources"][0]["url"]
- upstream_map, repo_name = os.path.split(full_repo)
- alias = "foo-" + kind
- aliased_repo = alias + ":" + repo_name
- element["sources"][0]["url"] = aliased_repo
- full_mirror = mirror_repo.source_config()["url"]
- mirror_map, _ = os.path.split(full_mirror)
- _yaml.roundtrip_dump(element, element_path)
-
- _set_project_mirrors_and_aliases(
- project_dir,
- [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
- {alias: upstream_map + "/"},
- )
-
- result = cli.run(project=project_dir, args=["source", "track", element_name])
- result.assert_success()
-
- # Tracking tries upstream first. Check the ref is from upstream.
- new_element = _yaml.load(element_path, shortname=element_name)
- source = new_element.get_sequence("sources").mapping_at(0)
- if "ref" in source:
- assert source.get_str("ref") == upstream_ref
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
- project_dir = str(datafiles)
- bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
- dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
- upstream_repodir = os.path.join(str(tmpdir), "upstream")
- mirror_repodir = os.path.join(str(tmpdir), "mirror")
- element_dir = os.path.join(project_dir, "elements")
-
- # Create repo objects of the upstream and mirror
- upstream_repo = create_repo(kind, upstream_repodir)
- upstream_ref = upstream_repo.create(bin_files_path)
- mirror_repo = upstream_repo.copy(mirror_repodir)
- mirror_ref = upstream_ref
- upstream_ref = upstream_repo.create(dev_files_path)
-
- element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
-
- element_name = "test.bst"
- element_path = os.path.join(element_dir, element_name)
- full_repo = element["sources"][0]["url"]
- _, repo_name = os.path.split(full_repo)
- alias = "foo-" + kind
- aliased_repo = alias + ":" + repo_name
- element["sources"][0]["url"] = aliased_repo
- full_mirror = mirror_repo.source_config()["url"]
- mirror_map, _ = os.path.split(full_mirror)
- _yaml.roundtrip_dump(element, element_path)
-
- _set_project_mirrors_and_aliases(
- project_dir,
- [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
- {alias: "http://www.example.com"},
- )
-
- result = cli.run(project=project_dir, args=["source", "track", element_name])
- result.assert_success()
-
- # Check that tracking fell back to the mirror
- new_element = _yaml.load(element_path, shortname=element_name)
- source = new_element.get_sequence("sources").mapping_at(0)
- if "ref" in source:
- assert source.get_str("ref") == mirror_ref
+class MirrorSourceTests(BaseSourceTests):
+ def test_mirror_fetch(self, cli, tmpdir, datafiles):
+ project_dir = str(datafiles)
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = self.REPO(upstream_repodir)
+ upstream_repo.create(bin_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+ upstream_ref = upstream_repo.create(dev_files_path)
+
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element["sources"][0]["url"]
+ upstream_map, repo_name = os.path.split(full_repo)
+ alias = "foo-" + self.KIND
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
+ mirror_map, _ = os.path.split(full_mirror)
+ _yaml.roundtrip_dump(element, element_path)
+
+ _set_project_mirrors_and_aliases(
+ project_dir,
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
+ {alias: upstream_map + "/"},
+ )
+
+ # No obvious ways of checking that the mirror has been fetched
+ # But at least we can be sure it succeeds
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
+ result.assert_success()
+
+ def test_mirror_fetch_upstream_absent(self, cli, tmpdir, datafiles):
+ project_dir = str(datafiles)
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(project_dir, "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = self.REPO(upstream_repodir)
+ ref = upstream_repo.create(dev_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=ref)]}
+
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element["sources"][0]["url"]
+ _, repo_name = os.path.split(full_repo)
+ alias = "foo-" + self.KIND
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
+ mirror_map, _ = os.path.split(full_mirror)
+ _yaml.roundtrip_dump(element, element_path)
+
+ _set_project_mirrors_and_aliases(
+ project_dir,
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"]},},],
+ {alias: "http://www.example.com"},
+ )
+
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
+ result.assert_success()
+
+ def test_mirror_from_includes(self, cli, tmpdir, datafiles):
+ project_dir = str(datafiles)
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = self.REPO(upstream_repodir)
+ upstream_ref = upstream_repo.create(bin_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element["sources"][0]["url"]
+ upstream_map, repo_name = os.path.split(full_repo)
+ alias = "foo-" + self.KIND
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
+ mirror_map, _ = os.path.split(full_mirror)
+ _yaml.roundtrip_dump(element, element_path)
+
+ config_project_dir = str(tmpdir.join("config"))
+ os.makedirs(config_project_dir, exist_ok=True)
+ config_project = {"name": "config", "min-version": "2.0"}
+ _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf"))
+ extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]}
+ _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml"))
+ generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst"))
+
+ _set_project_includes_and_aliases(
+ project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"},
+ )
+
+ # Now make the upstream unavailable.
+ os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo))
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
+ result.assert_success()
+
+ def test_mirror_junction_from_includes(self, cli, tmpdir, datafiles):
+ project_dir = str(datafiles)
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = self.REPO(upstream_repodir)
+ upstream_ref = upstream_repo.create(bin_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+
+ element = {"kind": "junction", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element["sources"][0]["url"]
+ upstream_map, repo_name = os.path.split(full_repo)
+ alias = "foo-" + self.KIND
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
+ mirror_map, _ = os.path.split(full_mirror)
+ _yaml.roundtrip_dump(element, element_path)
+
+ config_project_dir = str(tmpdir.join("config"))
+ os.makedirs(config_project_dir, exist_ok=True)
+ config_project = {"name": "config", "min-version": "2.0"}
+ _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf"))
+ extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]}
+ _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml"))
+ generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst"))
+
+ _set_project_includes_and_aliases(project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"})
+
+ # Now make the upstream unavailable.
+ os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo))
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
+ result.assert_main_error(ErrorDomain.STREAM, None)
+ # Now make the upstream available again.
+ os.rename("{}.bak".format(upstream_repo.repo), upstream_repo.repo)
+ result = cli.run(project=project_dir, args=["source", "fetch", element_name])
+ result.assert_success()
+
+ def test_mirror_track_upstream_present(self, cli, tmpdir, datafiles):
+ project_dir = str(datafiles)
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = self.REPO(upstream_repodir)
+ upstream_repo.create(bin_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+ upstream_ref = upstream_repo.create(dev_files_path)
+
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element["sources"][0]["url"]
+ upstream_map, repo_name = os.path.split(full_repo)
+ alias = "foo-" + self.KIND
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
+ mirror_map, _ = os.path.split(full_mirror)
+ _yaml.roundtrip_dump(element, element_path)
+
+ _set_project_mirrors_and_aliases(
+ project_dir,
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
+ {alias: upstream_map + "/"},
+ )
+
+ result = cli.run(project=project_dir, args=["source", "track", element_name])
+ result.assert_success()
+
+ # Tracking tries upstream first. Check the ref is from upstream.
+ new_element = _yaml.load(element_path, shortname=element_name)
+ source = new_element.get_sequence("sources").mapping_at(0)
+ if "ref" in source:
+ assert source.get_str("ref") == upstream_ref
+
+ def test_mirror_track_upstream_absent(self, cli, tmpdir, datafiles):
+ project_dir = str(datafiles)
+ bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr")
+ dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr")
+ upstream_repodir = os.path.join(str(tmpdir), "upstream")
+ mirror_repodir = os.path.join(str(tmpdir), "mirror")
+ element_dir = os.path.join(project_dir, "elements")
+
+ # Create repo objects of the upstream and mirror
+ upstream_repo = self.REPO(upstream_repodir)
+ upstream_ref = upstream_repo.create(bin_files_path)
+ mirror_repo = upstream_repo.copy(mirror_repodir)
+ mirror_ref = upstream_ref
+ upstream_ref = upstream_repo.create(dev_files_path)
+
+ element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]}
+
+ element_name = "test.bst"
+ element_path = os.path.join(element_dir, element_name)
+ full_repo = element["sources"][0]["url"]
+ _, repo_name = os.path.split(full_repo)
+ alias = "foo-" + self.KIND
+ aliased_repo = alias + ":" + repo_name
+ element["sources"][0]["url"] = aliased_repo
+ full_mirror = mirror_repo.source_config()["url"]
+ mirror_map, _ = os.path.split(full_mirror)
+ _yaml.roundtrip_dump(element, element_path)
+
+ _set_project_mirrors_and_aliases(
+ project_dir,
+ [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},],
+ {alias: "http://www.example.com"},
+ )
+
+ result = cli.run(project=project_dir, args=["source", "track", element_name])
+ result.assert_success()
+
+ # Check that tracking fell back to the mirror
+ new_element = _yaml.load(element_path, shortname=element_name)
+ source = new_element.get_sequence("sources").mapping_at(0)
+ if "ref" in source:
+ assert source.get_str("ref") == mirror_ref
[buildstream] 14/15: workspace.py: Adapt to use the standard source
tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 567a9be1a29cd528a8eabc6471566a09a3bbd853
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Fri Oct 9 12:07:17 2020 +0000
workspace.py: Adapt to use the standard source tests
---
src/buildstream/testing/_sourcetests/__init__.py | 2 +
src/buildstream/testing/_sourcetests/workspace.py | 54 +++++++++++------------
2 files changed, 29 insertions(+), 27 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/__init__.py b/src/buildstream/testing/_sourcetests/__init__.py
index e0b2e41..9c42e85 100644
--- a/src/buildstream/testing/_sourcetests/__init__.py
+++ b/src/buildstream/testing/_sourcetests/__init__.py
@@ -25,6 +25,7 @@ from .mirror import MirrorSourceTests
from .source_determinism import SourceDeterminismTests
from .track_cross_junction import TrackCrossJunctionTests
from .track import TrackSourceTests
+from .workspace import WorkspaceSourceTests
__all__ = ["SourceTests"]
@@ -36,5 +37,6 @@ class SourceTests(
SourceDeterminismTests,
TrackCrossJunctionTests,
TrackSourceTests,
+ WorkspaceSourceTests,
):
"""Definition of standardized tests that each source should pass."""
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index 9edd9d9..aac4c51 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -23,46 +23,46 @@ import os
import pytest
from buildstream import _yaml
-from .. import create_repo
from .. import cli # pylint: disable=unused-import
-from .utils import kind # pylint: disable=unused-import
+from .base import BaseSourceTests
# Project directory
TOP_DIR = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(TOP_DIR, "project")
-@pytest.mark.datafiles(DATA_DIR)
-def test_open(cli, tmpdir_factory, datafiles, kind):
- project_path = str(datafiles)
- bin_files_path = os.path.join(project_path, "files", "bin-files")
+class WorkspaceSourceTests(BaseSourceTests):
+ @pytest.mark.datafiles(DATA_DIR)
+ def test_open(self, cli, tmpdir_factory, datafiles):
+ project_path = str(datafiles)
+ bin_files_path = os.path.join(project_path, "files", "bin-files")
- element_name = "workspace-test-{}.bst".format(kind)
- element_path = os.path.join(project_path, "elements")
+ element_name = "workspace-test-{}.bst".format(self.KIND)
+ element_path = os.path.join(project_path, "elements")
- # Create our repo object of the given source type with
- # the bin files, and then collect the initial ref.
- repo = create_repo(kind, str(tmpdir_factory.mktemp("repo-{}".format(kind))))
- ref = repo.create(bin_files_path)
+ # Create our repo object of the given source type with
+ # the bin files, and then collect the initial ref.
+ repo = self.REPO(str(tmpdir_factory.mktemp("repo-{}".format(self.KIND))))
+ ref = repo.create(bin_files_path)
- # Write out our test target
- element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
+ # Write out our test target
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- # Assert that there is no reference, a fetch is needed
- assert cli.get_element_state(project_path, element_name) == "fetch needed"
+ # Assert that there is no reference, a fetch is needed
+ assert cli.get_element_state(project_path, element_name) == "fetch needed"
- workspace_dir = os.path.join(tmpdir_factory.mktemp("opened_workspace"))
+ workspace_dir = os.path.join(tmpdir_factory.mktemp("opened_workspace"))
- # Now open the workspace, this should have the effect of automatically
- # fetching the source from the repo.
- result = cli.run(project=project_path, args=["workspace", "open", "--directory", workspace_dir, element_name])
+ # Now open the workspace, this should have the effect of automatically
+ # fetching the source from the repo.
+ result = cli.run(project=project_path, args=["workspace", "open", "--directory", workspace_dir, element_name])
- result.assert_success()
+ result.assert_success()
- # Assert that we are now buildable because the source is now cached.
- assert cli.get_element_state(project_path, element_name) == "buildable"
+ # Assert that we are now buildable because the source is now cached.
+ assert cli.get_element_state(project_path, element_name) == "buildable"
- # Check that the executable hello file is found in each workspace
- filename = os.path.join(workspace_dir, "usr", "bin", "hello")
- assert os.path.exists(filename)
+ # Check that the executable hello file is found in each workspace
+ filename = os.path.join(workspace_dir, "usr", "bin", "hello")
+ assert os.path.exists(filename)
[buildstream] 03/15: workspace.py: Remove some abstractions and
write the whole test in place
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 22b6591950f79270e2a28d1930f2c83e56d87836
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Fri Oct 9 07:59:23 2020 +0000
workspace.py: Remove some abstractions and write the whole test in place
This is to remove some hoops that are currently used, that do not fit
well with most of the style in the code. It also removes indirections
and make the test easier to understand
---
src/buildstream/testing/_sourcetests/workspace.py | 77 +++++++++--------------
1 file changed, 31 insertions(+), 46 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index 881e85b..f99f91a 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -32,59 +32,44 @@ TOP_DIR = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = os.path.join(TOP_DIR, "project")
-class WorkspaceCreator:
- def __init__(self, cli, tmpdir, datafiles):
- self.cli = cli
- self.tmpdir = tmpdir
-
- self.project_path = str(datafiles)
- self.bin_files_path = os.path.join(self.project_path, "files", "bin-files")
-
- self.workspace_cmd = os.path.join(self.project_path, "workspace_cmd")
-
- def create_workspace_element(self, kind):
- element_name = "workspace-test-{}.bst".format(kind)
- element_path = os.path.join(self.project_path, "elements")
- # remove the '.bst' at the end of the element
- workspace_dir = os.path.join(self.workspace_cmd, element_name[-4:])
-
- # Create our repo object of the given source type with
- # the bin files, and then collect the initial ref.
- repo = create_repo(kind, str(self.tmpdir))
- ref = repo.create(self.bin_files_path)
-
- # Write out our test target
- element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
+@pytest.mark.datafiles(DATA_DIR)
+def test_open(cli, tmpdir, datafiles, kind):
+ project_path = str(datafiles)
+ bin_files_path = os.path.join(project_path, "files", "bin-files")
- # Assert that there is no reference, a fetch is needed
- assert self.cli.get_element_state(self.project_path, element_name) == "fetch needed"
- return element_name, workspace_dir
+ element_name = "workspace-test-{}.bst".format(kind)
+ element_path = os.path.join(project_path, "elements")
- def open_workspace(self, kind):
+ # Create our repo object of the given source type with
+ # the bin files, and then collect the initial ref.
+ repo = create_repo(kind, str(tmpdir))
+ ref = repo.create(bin_files_path)
- element_name, workspace_dir = self.create_workspace_element(kind)
- os.makedirs(self.workspace_cmd, exist_ok=True)
+ # Write out our test target
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- # Now open the workspace, this should have the effect of automatically
- # fetching the source from the repo.
- args = ["workspace", "open"]
- args.extend(["--directory", workspace_dir])
+ # Assert that there is no reference, a fetch is needed
+ assert cli.get_element_state(project_path, element_name) == "fetch needed"
- args.append(element_name)
- result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
+ workspace_cmd = os.path.join(project_path, "workspace_cmd")
+ os.makedirs(workspace_cmd, exist_ok=True)
+ # remove the '.bst' at the end of the element
+ workspace_dir = os.path.join(workspace_cmd, element_name[-4:])
- result.assert_success()
+ # Now open the workspace, this should have the effect of automatically
+ # fetching the source from the repo.
+ args = ["workspace", "open"]
+ args.extend(["--directory", workspace_dir])
- # Assert that we are now buildable because the source is now cached.
- assert self.cli.get_element_state(self.project_path, element_name) == "buildable"
+ args.append(element_name)
+ result = cli.run(cwd=workspace_cmd, project=project_path, args=args)
- # Check that the executable hello file is found in each workspace
- filename = os.path.join(workspace_dir, "usr", "bin", "hello")
- assert os.path.exists(filename)
+ result.assert_success()
+ # Assert that we are now buildable because the source is now cached.
+ assert cli.get_element_state(project_path, element_name) == "buildable"
-@pytest.mark.datafiles(DATA_DIR)
-def test_open(cli, tmpdir, datafiles, kind):
- workspace_object = WorkspaceCreator(cli, tmpdir, datafiles)
- workspace_object.open_workspace(kind)
+ # Check that the executable hello file is found in each workspace
+ filename = os.path.join(workspace_dir, "usr", "bin", "hello")
+ assert os.path.exists(filename)
[buildstream] 05/15: workspace.py: Don't use the same directory for
everything
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit e8280a90b166887a199bd3b4c82a2e836537bcf4
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Fri Oct 9 10:53:24 2020 +0000
workspace.py: Don't use the same directory for everything
---
src/buildstream/testing/_sourcetests/workspace.py | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index 40dcb6a..9edd9d9 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -33,7 +33,7 @@ DATA_DIR = os.path.join(TOP_DIR, "project")
@pytest.mark.datafiles(DATA_DIR)
-def test_open(cli, tmpdir, datafiles, kind):
+def test_open(cli, tmpdir_factory, datafiles, kind):
project_path = str(datafiles)
bin_files_path = os.path.join(project_path, "files", "bin-files")
@@ -42,7 +42,7 @@ def test_open(cli, tmpdir, datafiles, kind):
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
- repo = create_repo(kind, str(tmpdir))
+ repo = create_repo(kind, str(tmpdir_factory.mktemp("repo-{}".format(kind))))
ref = repo.create(bin_files_path)
# Write out our test target
@@ -52,10 +52,7 @@ def test_open(cli, tmpdir, datafiles, kind):
# Assert that there is no reference, a fetch is needed
assert cli.get_element_state(project_path, element_name) == "fetch needed"
- workspace_cmd = os.path.join(project_path, "workspace_cmd")
- os.makedirs(workspace_cmd, exist_ok=True)
- # remove the '.bst' at the end of the element
- workspace_dir = os.path.join(workspace_cmd, element_name[-4:])
+ workspace_dir = os.path.join(tmpdir_factory.mktemp("opened_workspace"))
# Now open the workspace, this should have the effect of automatically
# fetching the source from the repo.
[buildstream] 09/15: build_checkout.py: Adapt to use the standard
source tests
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 984b44e6f856773755c9fc8f5e25ddf1e26c503f
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Sun Aug 23 10:28:02 2020 +0000
build_checkout.py: Adapt to use the standard source tests
---
src/buildstream/testing/_sourcetests/__init__.py | 3 +-
.../testing/_sourcetests/build_checkout.py | 64 ++++++++++------------
2 files changed, 31 insertions(+), 36 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/__init__.py b/src/buildstream/testing/_sourcetests/__init__.py
index 3a8a506..f033c57 100644
--- a/src/buildstream/testing/_sourcetests/__init__.py
+++ b/src/buildstream/testing/_sourcetests/__init__.py
@@ -19,10 +19,11 @@
# been refactored
# pylint: disable=cyclic-import
+from .build_checkout import BuildCheckoutSourceTests
from .fetch import FetchSourceTests
__all__ = ["SourceTests"]
-class SourceTests(FetchSourceTests):
+class SourceTests(BuildCheckoutSourceTests, FetchSourceTests):
"""Definition of standardized tests that each source should pass."""
diff --git a/src/buildstream/testing/_sourcetests/build_checkout.py b/src/buildstream/testing/_sourcetests/build_checkout.py
index 782d998..8209fce 100644
--- a/src/buildstream/testing/_sourcetests/build_checkout.py
+++ b/src/buildstream/testing/_sourcetests/build_checkout.py
@@ -22,14 +22,8 @@
import os
import pytest
-from buildstream.testing import create_repo
-from buildstream.testing import cli # pylint: disable=unused-import
from buildstream import _yaml
-from .utils import kind # pylint: disable=unused-import
-
-# Project directory
-TOP_DIR = os.path.dirname(os.path.realpath(__file__))
-DATA_DIR = os.path.join(TOP_DIR, "project")
+from .base import BaseSourceTests
def strict_args(args, strict):
@@ -38,36 +32,36 @@ def strict_args(args, strict):
return args
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("strict", ["strict", "non-strict"])
-def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind):
- checkout = os.path.join(cli.directory, "checkout")
- project = str(datafiles)
- dev_files_path = os.path.join(project, "files", "dev-files")
- element_path = os.path.join(project, "elements")
- element_name = "build-test-{}.bst".format(kind)
+class BuildCheckoutSourceTests(BaseSourceTests):
+ @pytest.mark.parametrize("strict", ["strict", "non-strict"])
+ def test_fetch_build_checkout(self, cli, tmpdir, datafiles, strict):
+ checkout = os.path.join(cli.directory, "checkout")
+ project = str(datafiles)
+ dev_files_path = os.path.join(project, "files", "dev-files")
+ element_path = os.path.join(project, "elements")
+ element_name = "build-test-{}.bst".format(self.KIND)
- # Create our repo object of the given source type with
- # the dev files, and then collect the initial ref.
- #
- repo = create_repo(kind, str(tmpdir))
- ref = repo.create(dev_files_path)
+ # Create our repo object of the given source type with
+ # the dev files, and then collect the initial ref.
+ #
+ repo = self.REPO(str(tmpdir))
+ ref = repo.create(dev_files_path)
- # Write out our test target
- element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
+ # Write out our test target
+ element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
- assert cli.get_element_state(project, element_name) == "fetch needed"
- result = cli.run(project=project, args=strict_args(["build", element_name], strict))
- result.assert_success()
- assert cli.get_element_state(project, element_name) == "cached"
+ assert cli.get_element_state(project, element_name) == "fetch needed"
+ result = cli.run(project=project, args=strict_args(["build", element_name], strict))
+ result.assert_success()
+ assert cli.get_element_state(project, element_name) == "cached"
- # Now check it out
- result = cli.run(
- project=project, args=strict_args(["artifact", "checkout", element_name, "--directory", checkout], strict)
- )
- result.assert_success()
+ # Now check it out
+ result = cli.run(
+ project=project, args=strict_args(["artifact", "checkout", element_name, "--directory", checkout], strict)
+ )
+ result.assert_success()
- # Check that the pony.h include from files/dev-files exists
- filename = os.path.join(checkout, "usr", "include", "pony.h")
- assert os.path.exists(filename)
+ # Check that the pony.h include from files/dev-files exists
+ filename = os.path.join(checkout, "usr", "include", "pony.h")
+ assert os.path.exists(filename)
[buildstream] 02/15: workspace.py: Remove arguments that are unused
during the test
Posted by no...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
not-in-ldap pushed a commit to branch bschubert/standardize-source-tests
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 14a136e8e1311b9b6109f20c0cf96d96a2886c8b
Author: Benjamin Schubert <co...@benschubert.me>
AuthorDate: Thu Oct 8 21:30:06 2020 +0000
workspace.py: Remove arguments that are unused during the test
---
src/buildstream/testing/_sourcetests/workspace.py | 46 ++++++++---------------
1 file changed, 15 insertions(+), 31 deletions(-)
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index 7f3c5de..881e85b 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -20,7 +20,6 @@
# pylint: disable=redefined-outer-name
import os
-import shutil
import pytest
from buildstream import _yaml
@@ -34,28 +33,20 @@ DATA_DIR = os.path.join(TOP_DIR, "project")
class WorkspaceCreator:
- def __init__(self, cli, tmpdir, datafiles, project_path=None):
+ def __init__(self, cli, tmpdir, datafiles):
self.cli = cli
self.tmpdir = tmpdir
- self.datafiles = datafiles
- if not project_path:
- project_path = str(datafiles)
- else:
- shutil.copytree(str(datafiles), project_path)
-
- self.project_path = project_path
- self.bin_files_path = os.path.join(project_path, "files", "bin-files")
+ self.project_path = str(datafiles)
+ self.bin_files_path = os.path.join(self.project_path, "files", "bin-files")
self.workspace_cmd = os.path.join(self.project_path, "workspace_cmd")
- def create_workspace_element(self, kind, suffix="", workspace_dir=None, element_attrs=None):
- element_name = "workspace-test-{}{}.bst".format(kind, suffix)
+ def create_workspace_element(self, kind):
+ element_name = "workspace-test-{}.bst".format(kind)
element_path = os.path.join(self.project_path, "elements")
- if not workspace_dir:
- workspace_dir = os.path.join(self.workspace_cmd, element_name)
- if workspace_dir[-4:] == ".bst":
- workspace_dir = workspace_dir[:-4]
+ # remove the '.bst' at the end of the element
+ workspace_dir = os.path.join(self.workspace_cmd, element_name[-4:])
# Create our repo object of the given source type with
# the bin files, and then collect the initial ref.
@@ -64,40 +55,33 @@ class WorkspaceCreator:
# Write out our test target
element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
- if element_attrs:
- element = {**element, **element_attrs}
_yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
# Assert that there is no reference, a fetch is needed
assert self.cli.get_element_state(self.project_path, element_name) == "fetch needed"
return element_name, workspace_dir
- def open_workspace(self, kind, suffix=None, workspace_dir=None, element_attrs=None, no_checkout=False):
+ def open_workspace(self, kind):
- element_name, workspace_dir = self.create_workspace_element(kind, suffix, workspace_dir, element_attrs)
+ element_name, workspace_dir = self.create_workspace_element(kind)
os.makedirs(self.workspace_cmd, exist_ok=True)
# Now open the workspace, this should have the effect of automatically
# fetching the source from the repo.
args = ["workspace", "open"]
-
- if no_checkout:
- args.append("--no-checkout")
- if workspace_dir is not None:
- args.extend(["--directory", workspace_dir])
+ args.extend(["--directory", workspace_dir])
args.append(element_name)
result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
result.assert_success()
- if not no_checkout:
- # Assert that we are now buildable because the source is now cached.
- assert self.cli.get_element_state(self.project_path, element_name) == "buildable"
+ # Assert that we are now buildable because the source is now cached.
+ assert self.cli.get_element_state(self.project_path, element_name) == "buildable"
- # Check that the executable hello file is found in each workspace
- filename = os.path.join(workspace_dir, "usr", "bin", "hello")
- assert os.path.exists(filename)
+ # Check that the executable hello file is found in each workspace
+ filename = os.path.join(workspace_dir, "usr", "bin", "hello")
+ assert os.path.exists(filename)
@pytest.mark.datafiles(DATA_DIR)