[buildstream] 09/16: tests: Add a test for fetching when a source uses multiple aliases

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a commit to branch jonathan/mirror-client
in repository https://gitbox.apache.org/repos/asf/buildstream.git

commit d41bbc5c68c225c4c821d8ce8be3dff38476f049
Author: Jonathan Maw <jo...@codethink.co.uk>
AuthorDate: Thu May 3 17:30:50 2018 +0100

    tests: Add a test for fetching when a source uses multiple aliases
---
 tests/frontend/mirror.py                       | 63 ++++++++++++++++++++++
 tests/frontend/project/sources/fetch_source.py | 73 ++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)

diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index b45aa29..9fdbd38 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -68,3 +68,66 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
     # But at least we can be sure it succeeds
     result = cli.run(project=project_dir, args=['fetch', element_name])
     result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_multi(cli, tmpdir, datafiles):
+    output_file = os.path.join(str(tmpdir), "output.txt")
+    project_dir = str(tmpdir)
+    element_dir = os.path.join(project_dir, 'elements')
+    os.makedirs(element_dir, exist_ok=True)
+    element_name = "test.bst"
+    element_path = os.path.join(element_dir, element_name)
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'fetch_source',
+                "output-text": output_file,
+                "urls": ["foo:repo1", "bar:repo2"],
+                "fetch-succeeds": {
+                    "FOO/repo1": True,
+                    "BAR/repo2": False,
+                    "OOF/repo1": False,
+                    "RAB/repo2": True
+                }
+            }
+        ]
+    }
+    _yaml.dump(element, element_path)
+
+    project_file = os.path.join(project_dir, 'project.conf')
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            "foo": "FOO/",
+            "bar": "BAR/"
+        },
+        'mirrors': [
+            {
+                'location-name': 'middle-earth',
+                'aliases': {
+                    "foo": ["OOF/"],
+                    "bar": ["RAB/"]
+                },
+            },
+        ],
+        'plugins': [
+            {
+                'origin': 'local',
+                'path': 'sources',
+                'sources': {
+                    'fetch_source': 0
+                }
+            }
+        ]
+    }
+    _yaml.dump(project, project_file)
+
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+    with open(output_file) as f:
+        contents = f.read()
+        assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents
+        assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents
diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py
new file mode 100644
index 0000000..81b9777
--- /dev/null
+++ b/tests/frontend/project/sources/fetch_source.py
@@ -0,0 +1,73 @@
+import os
+import sys
+
+from buildstream import Source, Consistency, SourceError
+
+# Expected config
+# sources:
+# - output-text: $FILE
+#   urls:
+#   - foo:bar
+#   - baz:quux
+#   fetch-succeeds:
+#     Foo/bar: true
+#     ooF/bar: false
+
+
+class FetchSource(Source):
+    # Read config to know which URLs to fetch
+    def configure(self, node):
+        self.original_urls = self.node_get_member(node, list, 'urls')
+        self.urls = [self.translate_url(url) for url in self.original_urls]
+        self.output_file = self.node_get_member(node, str, 'output-text')
+        self.fetch_succeeds = {}
+        if 'fetch-succeeds' in node:
+            self.fetch_succeeds = {x[0]: x[1] for x in self.node_items(node['fetch-succeeds'])}
+        self.urls_cached = False
+
+    def preflight(self):
+        output_dir = os.path.dirname(self.output_file)
+        if not os.path.exists(output_dir):
+            raise SourceError("Directory '{}' does not exist".format(output_dir))
+
+    def get_unique_key(self):
+        return {"urls": self.original_urls, "output_file": self.output_file}
+
+    def get_consistency(self):
+        if not os.path.exists(self.output_file):
+            return Consistency.RESOLVED
+
+        all_fetched = True
+        with open(self.output_file, "r") as f:
+            contents = f.read()
+            for url in self.original_urls:
+                if url not in contents:
+                    return Consistency.RESOLVED
+
+        return Consistency.CACHED
+
+    # We dont have a ref, we're a local file...
+    def load_ref(self, node):
+        pass
+
+    def get_ref(self):
+        return None  # pragma: nocover
+
+    def set_ref(self, ref, node):
+        pass  # pragma: nocover
+
+    def fetch(self):
+        with open(self.output_file, "a") as f:
+            for i, url in enumerate(self.urls):
+                origin_url = self.original_urls[i]
+                success = url in self.fetch_succeeds and self.fetch_succeeds[url]
+                message = "Fetch {} {} from {}\n".format(origin_url,
+                                                         "succeeded" if success else "failed",
+                                                         url)
+                f.write(message)
+                if not success:
+                    raise SourceError("Failed to fetch {}".format(url))
+
+
+def setup():
+    return FetchSource