You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@sdap.apache.org by GitBox <gi...@apache.org> on 2020/06/18 00:24:47 UTC
[GitHub] [incubator-sdap-ingester] eamonford commented on a change in pull request #1: SDAP-234: Test suite
eamonford commented on a change in pull request #1:
URL: https://github.com/apache/incubator-sdap-ingester/pull/1#discussion_r441902633
##########
File path: collection_manager/tests/services/test_CollectionWatcher.py
##########
@@ -0,0 +1,203 @@
+import os
+import tempfile
+import time
+import unittest
+from datetime import datetime
+from unittest.mock import Mock
+
+from collection_manager.entities import Collection
+from collection_manager.entities.exceptions import YamlParsingError, CollectionConfigFileNotFoundError, \
+ RelativePathCollectionError, ConflictingPathCollectionError
+from collection_manager.services import CollectionWatcher
+
+
+class TestCollectionWatcher(unittest.TestCase):
+
+ def test_collections_returns_all_collections(self):
+ collection_watcher = CollectionWatcher('/foo', Mock(), Mock())
+ collection_watcher._collections_by_dir = {
+ "/foo": {
+ Collection("id1", "var1", "path1", 1, 2, datetime.now(), datetime.now()),
+ Collection("id2", "var2", "path2", 3, 4, datetime.now(), datetime.now()),
+ },
+ "/bar": {
+ Collection("id3", "var3", "path3", 5, 6, datetime.now(), datetime.now()),
+ Collection("id4", "var4", "path4", 7, 8, datetime.now(), datetime.now()),
+ }
+ }
+ flattened_collections = collection_watcher.collections()
+ self.assertEqual(len(flattened_collections), 4)
+
+ def test_load_collections_loads_all_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+ collection_watcher._load_collections()
+
+ self.assertEquals(len(collection_watcher._collections_by_dir), 2)
+ self.assertEquals(len(collection_watcher._collections_by_dir['/opt/data/grace']), 2)
+ self.assertEquals(len(collection_watcher._collections_by_dir['/opt/data/avhrr']), 1)
+
+ def test_load_collections_with_bad_yaml_syntax(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections_bad.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+
+ self.assertRaises(YamlParsingError, collection_watcher._load_collections)
+
+ def test_load_collections_with_file_not_found(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/does_not_exist.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+
+ self.assertRaises(CollectionConfigFileNotFoundError, collection_watcher._load_collections)
+
+ def test_get_updated_collections_returns_all_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+
+ updated_collections = collection_watcher._get_updated_collections()
+ self.assertSetEqual(updated_collections, collection_watcher.collections())
+
+ def test_get_updated_collections_returns_no_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+ collection_watcher._load_collections()
+ updated_collections = collection_watcher._get_updated_collections()
+
+ self.assertEquals(len(updated_collections), 0)
+
+ def test_get_updated_collections_returns_some_collections(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+ collection_watcher._load_collections()
+
+ new_collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections_alternate.yml')
+ collection_watcher._collections_path = new_collections_path
+ updated_collections = collection_watcher._get_updated_collections()
+
+ self.assertEquals(len(updated_collections), 1)
+
+ def test_validate_collection(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+
+ collection = Collection(dataset_id="test_dataset",
+ path="/absolute/path",
+ variable="test_variable",
+ historical_priority=1,
+ forward_processing_priority=2,
+ date_from=None,
+ date_to=None)
+ collection_watcher._validate_collection(collection)
+
+ def test_validate_collection_with_relative_path(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '../resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+
+ collection = Collection(dataset_id="test_dataset",
+ path="relative/path",
+ variable="test_variable",
+ historical_priority=1,
+ forward_processing_priority=2,
+ date_from=None,
+ date_to=None)
+ self.assertRaises(RelativePathCollectionError, collection_watcher._validate_collection, collection)
+
+ def test_validate_collection_with_conflicting_path(self):
+ collections_path = os.path.join(os.path.dirname(__file__), '/resources/collections.yml')
+ collection_watcher = CollectionWatcher(collections_path, Mock(), Mock())
+
+ collection = Collection(dataset_id="test_dataset",
+ path="/resources/*.nc",
+ variable="test_variable",
+ historical_priority=1,
+ forward_processing_priority=2,
+ date_from=None,
+ date_to=None)
+ self.assertRaises(ConflictingPathCollectionError, collection_watcher._validate_collection, collection)
+
+ def test_collection_callback_is_called(self):
Review comment:
That failure is a bit sporadic, as this test is checking that an asynchronous callback was called. Unfortunately, I'm not able to reproduce that error on my end anymore, it succeeds every time for me -- could you try running it again and see if it still fails for you? Thanks!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org