You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/08/11 03:35:19 UTC
[incubator-sdap-ingester] branch async-history updated: fix tests
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch async-history
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/async-history by this push:
new 5139bc2 fix tests
5139bc2 is described below
commit 5139bc2f89ef233a0611bff7af1ba180116869ea
Author: Eamon Ford <ea...@gmail.com>
AuthorDate: Mon Aug 10 20:35:00 2020 -0700
fix tests
---
.../collection_manager/services/CollectionWatcher.py | 1 -
.../services/history_manager/test_FileIngestionHistory.py | 14 ++++++++------
.../tests/services/test_CollectionProcessor.py | 14 ++++++++------
.../tests/services/test_CollectionWatcher.py | 1 -
granule_ingester/docker/Dockerfile | 14 ++++++++------
5 files changed, 24 insertions(+), 20 deletions(-)
diff --git a/collection_manager/collection_manager/services/CollectionWatcher.py b/collection_manager/collection_manager/services/CollectionWatcher.py
index 8911806..0d5eabd 100644
--- a/collection_manager/collection_manager/services/CollectionWatcher.py
+++ b/collection_manager/collection_manager/services/CollectionWatcher.py
@@ -6,7 +6,6 @@ from typing import Dict, Callable, Set, Optional, Awaitable
import yaml
from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer
-from yaml.scanner import ScannerError
from collection_manager.entities import Collection
from collection_manager.entities.exceptions import RelativePathError, CollectionConfigParsingError, \
diff --git a/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py b/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py
index ff1fb3c..07ab0e1 100644
--- a/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py
+++ b/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py
@@ -29,23 +29,25 @@ class TestFileIngestionHistory(unittest.TestCase):
result = await ingestion_history._get_signature("green")
self.assertIsNone(result)
- def test_already_ingested(self):
+ @async_test
+ async def test_already_ingested(self):
with tempfile.TemporaryDirectory() as history_dir:
ingestion_history = FileIngestionHistory(history_dir, DATASET_ID, md5sum_from_filepath)
# history_manager with this file
current_file_path = pathlib.Path(__file__)
- ingestion_history.push(str(current_file_path))
- self.assertTrue(ingestion_history.already_ingested(str(current_file_path)))
+ await ingestion_history.push(str(current_file_path))
+ self.assertTrue(await ingestion_history.already_ingested(str(current_file_path)))
del ingestion_history
- def test_already_ingested_with_latest_modifcation_signature(self):
+ @async_test
+ async def test_already_ingested_with_latest_modifcation_signature(self):
with tempfile.TemporaryDirectory() as history_dir:
ingestion_history = FileIngestionHistory(history_dir, DATASET_ID, os.path.getmtime)
# history_manager with this file
current_file_path = pathlib.Path(__file__)
- ingestion_history.push(str(current_file_path))
- self.assertTrue(ingestion_history.already_ingested(str(current_file_path)))
+ await ingestion_history.push(str(current_file_path))
+ self.assertTrue(await ingestion_history.already_ingested(str(current_file_path)))
del ingestion_history
diff --git a/collection_manager/tests/services/test_CollectionProcessor.py b/collection_manager/tests/services/test_CollectionProcessor.py
index fec4726..a7059d6 100644
--- a/collection_manager/tests/services/test_CollectionProcessor.py
+++ b/collection_manager/tests/services/test_CollectionProcessor.py
@@ -65,7 +65,7 @@ class TestCollectionProcessor(unittest.TestCase):
self.assertEqual(filled, expected)
@async_test
- @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', autospec=True)
+ @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', new_callable=AsyncMock)
@mock.patch('collection_manager.services.history_manager.FileIngestionHistoryBuilder', autospec=True)
@mock.patch('collection_manager.services.MessagePublisher', new_callable=AsyncMock)
async def test_process_granule_with_historical_granule(self, mock_publisher, mock_history_builder, mock_history):
@@ -87,10 +87,12 @@ class TestCollectionProcessor(unittest.TestCase):
mock_history.push.assert_called()
@async_test
- @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', autospec=True)
- @mock.patch('collection_manager.services.history_manager.FileIngestionHistoryBuilder', autospec=True)
+ @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', new_callable=AsyncMock)
+ @mock.patch('collection_manager.services.history_manager.FileIngestionHistoryBuilder', autospec=True)
@mock.patch('collection_manager.services.MessagePublisher', new_callable=AsyncMock)
- async def test_process_granule_with_forward_processing_granule(self, mock_publisher, mock_history_builder,
+ async def test_process_granule_with_forward_processing_granule(self,
+ mock_publisher,
+ mock_history_builder,
mock_history):
mock_history.get_granule_status.return_value = GranuleStatus.DESIRED_FORWARD_PROCESSING
mock_history_builder.build.return_value = mock_history
@@ -110,7 +112,7 @@ class TestCollectionProcessor(unittest.TestCase):
mock_history.push.assert_called()
@async_test
- @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', autospec=True)
+ @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', new_callable=AsyncMock)
@mock.patch('collection_manager.services.history_manager.FileIngestionHistoryBuilder', autospec=True)
@mock.patch('collection_manager.services.MessagePublisher', new_callable=AsyncMock)
async def test_process_granule_with_forward_processing_granule_and_no_priority(self, mock_publisher,
@@ -132,7 +134,7 @@ class TestCollectionProcessor(unittest.TestCase):
mock_history.push.assert_called()
@async_test
- @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', autospec=True)
+ @mock.patch('collection_manager.services.history_manager.FileIngestionHistory', new_callable=AsyncMock)
@mock.patch('collection_manager.services.history_manager.FileIngestionHistoryBuilder', autospec=True)
@mock.patch('collection_manager.services.MessagePublisher', new_callable=AsyncMock)
async def test_process_granule_with_undesired_granule(self, mock_publisher, mock_history_builder, mock_history):
diff --git a/collection_manager/tests/services/test_CollectionWatcher.py b/collection_manager/tests/services/test_CollectionWatcher.py
index 14e7c3c..c9a75c0 100644
--- a/collection_manager/tests/services/test_CollectionWatcher.py
+++ b/collection_manager/tests/services/test_CollectionWatcher.py
@@ -210,4 +210,3 @@ collections:
callback = AsyncMock()
await CollectionWatcher._run_periodically(None, 0.1, callback)
await AsyncAssert.assert_called_within_timeout(callback, timeout_sec=0.3, call_count=2)
-
diff --git a/granule_ingester/docker/Dockerfile b/granule_ingester/docker/Dockerfile
index 4b25318..57bacff 100644
--- a/granule_ingester/docker/Dockerfile
+++ b/granule_ingester/docker/Dockerfile
@@ -6,14 +6,16 @@ ENV PATH="/opt/conda/bin:$PATH"
RUN apk update --no-cache && apk add --no-cache --virtual .build-deps git openjdk8
-COPY /granule_ingester /sdap/granule_ingester
-COPY /setup.py /sdap/setup.py
-COPY /requirements.txt /sdap/requirements.txt
-COPY /conda-requirements.txt /sdap/conda-requirements.txt
-COPY /docker/install_nexusproto.sh /install_nexusproto.sh
-COPY /docker/entrypoint.sh /entrypoint.sh
+COPY common /common
+COPY granule_ingester/granule_ingester /sdap/granule_ingester
+COPY granule_ingester/setup.py /sdap/setup.py
+COPY granule_ingester/requirements.txt /sdap/requirements.txt
+COPY granule_ingester/conda-requirements.txt /sdap/conda-requirements.txt
+COPY granule_ingester/docker/install_nexusproto.sh /install_nexusproto.sh
+COPY granule_ingester/docker/entrypoint.sh /entrypoint.sh
RUN ./install_nexusproto.sh
+RUN cd /common && python setup.py install
RUN cd /sdap && python setup.py install
RUN apk del .build-deps