You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/10/02 01:20:36 UTC
[incubator-sdap-ingester] branch s3-support updated: Make some
public methods private, for clarity
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch s3-support
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/s3-support by this push:
new 38b254c Make some public methods private, for clarity
38b254c is described below
commit 38b254c13b1c5790f08d6b8250a8e5b086131cb3
Author: Eamon Ford <ea...@gmail.com>
AuthorDate: Thu Oct 1 18:20:22 2020 -0700
Make some public methods private, for clarity
---
.../services/history_manager/IngestionHistory.py | 46 +++++++++++-----------
.../history_manager/test_FileIngestionHistory.py | 6 +--
2 files changed, 26 insertions(+), 26 deletions(-)
diff --git a/collection_manager/collection_manager/services/history_manager/IngestionHistory.py b/collection_manager/collection_manager/services/history_manager/IngestionHistory.py
index ef73ccb..b71c32f 100644
--- a/collection_manager/collection_manager/services/history_manager/IngestionHistory.py
+++ b/collection_manager/collection_manager/services/history_manager/IngestionHistory.py
@@ -55,27 +55,6 @@ class IngestionHistory(ABC):
await self._save_latest_timestamp()
- def latest_ingested_mtime(self) -> Optional[datetime]:
- """
- Return the modified time of the most recently modified file that was ingested.
- :return: A datetime or None
- """
- if self._latest_ingested_file_update:
- return datetime.fromtimestamp(self._latest_ingested_file_update)
- else:
- return None
-
- async def already_ingested(self, file_path: str) -> bool:
- """
- Return a boolean indicating whether the specified file has already been ingested, based on its signature.
- :param file_path: The full path of a file to search for in the history.
- :return: A boolean indicating whether this file has already been ingested or not
- """
- file_path = file_path.strip()
- file_name = os.path.basename(file_path)
- signature = self._signature_fun(file_path)
- return signature == await self._get_signature(file_name)
-
async def get_granule_status(self,
file_path: str,
date_from: datetime = None,
@@ -94,13 +73,34 @@ class IngestionHistory(ABC):
should fall in order to be "desired".
:return: A GranuleStatus enum.
"""
- if self._in_time_range(file_path, date_from=self.latest_ingested_mtime()):
+ if self._in_time_range(file_path, date_from=self._latest_ingested_mtime()):
return GranuleStatus.DESIRED_FORWARD_PROCESSING
- elif self._in_time_range(file_path, date_from, date_to) and not await self.already_ingested(file_path):
+ elif self._in_time_range(file_path, date_from, date_to) and not await self._already_ingested(file_path):
return GranuleStatus.DESIRED_HISTORICAL
else:
return GranuleStatus.UNDESIRED
+ def _latest_ingested_mtime(self) -> Optional[datetime]:
+ """
+ Return the modified time of the most recently modified file that was ingested.
+ :return: A datetime or None
+ """
+ if self._latest_ingested_file_update:
+ return datetime.fromtimestamp(self._latest_ingested_file_update)
+ else:
+ return None
+
+ async def _already_ingested(self, file_path: str) -> bool:
+ """
+ Return a boolean indicating whether the specified file has already been ingested, based on its signature.
+ :param file_path: The full path of a file to search for in the history.
+ :return: A boolean indicating whether this file has already been ingested or not
+ """
+ file_path = file_path.strip()
+ file_name = os.path.basename(file_path)
+ signature = self._signature_fun(file_path)
+ return signature == await self._get_signature(file_name)
+
@abstractmethod
async def _save_latest_timestamp(self):
pass
diff --git a/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py b/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py
index 07ab0e1..8bd939e 100644
--- a/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py
+++ b/collection_manager/tests/services/history_manager/test_FileIngestionHistory.py
@@ -36,7 +36,7 @@ class TestFileIngestionHistory(unittest.TestCase):
# history_manager with this file
current_file_path = pathlib.Path(__file__)
await ingestion_history.push(str(current_file_path))
- self.assertTrue(await ingestion_history.already_ingested(str(current_file_path)))
+ self.assertTrue(await ingestion_history._already_ingested(str(current_file_path)))
del ingestion_history
@@ -47,7 +47,7 @@ class TestFileIngestionHistory(unittest.TestCase):
# history_manager with this file
current_file_path = pathlib.Path(__file__)
await ingestion_history.push(str(current_file_path))
- self.assertTrue(await ingestion_history.already_ingested(str(current_file_path)))
+ self.assertTrue(await ingestion_history._already_ingested(str(current_file_path)))
del ingestion_history
@@ -57,7 +57,7 @@ class TestFileIngestionHistory(unittest.TestCase):
ingestion_history = FileIngestionHistory(history_dir, DATASET_ID, md5sum_from_filepath)
# history_manager with this file
current_file_path = pathlib.Path(__file__)
- self.assertFalse(await ingestion_history.already_ingested(str(current_file_path)))
+ self.assertFalse(await ingestion_history._already_ingested(str(current_file_path)))
if __name__ == '__main__':