You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/06/22 20:54:59 UTC
[incubator-sdap-ingester] branch bug_fixes updated: fixed timestamp
bug
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch bug_fixes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
The following commit(s) were added to refs/heads/bug_fixes by this push:
new ab87a7a fixed timestamp bug
ab87a7a is described below
commit ab87a7a56a13f5eed720a22110f028cf1f6aa59d
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Mon Jun 22 13:54:46 2020 -0700
fixed timestamp bug
---
.../services/history_manager/FileIngestionHistory.py | 4 ++--
.../collection_manager/services/history_manager/IngestionHistory.py | 6 ++++++
.../services/history_manager/SolrIngestionHistory.py | 3 +--
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/collection_manager/collection_manager/services/history_manager/FileIngestionHistory.py b/collection_manager/collection_manager/services/history_manager/FileIngestionHistory.py
index 0a92317..50f2170 100644
--- a/collection_manager/collection_manager/services/history_manager/FileIngestionHistory.py
+++ b/collection_manager/collection_manager/services/history_manager/FileIngestionHistory.py
@@ -20,6 +20,7 @@ class FileIngestionHistoryBuilder(IngestionHistoryBuilder):
signature_fun=self._signature_fun)
+# TODO: clean this up, better tests
class FileIngestionHistory(IngestionHistory):
def __init__(self, history_path: str, dataset_id: str, signature_fun=None):
@@ -55,7 +56,6 @@ class FileIngestionHistory(IngestionHistory):
def __del__(self):
self._history_file.close()
self._purge()
- self._save_latest_timestamp()
del self._history_dict
def reset_cache(self):
@@ -91,8 +91,8 @@ class FileIngestionHistory(IngestionHistory):
def _push_record(self, file_name, signature):
self._history_dict[file_name] = signature
+
self._history_file.write(f'{file_name},{signature}\n')
- return None
def _get_signature(self, file_name):
return self._history_dict.get(file_name, None)
diff --git a/collection_manager/collection_manager/services/history_manager/IngestionHistory.py b/collection_manager/collection_manager/services/history_manager/IngestionHistory.py
index b14b409..d92cb24 100644
--- a/collection_manager/collection_manager/services/history_manager/IngestionHistory.py
+++ b/collection_manager/collection_manager/services/history_manager/IngestionHistory.py
@@ -53,6 +53,8 @@ class IngestionHistory(ABC):
else:
self._latest_ingested_file_update = max(self._latest_ingested_file_update, os.path.getmtime(file_path))
+ self._save_latest_timestamp()
+
def latest_ingested_mtime(self) -> Optional[datetime]:
"""
Return the modified time of the most recently modified file that was ingested.
@@ -100,6 +102,10 @@ class IngestionHistory(ABC):
return GranuleStatus.UNDESIRED
@abstractmethod
+ def _save_latest_timestamp(self):
+ pass
+
+ @abstractmethod
def _push_record(self, file_name, signature):
pass
diff --git a/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py b/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py
index 2d0438f..1ae7156 100644
--- a/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py
+++ b/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py
@@ -44,7 +44,6 @@ class SolrIngestionHistory(IngestionHistory):
raise DatasetIngestionHistorySolrException(f"solr instance unreachable {solr_url}")
def __del__(self):
- self._push_latest_ingested_date()
self._req_session.close()
def _push_record(self, file_name, signature):
@@ -58,7 +57,7 @@ class SolrIngestionHistory(IngestionHistory):
self._solr_granules.commit()
return None
- def _push_latest_ingested_date(self):
+ def _save_latest_timestamp(self):
if self._solr_datasets:
self._solr_datasets.delete(q=f"id:{self._dataset_id}")
self._solr_datasets.add([{