You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/07/30 02:26:04 UTC
[incubator-sdap-ingester] branch solr-history-fix created (now
134ecbd)
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a change to branch solr-history-fix
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git.
at 134ecbd fix solr history
This branch includes the following new commits:
new 134ecbd fix solr history
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-ingester] 01/01: fix solr history
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch solr-history-fix
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit 134ecbda3475c76a18cdb5ba54bc5d9543683228
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Wed Jul 29 19:25:50 2020 -0700
fix solr history
---
.../services/history_manager/SolrIngestionHistory.py | 19 ++++++++-----------
1 file changed, 8 insertions(+), 11 deletions(-)
diff --git a/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py b/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py
index 1ae7156..4e6d3e5 100644
--- a/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py
+++ b/collection_manager/collection_manager/services/history_manager/SolrIngestionHistory.py
@@ -35,8 +35,8 @@ class SolrIngestionHistory(IngestionHistory):
try:
self._solr_url = solr_url
self._create_collection_if_needed()
- self._solr_granules = pysolr.Solr('/'.join([solr_url.strip('/'), self._granule_collection_name]))
- self._solr_datasets = pysolr.Solr('/'.join([solr_url.strip('/'), self._dataset_collection_name]))
+ self._solr_granules = pysolr.Solr(f"{solr_url.strip('/')}/solr/{self._granule_collection_name}")
+ self._solr_datasets = pysolr.Solr(f"{solr_url.strip('/')}/solr/{self._dataset_collection_name}")
self._dataset_id = dataset_id
self._signature_fun = md5sum_from_filepath if signature_fun is None else signature_fun
self._latest_ingested_file_update = self._get_latest_file_update()
@@ -63,7 +63,7 @@ class SolrIngestionHistory(IngestionHistory):
self._solr_datasets.add([{
'id': self._dataset_id,
'dataset_s': self._dataset_id,
- 'latest_update_l': self._latest_ingested_file_update}])
+ 'latest_update_l': int(self._latest_ingested_file_update)}])
self._solr_datasets.commit()
def _get_latest_file_update(self):
@@ -87,8 +87,7 @@ class SolrIngestionHistory(IngestionHistory):
self._req_session = requests.session()
payload = {'action': 'CLUSTERSTATUS'}
- result = self._req_session.get('/'.join([self._solr_url.strip('/'), 'admin', 'collections']),
- params=payload)
+ result = self._req_session.get(f"{self._solr_url.strip('/')}/solr/admin/collections", params=payload)
response = result.json()
node_number = len(response['cluster']['live_nodes'])
@@ -100,12 +99,11 @@ class SolrIngestionHistory(IngestionHistory):
'name': self._granule_collection_name,
'numShards': node_number
}
- result = self._req_session.get('/'.join([self._solr_url.strip("/"), 'admin', 'collections']),
- params=payload)
+ result = self._req_session.get(f"{self._solr_url.strip('/')}/solr/admin/collections", params=payload)
response = result.json()
logger.info(f"solr collection created {response}")
# Update schema
- schema_url = '/'.join([self._solr_url.strip('/'), self._granule_collection_name, 'schema'])
+ schema_url = f"{self._solr_url.strip('/')}/{self._granule_collection_name}/schema"
# granule_s # dataset_s so that all the granule of a dataset are less likely to be on the same shard
# self.add_unique_key_field(schema_url, "uniqueKey_s", "StrField")
self._add_field(schema_url, "dataset_s", "StrField")
@@ -121,13 +119,12 @@ class SolrIngestionHistory(IngestionHistory):
'name': self._dataset_collection_name,
'numShards': node_number
}
- result = self._req_session.get('/'.join([self._solr_url.strip('/'), 'admin', 'collections']),
- params=payload)
+ result = self._req_session.get(f"{self._solr_url.strip('/')}/solr/admin/collections", params=payload)
response = result.json()
logger.info(f"solr collection created {response}")
# Update schema
# http://localhost:8983/solr/nexusdatasets/schema?_=1588555874864&wt=json
- schema_url = '/'.join([self._solr_url.strip('/'), self._dataset_collection_name, 'schema'])
+ schema_url = f"{self._solr_url.strip('/')}/{self._granule_collection_name}/schema"
# self.add_unique_key_field(schema_url, "uniqueKey_s", "StrField")
self._add_field(schema_url, "dataset_s", "StrField")
self._add_field(schema_url, "latest_update_l", "TrieLongField")