You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2021/11/28 17:15:33 UTC
[incubator-sdap-ingester] branch solr_stable_359 created (now f206e91)
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a change to branch solr_stable_359
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git.
at f206e91 upgrade solr connection status control for latest solr versions
This branch includes the following new commits:
new f206e91 upgrade solr connection status control for latest solr versions
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-sdap-ingester] 01/01: upgrade solr connection status control for latest solr versions
Posted by tl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch solr_stable_359
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f206e91162fe008a8891f802f835b984eea0d526
Author: Thomas Loubrieu <lo...@jpl.nasa.gov>
AuthorDate: Sun Nov 28 12:15:18 2021 -0500
upgrade solr connection status control for latest solr versions
---
.../granule_ingester/writers/SolrStore.py | 37 +++++++++++++++++++---
1 file changed, 33 insertions(+), 4 deletions(-)
diff --git a/granule_ingester/granule_ingester/writers/SolrStore.py b/granule_ingester/granule_ingester/writers/SolrStore.py
index def70c5..5c5f088 100644
--- a/granule_ingester/granule_ingester/writers/SolrStore.py
+++ b/granule_ingester/granule_ingester/writers/SolrStore.py
@@ -50,13 +50,42 @@ class SolrStore(MetadataStore):
self.log.setLevel(logging.DEBUG)
self._solr = None
+ def _get_collections(self, zk, parent_nodes):
+ """
+ try to get list of collection from zookeper, on a list of candidate nodes,
+ return the first successful request result
+ """
+
+ try:
+ logger.debug("getting solr configuration from zookeeper, node '%s'", parent_nodes[0])
+ return parent_nodes[0], zk.zk.get_children(parent_nodes[0])
+ except NoNodeError:
+ logger.debug("solr configuration not found in node '%s'", parent_nodes[0])
+ if len(parent_nodes)>1:
+ return self._get_collections(zk, parent_nodes[1:])
+ else:
+ raise
+
+ def _set_solr_status(self, zk):
+ """ because of something not working right between zookeeper and solr
+ we need to manually update the solr status on zookeeper
+ see https://github.com/django-haystack/pysolr/issues/189
+ """
+ collections = {}
+ parent_node, zk_collections = self._get_collections(zk,
+ ['collections',
+ 'solr/collections']
+ # with bitnami/solr 0.3.3 helm chart deployment
+ )
+
+ for c in zk_collections:
+ collections.update(json.loads(zk.zk.get(f"{parent_node}/{c}/state.json")[0].decode("utf-8")))
+ zk.collections = collections
+
def _get_connection(self) -> pysolr.Solr:
if self._zk_url:
zk = pysolr.ZooKeeper(f"{self._zk_url}")
- collections = {}
- for c in zk.zk.get_children("collections"):
- collections.update(json.loads(zk.zk.get("collections/{}/state.json".format(c))[0].decode("ascii")))
- zk.collections = collections
+ self._set_solr_status(zk)
return pysolr.SolrCloud(zk, self._collection, always_commit=True)
elif self._solr_url:
return pysolr.Solr(f'{self._solr_url}/solr/{self._collection}', always_commit=True)