You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by tl...@apache.org on 2021/11/28 17:15:34 UTC
[incubator-sdap-ingester] 01/01: upgrade solr connection status control for latest solr versions
This is an automated email from the ASF dual-hosted git repository.
tloubrieu pushed a commit to branch solr_stable_359
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-ingester.git
commit f206e91162fe008a8891f802f835b984eea0d526
Author: Thomas Loubrieu <lo...@jpl.nasa.gov>
AuthorDate: Sun Nov 28 12:15:18 2021 -0500
upgrade solr connection status control for latest solr versions
---
.../granule_ingester/writers/SolrStore.py | 37 +++++++++++++++++++---
1 file changed, 33 insertions(+), 4 deletions(-)
diff --git a/granule_ingester/granule_ingester/writers/SolrStore.py b/granule_ingester/granule_ingester/writers/SolrStore.py
index def70c5..5c5f088 100644
--- a/granule_ingester/granule_ingester/writers/SolrStore.py
+++ b/granule_ingester/granule_ingester/writers/SolrStore.py
@@ -50,13 +50,42 @@ class SolrStore(MetadataStore):
self.log.setLevel(logging.DEBUG)
self._solr = None
+ def _get_collections(self, zk, parent_nodes):
+ """
+ try to get list of collection from zookeper, on a list of candidate nodes,
+ return the first successful request result
+ """
+
+ try:
+ logger.debug("getting solr configuration from zookeeper, node '%s'", parent_nodes[0])
+ return parent_nodes[0], zk.zk.get_children(parent_nodes[0])
+ except NoNodeError:
+ logger.debug("solr configuration not found in node '%s'", parent_nodes[0])
+ if len(parent_nodes)>1:
+ return self._get_collections(zk, parent_nodes[1:])
+ else:
+ raise
+
+ def _set_solr_status(self, zk):
+ """ because of something not working right between zookeeper and solr
+ we need to manually update the solr status on zookeeper
+ see https://github.com/django-haystack/pysolr/issues/189
+ """
+ collections = {}
+ parent_node, zk_collections = self._get_collections(zk,
+ ['collections',
+ 'solr/collections']
+ # with bitnami/solr 0.3.3 helm chart deployment
+ )
+
+ for c in zk_collections:
+ collections.update(json.loads(zk.zk.get(f"{parent_node}/{c}/state.json")[0].decode("utf-8")))
+ zk.collections = collections
+
def _get_connection(self) -> pysolr.Solr:
if self._zk_url:
zk = pysolr.ZooKeeper(f"{self._zk_url}")
- collections = {}
- for c in zk.zk.get_children("collections"):
- collections.update(json.loads(zk.zk.get("collections/{}/state.json".format(c))[0].decode("ascii")))
- zk.collections = collections
+ self._set_solr_status(zk)
return pysolr.SolrCloud(zk, self._collection, always_commit=True)
elif self._solr_url:
return pysolr.Solr(f'{self._solr_url}/solr/{self._collection}', always_commit=True)