[GitHub] kasakrisz commented on a change in pull request #40: AMBARI-24949. Log Search: add Solr node discovery + small cleanups

[GitBox <gi...@apache.org>]

kasakrisz commented on a change in pull request #40: AMBARI-24949. Log Search: add Solr node discovery + small cleanups
URL: https://github.com/apache/ambari-logsearch/pull/40#discussion_r236003086
 
 

 ##########
 File path: ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederSolrClientFactory.java
 ##########
 @@ -38,32 +49,132 @@
    * @param zkConnectionString zookeeper connection string, e.g.: localhost1:2181,localhost2:2181/solr
    * @param solrUrls list of solr urls
    * @param collection name of the Solr collection
+   * @param discover use cloud solr client to discover solr nodes, then uses LB client
    * @return created client
    */
-  public SolrClient createSolrClient(String zkConnectionString, String[] solrUrls, String collection) {
+  public SolrClient createSolrClient(String zkConnectionString, String[] solrUrls, String collection, boolean discover) {
     logger.info("Creating solr client ...");
     logger.info("Using collection=" + collection);
-    if (solrUrls != null && solrUrls.length > 0) {
+    if (discover && zkConnectionString.length() > 0) {
+      final CloudSolrClient discoverNodesClient = createSolrCloudClient(zkConnectionString, collection);
+      return createLBClientsWithDiscoverNodes(discoverNodesClient, collection);
+    }
+    else if (solrUrls != null && solrUrls.length > 0) {
       logger.info("Using lbHttpSolrClient with urls: {}",
         StringUtils.join(appendTo("/" + collection, solrUrls), ","));
       LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
       builder.withBaseSolrUrls(solrUrls);
       return builder.build();
     } else {
-      logger.info("Using zookeepr. zkConnectString=" + zkConnectionString);
-      CloudSolrClient.Builder builder = new CloudSolrClient.Builder();
-      builder.withZkHost(zkConnectionString);
-      CloudSolrClient solrClient = builder.build();
-      solrClient.setDefaultCollection(collection);
-      return solrClient;
+      return createSolrCloudClient(zkConnectionString, collection);
+    }
+  }
+
+  @VisibleForTesting
+  ZkConnection createZKConnection(String zkConnectionString) {
+    String split[] = zkConnectionString.split("/", 2);
+    String zkChroot = null;
+    final List<String> zkHosts;
+    if (split.length == 1) {
+      zkHosts = Arrays.asList(split[0].split(","));
+    } else {
+      zkHosts = Arrays.asList(split[0].split(","));
+      zkChroot = ("/" + split[1]).replaceAll("/+", "/");
+      if (zkChroot.endsWith("/")) {
+        zkChroot = zkChroot.substring(0, zkChroot.lastIndexOf("/"));
+      }
     }
+    return new ZkConnection(zkHosts, zkChroot);
   }
 
-  private String[] appendTo(String toAppend, String... appendees) {
+  @VisibleForTesting
+  String[] appendTo(String toAppend, String... appendees) {
     for (int i = 0; i < appendees.length; i++) {
       appendees[i] = appendees[i] + toAppend;
     }
     return appendees;
   }
 
+  private CloudSolrClient createSolrCloudClient(String zkConnectionString, String collection) {
+    logger.info("Using zookeepr. zkConnectString=" + zkConnectionString);
+    final ZkConnection zkConnection = createZKConnection(zkConnectionString);
+    final CloudSolrClient.Builder builder =
+      new CloudSolrClient.Builder(zkConnection.getZkHosts(), Optional.ofNullable(zkConnection.getZkChroot()));
+    CloudSolrClient solrClient = builder.build();
+    solrClient.setDefaultCollection(collection);
+    return solrClient;
+  }
+
+  private LBHttpSolrClient createLBClientsWithDiscoverNodes(CloudSolrClient discoverClient, String collection) {
+    final List<String> baseUrls = waitUntilAvailableBaseUrls(discoverClient, collection);
+    final String[] finalBaseUrls = appendTo("/" + collection, baseUrls.toArray(new String[0]));
+    logger.info("Following URLs will be used for LB Solr client (collection: '{}'): {}", collection, StringUtils.join(finalBaseUrls));
+    return new LBHttpSolrClient.Builder()
+      .withBaseSolrUrls(finalBaseUrls)
+      .build();
+  }
+
+  private List<String> waitUntilAvailableBaseUrls(CloudSolrClient discoverClient, String collection) {
+    final List<String> baseUrls = new ArrayList<>();
+    while(true) {
+      ZkStateReader zkStateReader = discoverClient.getZkStateReader();
+      ClusterState clusterState = zkStateReader.getClusterState();
+      if (clusterState == null) {
+        continue;
 
 Review comment:
   1. if clusterState or docCollection or replicas is always null this will be an infinite loop.
   2. Should any sleep be added to this iteration?

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services