You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/10/19 10:36:54 UTC

[47/50] [abbrv] lucene-solr:jira/solr-11320: SOLR-11444: Improve consistency of collection alias handling and collection list references. Other refactorings of nearby code too.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
index e927ddb..4ff748c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
@@ -16,47 +16,132 @@
  */
 package org.apache.solr.common.cloud;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+/**
+ * Holds collection aliases -- virtual collections that point to one or more other collections.
+ * We might add other types of aliases here some day.
+ * Immutable.
+ */
 public class Aliases {
 
-  private Map<String,Map<String,String>> aliasMap;
+  public static final Aliases EMPTY = new Aliases(Collections.emptyMap());
+
+  /** Map of "collection" string constant to ->
+   *   alias name -> comma delimited list of collections */
+  private final Map<String,Map<String,String>> aliasMap; // not-null
+
+  private final Map<String, List<String>> collectionAliasListMap; // not-null; computed from aliasMap
+
+  public static Aliases fromJSON(byte[] bytes) {
+    if (bytes == null || bytes.length == 0) {
+      return EMPTY;
+    }
+    return new Aliases((Map<String,Map<String,String>>) Utils.fromJSON(bytes));
+  }
 
-  public Aliases(Map<String,Map<String,String>> aliasMap) {
+  private Aliases(Map<String, Map<String,String>> aliasMap) {
     this.aliasMap = aliasMap;
+    collectionAliasListMap = convertMapOfCommaDelimitedToMapOfList(getCollectionAliasMap());
   }
 
-  public Aliases() {
-    this.aliasMap = new HashMap<>();
+  public static Map<String, List<String>> convertMapOfCommaDelimitedToMapOfList(Map<String, String> collectionAliasMap) {
+    Map<String, List<String>> collectionAliasListMap = new LinkedHashMap<>(collectionAliasMap.size());
+    for (Map.Entry<String, String> entry : collectionAliasMap.entrySet()) {
+      collectionAliasListMap.put(entry.getKey(), StrUtils.splitSmart(entry.getValue(), ",", true));
+    }
+    return collectionAliasListMap;
   }
-  
+
+  /**
+   * Returns an unmodifiable Map of collection aliases mapped to a comma delimited list of what the alias maps to.
+   * Does not return null.
+   * Prefer use of {@link #getCollectionAliasListMap()} instead, where appropriate.
+   */
   public Map<String,String> getCollectionAliasMap() {
     Map<String,String> cam = aliasMap.get("collection");
-    if (cam == null) return null;
-    return Collections.unmodifiableMap(cam);
+    return cam == null ? Collections.emptyMap() : Collections.unmodifiableMap(cam);
+  }
+
+  /**
+   * Returns an unmodifiable Map of collection aliases mapped to a list of what the alias maps to.
+   * Does not return null.
+   */
+  public Map<String,List<String>> getCollectionAliasListMap() {
+    return Collections.unmodifiableMap(collectionAliasListMap);
   }
   
-  public Map<String,Map<String,String>> getAliasMap() {
-    return Collections.unmodifiableMap(aliasMap);
+  public boolean hasCollectionAliases() {
+    return !collectionAliasListMap.isEmpty();
   }
 
-  public int collectionAliasSize() {
-    Map<String,String> cam = aliasMap.get("collection");
-    if (cam == null) return 0;
-    return cam.size();
+  /**
+   * Returns a list of collections that the input alias name maps to. If there
+   * are none, the input is returned. One level of alias indirection is supported (alias to alias to collection).
+   * Treat the result as unmodifiable.
+   */
+  public List<String> resolveAliases(String aliasName) {
+    return resolveAliasesGivenAliasMap(collectionAliasListMap, aliasName);
   }
-  
-  @Override
-  public String toString() {
-    return "Aliases [aliasMap=" + aliasMap + "]";
+
+  /** @lucene.internal */
+  public static List<String> resolveAliasesGivenAliasMap(Map<String, List<String>> collectionAliasListMap, String aliasName) {
+    //return collectionAliasListMap.getOrDefault(aliasName, Collections.singletonList(aliasName));
+    // TODO deprecate and remove this dubious feature?
+    // Due to another level of indirection, this is more complicated...
+    List<String> level1 = collectionAliasListMap.get(aliasName);
+    if (level1 == null) {
+      return Collections.singletonList(aliasName);// is a collection
+    }
+    List<String> result = new ArrayList<>(level1.size());
+    for (String level1Alias : level1) {
+      List<String> level2 = collectionAliasListMap.get(level1Alias);
+      if (level2 == null) {
+        result.add(level1Alias);
+      } else {
+        result.addAll(level2);
+      }
+    }
+    return result;
   }
 
-  public String getCollectionAlias(String collectionName) {
-    Map<String,String> cam = aliasMap.get("collection");
-    if (cam == null) return null;
-    return cam.get(collectionName);
+  /**
+   * Creates a new Aliases instance with the same data as the current one but with a modification based on the
+   * parameters. If {@code collections} is null, then the {@code alias} is removed, otherwise it is added/updated.
+   */
+  public Aliases cloneWithCollectionAlias(String alias, String collections) {
+    Map<String,String> newCollectionMap = new HashMap<>(getCollectionAliasMap());
+    if (collections == null) {
+      newCollectionMap.remove(alias);
+    } else {
+      newCollectionMap.put(alias, collections);
+    }
+    if (newCollectionMap.isEmpty()) {
+      return EMPTY;
+    } else {
+      return new Aliases(Collections.singletonMap("collection", newCollectionMap));
+    }
   }
 
+  /** Serialize to ZooKeeper. */
+  public byte[] toJSON() {
+    if (collectionAliasListMap.isEmpty()) {
+      return null;
+    } else {
+      return Utils.toJSON(aliasMap);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "Aliases [aliasMap=" + aliasMap + "]";
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
index 8e52e41..bfe33c5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
@@ -232,15 +232,6 @@ public class ClusterState implements JSONWriter.Writable {
     return new ClusterState( liveNodes, collections,version);
   }
 
-  public static Aliases load(byte[] bytes) {
-    if (bytes == null || bytes.length == 0) {
-      return new Aliases();
-    }
-    Map<String,Map<String,String>> aliasMap = (Map<String,Map<String,String>>) Utils.fromJSON(bytes);
-
-    return new Aliases(aliasMap);
-  }
-
   // TODO move to static DocCollection.loadFromMap
   private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version, String znode) {
     Map<String,Object> props;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index e45565f..4695d7d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -257,7 +257,7 @@ public class ZkStateReader implements Closeable {
   
   private final boolean closeClient;
 
-  private volatile Aliases aliases = new Aliases();
+  private volatile Aliases aliases = Aliases.EMPTY;
 
   private volatile boolean closed = false;
 
@@ -385,8 +385,10 @@ public class ZkStateReader implements Closeable {
   public void updateLiveNodes() throws KeeperException, InterruptedException {
     refreshLiveNodes(null);
   }
-  
+
+  /** Never null. */
   public Aliases getAliases() {
+    assert aliases != null;
     return aliases;
   }
 
@@ -455,7 +457,7 @@ public class ZkStateReader implements Closeable {
                   final Watcher thisWatch = this;
                   final Stat stat = new Stat();
                   final byte[] data = zkClient.getData(ALIASES, thisWatch, stat, true);
-                  ZkStateReader.this.aliases = ClusterState.load(data);
+                  ZkStateReader.this.aliases = Aliases.fromJSON(data);
                   LOG.debug("New alias definition is: " + ZkStateReader.this.aliases.toString());
                 }
               } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
@@ -896,7 +898,7 @@ public class ZkStateReader implements Closeable {
 
   public void updateAliases() throws KeeperException, InterruptedException {
     final byte[] data = zkClient.getData(ALIASES, null, null, true);
-    this.aliases = ClusterState.load(data);
+    this.aliases = Aliases.fromJSON(data);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
index a20c6e2..117de57 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
@@ -88,6 +88,7 @@ public class StrUtils {
    * @param s  the string to split
    * @param separator the separator to split on
    * @param decode decode backslash escaping
+   * @return not null
    */
   public static List<String> splitSmart(String s, String separator, boolean decode) {
     ArrayList<String> lst = new ArrayList<>(2);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
index a3dd436..cfcfcd6 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
@@ -22,6 +22,7 @@ import java.net.SocketException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -130,13 +131,8 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 {
       }
 
       @Override
-      public String getAlias(String collection) {
-        return collection;
-      }
-
-      @Override
-      public String getCollectionName(String name) {
-        return name;
+      public List<String> resolveAlias(String collection) {
+        return Collections.singletonList(collection);
       }
 
       @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
index ce14907..bb07c45 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
@@ -28,7 +28,6 @@ import java.sql.Types;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.SortedSet;
@@ -555,13 +554,7 @@ public class JdbcTest extends SolrCloudTestCase {
       tables.addAll(collectionsSet);
 
       Aliases aliases = zkStateReader.getAliases();
-      if(aliases != null) {
-        Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap();
-        if(collectionAliasMap != null) {
-          Set<String> aliasesSet = collectionAliasMap.keySet();
-          tables.addAll(aliasesSet);
-        }
-      }
+      tables.addAll(aliases.getCollectionAliasListMap().keySet());
 
       try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) {
         for(String table : tables) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 8e129c7..9d9a9dc 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -84,6 +84,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
 import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
 import org.apache.solr.client.solrj.util.ClientUtils;
 import org.apache.solr.cloud.IpTables;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
@@ -2224,6 +2225,14 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
       super();
     }
 
+    public CloudSolrClient.Builder withCluster(MiniSolrCloudCluster cluster) {
+      if (random().nextBoolean()) {
+        return withZkHost(cluster.getZkServer().getZkAddress());
+      } else {
+        return withSolrUrl(cluster.getRandomJetty(random()).getBaseUrl().toString());
+      }
+    }
+
     @Override
     public CloudSolrClient.Builder sendDirectUpdatesToShardLeadersOnly() {
       configuredDUTflag = true;
@@ -2247,7 +2256,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
     @Override
     public CloudSolrClient build() {
       if (configuredDUTflag == false) {
-        // flag value not explicity configured
+        // flag value not explicitly configured
         if (random().nextBoolean()) {
           // so randomly choose a value
           randomlyChooseDirectUpdatesToLeadersOnly();
@@ -2262,7 +2271,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
 
   /**
    * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
-   * Tests that do not wish to have any randomized behavior should use the 
+   * Tests that do not wish to have any randomized behavior should use the
    * {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly
    */ 
   public static CloudSolrClient getCloudSolrClient(String zkHost) {
@@ -2270,7 +2279,18 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
         .withZkHost(zkHost)
         .build();
   }
-  
+
+  /**
+   * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
+   * Tests that do not wish to have any randomized behavior should use the
+   * {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly
+   */
+  public static CloudSolrClient getCloudSolrClient(MiniSolrCloudCluster cluster) {
+    return new CloudSolrClientBuilder()
+        .withCluster(cluster)
+        .build();
+  }
+
   /**
    * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
    * Tests that do not wish to have any randomized behavior should use the 
@@ -2300,7 +2320,11 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
         .sendUpdatesToAllReplicasInShard()
         .build();
   }
-  
+
+  public static CloudSolrClientBuilder newCloudSolrClient(String zkHost) {
+    return (CloudSolrClientBuilder) new CloudSolrClientBuilder().withZkHost(zkHost);
+  }
+
   /**
    * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
    * Tests that do not wish to have any randomized behavior should use the