You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/08/10 02:19:30 UTC
[lucene-solr] 01/05: Lazy SolrCluster impl
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr14680
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit f68d8b8d3b5c5a95420c38d6f747c87c596c6006
Author: noblepaul <no...@gmail.com>
AuthorDate: Mon Aug 10 12:08:30 2020 +1000
Lazy SolrCluster impl
---
.../org/apache/solr/cloud/TestLazySolrCluster.java | 80 ++++
.../api/CollectionConfig.java} | 14 +-
.../java/org/apache/solr/cluster/api/Config.java | 40 --
.../java/org/apache/solr/cluster/api/Resource.java | 20 +
.../java/org/apache/solr/cluster/api/Shard.java | 2 -
.../org/apache/solr/cluster/api/ShardReplica.java | 5 +-
.../{common/util => cluster/api}/SimpleMap.java | 34 +-
.../org/apache/solr/cluster/api/SolrCluster.java | 3 +-
.../apache/solr/cluster/api/SolrCollection.java | 6 +-
.../java/org/apache/solr/cluster/api/SolrNode.java | 2 -
.../org/apache/solr/common/LazySolrCluster.java | 414 +++++++++++++++++++++
.../java/org/apache/solr/common/SimpleZkMap.java | 114 ++++++
.../java/org/apache/solr/common/cloud/Aliases.java | 8 +
.../org/apache/solr/common/cloud/ClusterState.java | 4 +
.../apache/solr/common/cloud/ZkStateReader.java | 4 +
.../org/apache/solr/common/params/SolrParams.java | 2 +-
.../solr/common/util/LinkedSimpleHashMap.java | 11 +-
.../org/apache/solr/common/util/NamedList.java | 39 +-
.../java/org/apache/solr/common/util/Utils.java | 5 +-
.../apache/solr/common/util/WrappedSimpleMap.java | 32 ++
20 files changed, 754 insertions(+), 85 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLazySolrCluster.java b/solr/core/src/test/org/apache/solr/cloud/TestLazySolrCluster.java
new file mode 100644
index 0000000..0689509
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestLazySolrCluster.java
@@ -0,0 +1,80 @@
+package org.apache.solr.cloud;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cluster.api.CollectionConfig;
+import org.apache.solr.cluster.api.SimpleMap;
+import org.apache.solr.cluster.api.SolrCollection;
+import org.apache.solr.common.LazySolrCluster;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.zookeeper.CreateMode;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class TestLazySolrCluster extends SolrCloudTestCase {
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(5)
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .configure();
+ }
+
+ public void test() throws Exception {
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ String collection = "testLazyCluster1";
+ cloudClient.request(CollectionAdminRequest.createCollection(collection, "conf1", 2, 2));
+ cluster.waitForActiveCollection(collection, 2, 4);
+ collection = "testLazyCluster2";
+ cloudClient.request(CollectionAdminRequest.createCollection(collection, "conf1", 2, 2));
+ cluster.waitForActiveCollection(collection, 2, 4);
+
+ LazySolrCluster solrCluster = new LazySolrCluster(cluster.getSolrClient().getZkStateReader());
+ SimpleMap<SolrCollection> colls = solrCluster.collections();
+
+ SolrCollection c = colls.get("testLazyCluster1");
+ assertNotNull(c);
+ c = colls.get("testLazyCluster2");
+ assertNotNull(c);
+ int[] count = new int[1];
+ solrCluster.collections().forEachEntry((s, solrCollection) -> count[0]++);
+ assertEquals(2, count[0]);
+
+ count[0] = 0;
+
+ assertEquals(2, solrCluster.collections().get("testLazyCluster1").shards().size());
+ solrCluster.collections().get("testLazyCluster1").shards()
+ .forEachEntry((s, shard) -> shard.replicas().forEachEntry((s1, replica) -> count[0]++));
+ assertEquals(4, count[0]);
+
+ assertEquals(5, solrCluster.nodes().size());
+ SolrZkClient zkClient = cloudClient.getZkStateReader().getZkClient();
+ zkClient.create(ZkStateReader.CONFIGS_ZKNODE + "/conf1/a", null, CreateMode.PERSISTENT, true);
+ zkClient.create(ZkStateReader.CONFIGS_ZKNODE + "/conf1/a/aa1", new byte[1024], CreateMode.PERSISTENT, true);
+ zkClient.create(ZkStateReader.CONFIGS_ZKNODE + "/conf1/a/aa2", new byte[1024 * 2], CreateMode.PERSISTENT, true);
+
+ List<String> allFiles = new ArrayList<>();
+ byte[] buf = new byte[3*1024];
+ CollectionConfig conf1 = solrCluster.configs().get("conf1");
+ conf1.resources().abortableForEach((s, resource) -> {
+ allFiles.add(s);
+ if("a/aa1".equals(s)) {
+ resource.get(is -> assertEquals(1024, is.read(buf)));
+ }
+ if("a/aa2".equals(s)) {
+ resource.get(is -> assertEquals(2*1024, is.read(buf)));
+ }
+ if("a".equals(s)) {
+ resource.get(is -> assertEquals(-1, is.read()));
+ }
+ return Boolean.TRUE;
+ });
+ assertEquals(5, allFiles.size());
+
+ }
+
+
+}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java b/solr/solrj/src/java/org/apache/solr/cluster/api/CollectionConfig.java
similarity index 75%
copy from solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java
copy to solr/solrj/src/java/org/apache/solr/cluster/api/CollectionConfig.java
index 185b9a7..4f6b82c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/CollectionConfig.java
@@ -14,14 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.solr.common.util;
-import java.util.LinkedHashMap;
+package org.apache.solr.cluster.api;
-public class LinkedSimpleHashMap<T> extends LinkedHashMap<CharSequence, T> implements SimpleMap<T> {
- @Override
- public T get(CharSequence key) {
- return super.get(key);
- }
+
+public interface CollectionConfig {
+
+ String name();
+
+ SimpleMap<Resource> resources();
}
diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/Config.java b/solr/solrj/src/java/org/apache/solr/cluster/api/Config.java
deleted file mode 100644
index 084a9c4..0000000
--- a/solr/solrj/src/java/org/apache/solr/cluster/api/Config.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cluster.api;
-
-import java.io.InputStream;
-import java.util.Set;
-import java.util.function.Consumer;
-
-import org.apache.solr.common.SolrException;
-
-public interface Config {
-
- String name();
-
- /**set of files in the config */
- Set<String> resources() throws SolrException;
-
- /** read a file inside the config.
- * The caller should consume the stream completely and should not hold a reference to this stream.
- * This method closes the stream soon after the method returns
- * @param file name of the file e.g: schema.xml
- */
- void resource(Consumer<InputStream> file) throws SolrException;
-
-}
diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/Resource.java b/solr/solrj/src/java/org/apache/solr/cluster/api/Resource.java
new file mode 100644
index 0000000..d9624d7
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/Resource.java
@@ -0,0 +1,20 @@
+package org.apache.solr.cluster.api;
+
+import org.apache.solr.common.SolrException;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**A binary resource. The system is agnostic of the content type */
+public interface Resource {
+ /** read a file/resource.
+ * The caller should consume the stream completely and should not hold a reference to this stream.
+ * This method closes the stream soon after the method returns
+ * @param resourceConsumer This should be a full path. e.g schema.xml , solrconfig.xml , lang/stopwords.txt etc
+ */
+ void get(Consumer resourceConsumer) throws SolrException;
+
+ interface Consumer {
+ void read(InputStream is) throws IOException;
+ }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/Shard.java b/solr/solrj/src/java/org/apache/solr/cluster/api/Shard.java
index 98cf027..d618fcd 100644
--- a/solr/solrj/src/java/org/apache/solr/cluster/api/Shard.java
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/Shard.java
@@ -17,8 +17,6 @@
package org.apache.solr.cluster.api;
-import org.apache.solr.common.util.SimpleMap;
-
/**A shard of a collection */
public interface Shard {
diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/ShardReplica.java b/solr/solrj/src/java/org/apache/solr/cluster/api/ShardReplica.java
index cd1822a..1afaea5 100644
--- a/solr/solrj/src/java/org/apache/solr/cluster/api/ShardReplica.java
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/ShardReplica.java
@@ -39,9 +39,12 @@ public interface ShardReplica {
/** type of the replica */
Replica.Type type();
+ /** Is the replica alive now */
+ boolean alive();
+
/**Size of the index in bytes. Keep in mind that this may result in a network call.
* Also keep in mind that the value that you get is at best an approximation.
* The exact size may vary from replica to replica
*/
- long size();
+ long indexSize();
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/SimpleMap.java b/solr/solrj/src/java/org/apache/solr/cluster/api/SimpleMap.java
similarity index 73%
rename from solr/solrj/src/java/org/apache/solr/common/util/SimpleMap.java
rename to solr/solrj/src/java/org/apache/solr/cluster/api/SimpleMap.java
index 544083d..473da05 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/SimpleMap.java
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/SimpleMap.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.solr.common.util;
+package org.apache.solr.cluster.api;
import org.apache.solr.common.MapWriter;
@@ -33,45 +33,39 @@ import java.util.function.Function;
public interface SimpleMap<T> extends MapWriter {
/**get a value by key. If not present , null is returned */
- T get(CharSequence key);
+ T get(String key);
/**Navigate through all keys and values */
- void forEach(BiConsumer<? super CharSequence, ? super T> fun);
+ void forEachEntry(BiConsumer<String, ? super T> fun);
/** iterate through all keys
* The default impl is suboptimal. Proper implementations must do it more efficiently
* */
- default void forEachKey(Consumer<CharSequence> fun) {
- forEach((k, t) -> fun.accept(k));
+ default void forEachKey(Consumer<String> fun) {
+ forEachEntry((k, t) -> fun.accept(k));
}
+ int size();
/**
- * iterate through all keys but stop in between if required
+ * iterate through all keys but abort in between if required
* The default impl is suboptimal. Proper implementations must do it more efficiently
* @param fun Consume each key and return a boolean to signal whether to proceed or not. If true , continue. If false stop
* */
- default void conditionalForEachKey(Function<? super CharSequence, Boolean> fun){
- forEachKey(new Consumer<>() {
- boolean end = false;
- @Override
- public void accept(CharSequence k) {
- if (end) return;
- end = fun.apply(k);
- }
- });
+ default void abortableForEachKey(Function<String, Boolean> fun) {
+ abortableForEach((key, t) -> fun.apply(key));
}
/**
- * Navigate through all keys but stop in between if required.
+ * Navigate through all key-values but abort in between if required.
* The default impl is suboptimal. Proper implementations must do it more efficiently
* @param fun Consume each entry and return a boolean to signal whether to proceed or not. If true, continue, if false stop
*/
- default void conditionalForEach(BiFunction<? super CharSequence, ? super T, Boolean> fun) {
- forEach(new BiConsumer<>() {
+ default void abortableForEach(BiFunction<String, ? super T, Boolean> fun) {
+ forEachEntry(new BiConsumer<>() {
boolean end = false;
@Override
- public void accept(CharSequence k, T v) {
+ public void accept(String k, T v) {
if (end) return;
end = fun.apply(k, v);
}
@@ -81,6 +75,6 @@ public interface SimpleMap<T> extends MapWriter {
@Override
default void writeMap(EntryWriter ew) throws IOException {
- forEach(ew::putNoEx);
+ forEachEntry(ew::putNoEx);
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCluster.java b/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCluster.java
index 9ce578d..794309a 100644
--- a/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCluster.java
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCluster.java
@@ -18,7 +18,6 @@
package org.apache.solr.cluster.api;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.util.SimpleMap;
/** Represents a Solr cluster */
@@ -35,7 +34,7 @@ public interface SolrCluster {
/** Config sets in the cluster*/
- SimpleMap<Config> configs() throws SolrException;
+ SimpleMap<CollectionConfig> configs() throws SolrException;
/** Name of the node in which the overseer is running */
String overseerNode() throws SolrException;
diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCollection.java b/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCollection.java
index 712f210..04eeee7 100644
--- a/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCollection.java
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/SolrCollection.java
@@ -17,8 +17,6 @@
package org.apache.solr.cluster.api;
-import org.apache.solr.common.util.SimpleMap;
-
/** Represents a collection in Solr */
public interface SolrCollection {
@@ -27,8 +25,8 @@ public interface SolrCollection {
/** shards of a collection */
SimpleMap<Shard> shards();
- /** configset used by this collection */
- String configSet();
+ /** Name of the configset used by this collection */
+ String config();
/**Router used in this collection */
Router router();
diff --git a/solr/solrj/src/java/org/apache/solr/cluster/api/SolrNode.java b/solr/solrj/src/java/org/apache/solr/cluster/api/SolrNode.java
index 733bfec..67547ca 100644
--- a/solr/solrj/src/java/org/apache/solr/cluster/api/SolrNode.java
+++ b/solr/solrj/src/java/org/apache/solr/cluster/api/SolrNode.java
@@ -17,8 +17,6 @@
package org.apache.solr.cluster.api;
-import org.apache.solr.common.util.SimpleMap;
-
/** A read only view of a Solr node */
public interface SolrNode {
diff --git a/solr/solrj/src/java/org/apache/solr/common/LazySolrCluster.java b/solr/solrj/src/java/org/apache/solr/common/LazySolrCluster.java
new file mode 100644
index 0000000..3daca03
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/LazySolrCluster.java
@@ -0,0 +1,414 @@
+package org.apache.solr.common;
+
+import org.apache.solr.cluster.api.*;
+import org.apache.solr.common.cloud.*;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.common.util.WrappedSimpleMap;
+import org.apache.zookeeper.KeeperException;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiConsumer;
+
+import static org.apache.solr.common.cloud.ZkStateReader.URL_SCHEME;
+import static org.apache.solr.common.cloud.ZkStateReader.getCollectionPathRoot;
+
+/**
+ * Reference implementation for SolrCluster.
+ * As much as possible fetch all the values lazily because the value of anything
+ * can change any moment
+ * Creating an instance is a low cost operation. It does not result in a
+ * network call or large object creation
+ *
+ */
+
+public class LazySolrCluster implements SolrCluster {
+ final ZkStateReader zkStateReader;
+
+ private final Map<String, SolrCollectionImpl> cached = new ConcurrentHashMap<>();
+ private final SimpleMap<SolrCollection> collections;
+ private final SimpleMap<SolrCollection> collectionsAndAliases;
+ private final SimpleMap<SolrNode> nodes;
+ private SimpleMap<CollectionConfig> configs;
+
+ public LazySolrCluster(ZkStateReader zkStateReader) {
+ this.zkStateReader = zkStateReader;
+ collections = lazyCollectionsMap(zkStateReader);
+ collectionsAndAliases = lazyCollectionsWithAlias(zkStateReader);
+ nodes = lazyNodeMap();
+ }
+
+ private SimpleMap<CollectionConfig> lazyConfigMap() {
+ Set<String> configNames = new HashSet<>();
+ new SimpleZkMap(zkStateReader, ZkStateReader.CONFIGS_ZKNODE)
+ .abortableForEach((name, resource) -> {
+ if (!name.contains("/")) {
+ configNames.add(name);
+ return Boolean.TRUE;
+ }
+ return Boolean.FALSE;
+ });
+
+ return new SimpleMap<>() {
+ @Override
+ public CollectionConfig get(String key) {
+ if (configNames.contains(key)) {
+ return new ConfigImpl(key);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super CollectionConfig> fun) {
+ for (String name : configNames) {
+ fun.accept(name, new ConfigImpl(name));
+ }
+ }
+
+ @Override
+ public int size() {
+ return configNames.size();
+ }
+ };
+ }
+
+ private SimpleMap<SolrNode> lazyNodeMap() {
+ return new SimpleMap<>() {
+ @Override
+ public SolrNode get(String key) {
+ if (!zkStateReader.getClusterState().liveNodesContain(key)) {
+ return null;
+ }
+ return new Node(key);
+ }
+
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super SolrNode> fun) {
+ for (String s : zkStateReader.getClusterState().getLiveNodes()) {
+ fun.accept(s, new Node(s));
+ }
+ }
+
+ @Override
+ public int size() {
+ return zkStateReader.getClusterState().getLiveNodes().size();
+ }
+ };
+ }
+
+ private SimpleMap<SolrCollection> lazyCollectionsWithAlias(ZkStateReader zkStateReader) {
+ return new SimpleMap<>() {
+ @Override
+ public SolrCollection get(String key) {
+ SolrCollection result = collections.get(key);
+ if (result != null) return result;
+ Aliases aliases = zkStateReader.getAliases();
+ List<String> aliasNames = aliases.resolveAliases(key);
+ if (aliasNames == null || aliasNames.isEmpty()) return null;
+ return _collection(aliasNames.get(0), null);
+ }
+
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super SolrCollection> fun) {
+ collections.forEachEntry(fun);
+ Aliases aliases = zkStateReader.getAliases();
+ aliases.forEachAlias((s, colls) -> {
+ if (colls == null || colls.isEmpty()) return;
+ fun.accept(s, _collection(colls.get(0), null));
+ });
+
+ }
+
+ @Override
+ public int size() {
+ return collections.size() + zkStateReader.getAliases().size();
+ }
+ };
+ }
+
+ private SimpleMap<SolrCollection> lazyCollectionsMap(ZkStateReader zkStateReader) {
+ return new SimpleMap<>() {
+ @Override
+ public SolrCollection get(String key) {
+ return _collection(key, null);
+ }
+
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super SolrCollection> fun) {
+ zkStateReader.getClusterState().forEachCollection(coll -> fun.accept(coll.getName(), _collection(coll.getName(), coll)));
+ }
+
+ @Override
+ public int size() {
+ return zkStateReader.getClusterState().size();
+ }
+ };
+ }
+
+ private SolrCollection _collection(String key, DocCollection c) {
+ if (c == null) c = zkStateReader.getCollection(key);
+ if (c == null) {
+ cached.remove(key);
+ return null;
+ }
+ SolrCollectionImpl existing = cached.get(key);
+ if (existing == null || existing.coll != c) {
+ cached.put(key, existing = new SolrCollectionImpl(c, zkStateReader));
+ }
+ return existing;
+ }
+
+ @Override
+ public SimpleMap<SolrCollection> collections() throws SolrException {
+ return collections;
+ }
+
+ @Override
+ public SimpleMap<SolrCollection> collections(boolean includeAlias) throws SolrException {
+ return includeAlias ? collectionsAndAliases : collections;
+ }
+
+ @Override
+ public SimpleMap<SolrNode> nodes() throws SolrException {
+ return nodes;
+ }
+
+ @Override
+ public SimpleMap<CollectionConfig> configs() throws SolrException {
+ if (configs == null) {
+ //these are lightweight objects and we don't care even if multiple objects ar ecreated b/c of a race condition
+ configs = lazyConfigMap();
+ }
+ return configs;
+ }
+
+ @Override
+ public String overseerNode() throws SolrException {
+ return null;
+ }
+
+ @Override
+ public String thisNode() {
+ return null;
+ }
+
+ private class SolrCollectionImpl implements SolrCollection {
+ final DocCollection coll;
+ final SimpleMap<Shard> shards;
+ final ZkStateReader zkStateReader;
+ final Router router;
+ String confName;
+
+ private SolrCollectionImpl(DocCollection coll, ZkStateReader zkStateReader) {
+ this.coll = coll;
+ this.zkStateReader = zkStateReader;
+ this.router = key -> coll.getRouter().getTargetSlice(key, null, null, null, null).getName();
+ LinkedHashMap<String, Shard> map = new LinkedHashMap<>();
+ for (Slice slice : coll.getSlices()) {
+ map.put(slice.getName(), new ShardImpl(this, slice));
+ }
+ shards = new WrappedSimpleMap<>(map);
+
+ }
+
+ @Override
+ public String name() {
+ return coll.getName();
+ }
+
+ @Override
+ public SimpleMap<Shard> shards() {
+ return shards;
+ }
+
+ @Override
+ @SuppressWarnings("rawtypes")
+ public String config() {
+ if (confName == null) {
+ // do this lazily . It's usually not necessary
+ try {
+ byte[] d = zkStateReader.getZkClient().getData(getCollectionPathRoot(coll.getName()), null, null, true);
+ if (d == null || d.length == 0) return null;
+ Map m = (Map) Utils.fromJSON(d);
+ confName = (String) m.get("configName");
+ } catch (KeeperException | InterruptedException e) {
+ SimpleZkMap.throwZkExp(e);
+ //cannot read from ZK
+ return null;
+
+ }
+ }
+ return confName;
+ }
+
+ @Override
+ public Router router() {
+ return router;
+ }
+ }
+
+ private class ShardImpl implements Shard {
+ final SolrCollectionImpl collection;
+ final Slice slice;
+ final HashRange range;
+ final SimpleMap<ShardReplica> replicas;
+
+ private ShardImpl(SolrCollectionImpl collection, Slice slice) {
+ this.collection = collection;
+ this.slice = slice;
+ range = _range(slice);
+ replicas = _replicas();
+ }
+
+ private SimpleMap<ShardReplica> _replicas() {
+ Map<String, ShardReplica> replicas = new HashMap<>();
+ slice.forEach(replica -> replicas.put(replica.getName(), new ShardReplicaImpl(ShardImpl.this, replica)));
+ return new WrappedSimpleMap<>(replicas);
+ }
+
+ private HashRange _range(Slice slice) {
+ return slice.getRange() == null ?
+ null :
+ new HashRange() {
+ @Override
+ public int min() {
+ return slice.getRange().min;
+ }
+
+ @Override
+ public int max() {
+ return slice.getRange().max;
+ }
+ };
+ }
+
+ @Override
+ public String name() {
+ return slice.getName();
+ }
+
+ @Override
+ public String collection() {
+ return collection.name();
+ }
+
+ @Override
+ public HashRange range() {
+ return range;
+ }
+
+ @Override
+ public SimpleMap<ShardReplica> replicas() {
+ return replicas;
+ }
+
+ @Override
+ public String leader() {
+ Replica leader = slice.getLeader();
+ return leader == null ? null : leader.getName();
+ }
+ }
+
+ private class ShardReplicaImpl implements ShardReplica {
+ private final ShardImpl shard;
+ private final Replica replica;
+
+ private ShardReplicaImpl(ShardImpl shard, Replica replica) {
+ this.shard = shard;
+ this.replica = replica;
+ }
+
+ @Override
+ public String name() {
+ return replica.name;
+ }
+
+ @Override
+ public String shard() {
+ return shard.name();
+ }
+
+ @Override
+ public String collection() {
+ return shard.collection.name();
+ }
+
+ @Override
+ public String node() {
+ return replica.getNodeName();
+ }
+
+ @Override
+ public String core() {
+ return replica.getCoreName();
+ }
+
+ @Override
+ public Replica.Type type() {
+ return replica.type;
+ }
+
+ @Override
+ public boolean alive() {
+ return zkStateReader.getClusterState().getLiveNodes().contains(node())
+ && replica.getState() == Replica.State.ACTIVE;
+ }
+
+ @Override
+ public long indexSize() {
+ //todo implement later
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+ }
+
+ private class Node implements SolrNode {
+ private final String name;
+
+ private Node(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String name() {
+ return name;
+ }
+
+ @Override
+ public String baseUrl(boolean isV2) {
+ return Utils.getBaseUrlForNodeName(name, zkStateReader.getClusterProperty(URL_SCHEME, "http"), isV2);
+ }
+
+ @Override
+ public SimpleMap<ShardReplica> cores() {
+ //todo implement later
+ //this requires a call to the node
+ throw new UnsupportedOperationException("Not yet implemented");
+ }
+ }
+
+ private class ConfigImpl implements CollectionConfig {
+ final String name;
+ final SimpleMap<Resource> resources;
+ final String path;
+
+ private ConfigImpl(String name) {
+ this.name = name;
+ path = ZkStateReader.CONFIGS_ZKNODE + "/" + name;
+ this.resources = new SimpleZkMap(zkStateReader, path);
+ }
+
+ @Override
+ public SimpleMap<Resource> resources() {
+ return resources;
+ }
+
+ @Override
+ public String name() {
+ return name;
+ }
+
+ }
+
+}
+
diff --git a/solr/solrj/src/java/org/apache/solr/common/SimpleZkMap.java b/solr/solrj/src/java/org/apache/solr/common/SimpleZkMap.java
new file mode 100644
index 0000000..dcb5355
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/SimpleZkMap.java
@@ -0,0 +1,114 @@
+package org.apache.solr.common;
+
+import org.apache.solr.cluster.api.Resource;
+import org.apache.solr.cluster.api.SimpleMap;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.zookeeper.KeeperException;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+
+/**A view of ZK as a {@link SimpleMap} impl. This gives a flat view of all paths instead of a tree view
+ * eg: a, b, c , a/a1, a/a2, a/a1/aa1 etc
+ * If possible, use the {@link #abortableForEach(BiFunction)} to traverse
+ * DO not use the {@link #size()} method. It always return 0 because it is very expensive to compute that
+ *
+ */
+public class SimpleZkMap implements SimpleMap<Resource> {
+ private final ZkStateReader zkStateReader;
+ private final String basePath;
+
+ static final byte[] EMPTY_BYTES = new byte[0];
+
+
+ public SimpleZkMap(ZkStateReader zkStateReader, String path) {
+ this.zkStateReader = zkStateReader;
+ this.basePath = path;
+ }
+
+
+ @Override
+ public Resource get(String key) {
+ return readZkNode(basePath + key);
+ }
+
+ @Override
+ public void abortableForEach(BiFunction<String, ? super Resource, Boolean> fun) {
+ try {
+ recursiveRead("",
+ zkStateReader.getZkClient().getChildren(basePath, null, true),
+ fun);
+ } catch (KeeperException | InterruptedException e) {
+ throwZkExp(e);
+ }
+ }
+
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super Resource> fun) {
+ abortableForEach((path, resource) -> {
+ fun.accept(path, resource);
+ return Boolean.TRUE;
+ });
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+
+ private Resource readZkNode(String path) {
+ return consumer -> {
+ try {
+ byte[] data = zkStateReader.getZkClient().getData(basePath+"/"+ path, null, null, true);
+ if (data != null && data.length > 0) {
+ consumer.read(new ByteArrayInputStream(data));
+ } else {
+ consumer.read(new ByteArrayInputStream(EMPTY_BYTES));
+ }
+ } catch (KeeperException | InterruptedException e) {
+ throwZkExp(e);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Can;t read stream" , e);
+ }
+
+ };
+ }
+
+ private boolean recursiveRead(String parent, List<String> childrenList, BiFunction<String, ? super Resource, Boolean> fun) {
+ if(childrenList == null || childrenList.isEmpty()) return true;
+ try {
+ Map<String, List<String>> withKids = new LinkedHashMap<>();
+ for (String child : childrenList) {
+ String relativePath = parent.isBlank() ? child: parent+"/"+child;
+ if(!fun.apply(relativePath, readZkNode(relativePath))) return false;
+ List<String> l1 = zkStateReader.getZkClient().getChildren(basePath+ "/"+ relativePath, null, true);
+ if(l1 != null && !l1.isEmpty()) {
+ withKids.put(relativePath, l1);
+ }
+ }
+ //now we iterate through all nodes with sub paths
+ for (Map.Entry<String, List<String>> e : withKids.entrySet()) {
+ //has children
+ if(!recursiveRead(e.getKey(), e.getValue(), fun)) {
+ return false;
+ }
+ }
+ } catch (KeeperException | InterruptedException e) {
+ throwZkExp(e);
+ }
+ return true;
+ }
+
+ static void throwZkExp(Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ZK errror", e);
+ }
+
+}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
index 45fc9d8..a87735f 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.function.BiConsumer;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
@@ -72,6 +73,13 @@ public class Aliases {
this.zNodeVersion = zNodeVersion;
}
+ public void forEachAlias(BiConsumer<String, List<String>> consumer) {
+ collectionAliases.forEach((s, colls) -> consumer.accept(s, Collections.unmodifiableList(colls)));
+ }
+ public int size() {
+ return collectionAliases.size();
+ }
+
/**
* Create an instance from the JSON bytes read from zookeeper. Generally this should
* only be done by a ZkStateReader.
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
index 3c518e0..ebed0ff 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
@@ -380,4 +380,8 @@ public class ClusterState implements JSONWriter.Writable {
}
+ public int size() {
+ return collectionStates.size();
+ }
+
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index a426c6e..4273163 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -2178,4 +2178,8 @@ public class ZkStateReader implements SolrCloseable {
return result;
}
}
+
+ public DocCollection getCollection(String collection) {
+ return clusterState.getCollectionOrNull(collection);
+ }
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java b/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
index c729534..a450ec4 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
@@ -73,7 +73,7 @@ public abstract class SolrParams implements Serializable, MapWriter, Iterable<Ma
public void writeMap(EntryWriter ew) throws IOException {
//TODO don't call toNamedList; more efficiently implement here
//note: multiple values, if present, are a String[] under 1 key
- toNamedList().forEach((k, v) -> {
+ toNamedList().forEachEntry((k, v) -> {
if (v == null || "".equals(v)) return;
try {
ew.put(k, v);
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java b/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java
index 185b9a7..1fc6afc 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/LinkedSimpleHashMap.java
@@ -16,12 +16,19 @@
*/
package org.apache.solr.common.util;
+import org.apache.solr.cluster.api.SimpleMap;
+
import java.util.LinkedHashMap;
+import java.util.function.BiConsumer;
-public class LinkedSimpleHashMap<T> extends LinkedHashMap<CharSequence, T> implements SimpleMap<T> {
+public class LinkedSimpleHashMap<T> extends LinkedHashMap<String, T> implements SimpleMap<T> {
@Override
- public T get(CharSequence key) {
+ public T get(String key) {
return super.get(key);
}
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super T> fun) {
+
+ }
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
index 694eb93..9156619 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
@@ -30,7 +30,11 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import org.apache.solr.cluster.api.SimpleMap;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MultiMapSolrParams;
@@ -63,7 +67,7 @@ import org.apache.solr.common.params.SolrParams;
*
*/
@SuppressWarnings({"unchecked", "rawtypes"})
-public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> , MapWriter {
+public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> , MapWriter, SimpleMap<T> {
private static final long serialVersionUID = 1957981902839867821L;
protected final List<Object> nvPairs;
@@ -509,7 +513,7 @@ public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry
@Override
public void forEach(BiConsumer action) {
- NamedList.this.forEach(action);
+ NamedList.this.forEachEntry(action);
}
};
}
@@ -854,10 +858,39 @@ public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry
return this.nvPairs.equals(nl.nvPairs);
}
- public void forEach(BiConsumer<String, T> action) {
+
+ @Override
+ public void abortableForEach(BiFunction<String, ? super T, Boolean> fun) {
+ int sz = size();
+ for (int i = 0; i < sz; i++) {
+ if(!fun.apply(getName(i), getVal(i))) break;
+ }
+ }
+
+ @Override
+ public void abortableForEachKey(Function<String, Boolean> fun) {
+ int sz = size();
+ for (int i = 0; i < sz; i++) {
+ if(!fun.apply(getName(i))) break;
+ }
+ }
+
+ @Override
+ public void forEachKey(Consumer<String> fun) {
+ int sz = size();
+ for (int i = 0; i < sz; i++) {
+ fun.accept(getName(i));
+ }
+ }
+ public void forEach(BiConsumer<String, ? super T> action) {
int sz = size();
for (int i = 0; i < sz; i++) {
action.accept(getName(i), getVal(i));
}
}
+
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super T> fun) {
+ forEach(fun);
+ }
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 2e68b19..8fd137b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -728,13 +728,16 @@ public class Utils {
}
public static String getBaseUrlForNodeName(final String nodeName, String urlScheme) {
+ return getBaseUrlForNodeName(nodeName, urlScheme, false);
+ }
+ public static String getBaseUrlForNodeName(final String nodeName, String urlScheme, boolean isV2) {
final int _offset = nodeName.indexOf("_");
if (_offset < 0) {
throw new IllegalArgumentException("nodeName does not contain expected '_' separator: " + nodeName);
}
final String hostAndPort = nodeName.substring(0, _offset);
final String path = URLDecoder.decode(nodeName.substring(1 + _offset), UTF_8);
- return urlScheme + "://" + hostAndPort + (path.isEmpty() ? "" : ("/" + path));
+ return urlScheme + "://" + hostAndPort + (path.isEmpty() ? "" : ("/" + (isV2? "api": path)));
}
public static long time(TimeSource timeSource, TimeUnit unit) {
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/WrappedSimpleMap.java b/solr/solrj/src/java/org/apache/solr/common/util/WrappedSimpleMap.java
new file mode 100644
index 0000000..3a43305
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/util/WrappedSimpleMap.java
@@ -0,0 +1,32 @@
+package org.apache.solr.common.util;
+
+import org.apache.solr.cluster.api.SimpleMap;
+
+import java.util.Map;
+import java.util.function.BiConsumer;
+
+public class WrappedSimpleMap<T> implements SimpleMap<T> {
+ private final Map<String, T> delegate;
+
+ @Override
+ public T get(String key) {
+ return delegate.get(key);
+ }
+
+ @Override
+ public void forEachEntry(BiConsumer<String, ? super T> fun) {
+ delegate.forEach(fun);
+
+ }
+
+ @Override
+ public int size() {
+ return delegate.size();
+ }
+
+
+ public WrappedSimpleMap(Map<String, T> delegate) {
+ this.delegate = delegate;
+ }
+
+}