You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/10/08 02:30:23 UTC
[01/30] lucene-solr:jira/http2: SOLR-12792: extract out test data
Repository: lucene-solr
Updated Branches:
refs/heads/jira/http2 bb0b9a144 -> 2d51180bb
SOLR-12792: extract out test data
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/86e00405
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/86e00405
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/86e00405
Branch: refs/heads/jira/http2
Commit: 86e00405b0b79a8ab6494a10ec2bc263ff3485e8
Parents: 93e2ae3
Author: Noble Paul <no...@apache.org>
Authored: Mon Oct 1 22:00:09 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Mon Oct 1 22:00:09 2018 +1000
----------------------------------------------------------------------
.../solr/cloud/CollectionsAPISolrJTest.java | 13 ++-
.../org/apache/solr/common/IteratorWriter.java | 9 ++
.../java/org/apache/solr/common/MapWriter.java | 20 ++++
.../org/apache/solr/common/util/NamedList.java | 10 +-
.../java/org/apache/solr/common/util/Utils.java | 49 +++++++--
.../testMoveReplicasInMultipleCollections.json | 88 +++++++++++++++
.../solrj/cloud/autoscaling/TestPolicy.java | 106 ++-----------------
.../solrj/cloud/autoscaling/TestPolicy2.java | 6 +-
8 files changed, 184 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index f61fa5e..5390704 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
@@ -48,12 +47,12 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.junit.Test;
+import static java.util.Arrays.asList;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
@@ -191,14 +190,14 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
String nodeName = ((NamedList) response.getResponse().get("success")).getName(0);
- String corename = (String) ((NamedList) ((NamedList) response.getResponse().get("success")).getVal(0)).get("core");
+ String corename = (String) response.getResponse()._get(asList("success", nodeName,"core"),null);
try (HttpSolrClient coreclient = getHttpSolrClient(cluster.getSolrClient().getZkStateReader().getBaseUrlForNodeName(nodeName))) {
CoreAdminResponse status = CoreAdminRequest.getStatus(corename, coreclient);
- Map m = status.getResponse().asMap(5);
- assertEquals(collectionName, Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "collection")));
- assertNotNull(Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "shard")));
- assertNotNull(Utils.getObjectByPath(m, true, Arrays.asList("status", corename, "cloud", "replica")));
+ NamedList m = status.getResponse();
+ assertEquals(collectionName, m._get(asList("status", corename, "cloud", "collection"), null));
+ assertNotNull(m._get(asList("status", corename, "cloud", "shard"), null));
+ assertNotNull(m._get(asList("status", corename, "cloud", "replica"), null));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
index cbfb584..ec11c78 100644
--- a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java
@@ -38,6 +38,15 @@ public interface IteratorWriter {
*/
ItemWriter add(Object o) throws IOException;
+ default ItemWriter addNoEx(Object o) {
+ try {
+ add(o);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return this;
+ }
+
default ItemWriter add(int v) throws IOException {
add((Integer) v);
return this;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index 9b1861a..3ec37e7 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.BiConsumer;
import java.util.function.BiPredicate;
import org.apache.solr.common.util.Utils;
@@ -90,6 +91,25 @@ public interface MapWriter extends MapSerializable {
Object v = Utils.getObjectByPath(this, false, path);
return v == null ? def : v;
}
+ default void _forEachEntry(String path, BiConsumer fun) {
+ Utils.forEachMapEntry(this, path, fun);
+ }
+
+ default void _forEachEntry(BiConsumer fun) {
+ Utils.forEachMapEntry(this, null, fun);
+ }
+
+ /**
+ * Get a child object value using json path
+ *
+ * @param path the full path to that object such as ["a","b","c[4]","d"] etc
+ * @param def the default
+ * @return the found value or default
+ */
+ default Object _get(List<String> path, Object def) {
+ Object v = Utils.getObjectByPath(this, false, path);
+ return v == null ? def : v;
+ }
/**
* An interface to push one entry at a time to the output.
* The order of the keys is not defined, but we assume they are distinct -- don't call {@code put} more than once
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
index 1650602..71cb78b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.common.util;
+import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -29,6 +30,7 @@ import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.MultiMapSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -59,7 +61,7 @@ import org.apache.solr.common.params.SolrParams;
* </p>
*
*/
-public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> {
+public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> , MapWriter {
private static final long serialVersionUID = 1957981902839867821L;
protected final List<Object> nvPairs;
@@ -74,6 +76,12 @@ public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry
nvPairs = new ArrayList<>(sz<<1);
}
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ for (int i = 0; i < nvPairs.size(); i+=2) {
+ ew.put((String) nvPairs.get(i), nvPairs.get(i+1));
+ }
+ }
/**
* Creates a NamedList instance containing the "name,value" pairs contained in the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 800c2c1..2d1df42 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -43,6 +43,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -102,6 +103,26 @@ public class Utils {
return mutable ? copy : Collections.unmodifiableMap(copy);
}
+ public static void forEachMapEntry(MapWriter mw, String path, BiConsumer fun) {
+ Object o = path == null ? mw : Utils.getObjectByPath(mw, false, path);
+ if (o instanceof MapWriter) {
+ MapWriter m = (MapWriter) o;
+ try {
+ m.writeMap(new MapWriter.EntryWriter() {
+ @Override
+ public MapWriter.EntryWriter put(String k, Object v) {
+ fun.accept(k, v);
+ return this;
+ }
+ });
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else if (o instanceof Map) {
+ ((Map) o).forEach((k, v) -> fun.accept(k, v));
+ }
+ }
+
private static Object makeDeepCopy(Object v, int maxDepth, boolean mutable, boolean sorted) {
if (v instanceof MapWriter && maxDepth > 1) {
v = ((MapWriter) v).toMap(new LinkedHashMap<>());
@@ -295,7 +316,7 @@ public class Utils {
}
}
if (i < hierarchy.size() - 1) {
- Object o = getVal(obj, s);
+ Object o = getVal(obj, s, -1);
if (o == null) return false;
if (idx > -1) {
List l = (List) o;
@@ -315,7 +336,7 @@ public class Utils {
}
return true;
} else {
- Object v = getVal(obj, s);
+ Object v = getVal(obj, s, -1);
if (v instanceof List) {
List list = (List) v;
if (idx == -1) {
@@ -352,7 +373,7 @@ public class Utils {
}
}
if (i < hierarchy.size() - 1) {
- Object o = getVal(obj, s);
+ Object o = getVal(obj, s, -1);
if (o == null) return null;
if (idx > -1) {
List l = (List) o;
@@ -361,10 +382,13 @@ public class Utils {
if (!isMapLike(o)) return null;
obj = o;
} else {
- Object val = getVal(obj, s);
+ Object val = getVal(obj, s, -1);
if (val == null) return null;
if (idx > -1) {
- if (val instanceof IteratorWriter) {
+ if (val instanceof MapWriter) {
+ val = getVal((MapWriter) val, null, idx);
+
+ } else if (val instanceof IteratorWriter) {
val = getValueAt((IteratorWriter) val, idx);
} else {
List l = (List) val;
@@ -381,6 +405,7 @@ public class Utils {
return false;
}
+
private static Object getValueAt(IteratorWriter iteratorWriter, int idx) {
Object[] result = new Object[1];
try {
@@ -406,14 +431,20 @@ public class Utils {
return o instanceof Map || o instanceof NamedList || o instanceof MapWriter;
}
- private static Object getVal(Object obj, String key) {
+ private static Object getVal(Object obj, String key, int idx) {
if (obj instanceof MapWriter) {
Object[] result = new Object[1];
try {
((MapWriter) obj).writeMap(new MapWriter.EntryWriter() {
+ int count = -1;
@Override
- public MapWriter.EntryWriter put(String k, Object v) throws IOException {
- if (key.equals(k)) result[0] = v;
+ public MapWriter.EntryWriter put(String k, Object v) {
+ if (result[0] != null) return this;
+ if (k != null) {
+ if (key.equals(k)) result[0] = v;
+ } else {
+ if (++count == idx) result[0] = v;
+ }
return this;
}
});
@@ -422,8 +453,6 @@ public class Utils {
}
return result[0];
}
-
- if (obj instanceof NamedList) return ((NamedList) obj).get(key);
else if (obj instanceof Map) return ((Map) obj).get(key);
else throw new RuntimeException("must be a NamedList or Map");
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json
new file mode 100644
index 0000000..16ba1a7
--- /dev/null
+++ b/solr/solrj/src/test-files/solrj/solr/autoscaling/testMoveReplicasInMultipleCollections.json
@@ -0,0 +1,88 @@
+{"collection1":{
+ "pullReplicas":"0",
+ "replicationFactor":"2",
+ "shards":{
+ "shard1":{
+ "range":"80000000-ffffffff",
+ "state":"active",
+ "replicas":{
+ "core_node1":{
+ "core":"collection1_shard1_replica_n1",
+ "base_url":"http://127.0.0.1:51650/solr",
+ "node_name":"node1",
+ "state":"active",
+ "type":"NRT",
+ "leader":"true"},
+ "core_node6":{
+ "core":"collection1_shard1_replica_n3",
+ "base_url":"http://127.0.0.1:51651/solr",
+ "node_name":"node3",
+ "state":"active",
+ "type":"NRT"}}},
+ "shard2":{
+ "range":"0-7fffffff",
+ "state":"active",
+ "replicas":{
+ "core_node3":{
+ "core":"collection1_shard2_replica_n1",
+ "base_url":"http://127.0.0.1:51650/solr",
+ "node_name":"node1",
+ "state":"active",
+ "type":"NRT",
+ "leader":"true"},
+ "core_node5":{
+ "core":"collection1_shard2_replica_n3",
+ "base_url":"http://127.0.0.1:51651/solr",
+ "node_name":"node3",
+ "state":"active",
+ "type":"NRT"}}}},
+ "router":{
+ "name":"compositeId"},
+ "maxShardsPerNode":"2",
+ "autoAddReplicas":"true",
+ "nrtReplicas":"2",
+ "tlogReplicas":"0"},
+ "collection2":{
+ "pullReplicas":"0",
+ "replicationFactor":"2",
+ "shards":{
+ "shard1":{
+ "range":"80000000-ffffffff",
+ "state":"active",
+ "replicas":{
+ "core_node1":{
+ "core":"collection2_shard1_replica_n1",
+ "base_url":"http://127.0.0.1:51649/solr",
+ "node_name":"node2",
+ "state":"active",
+ "type":"NRT"},
+ "core_node2":{
+ "core":"collection2_shard1_replica_n2",
+ "base_url":"http://127.0.0.1:51651/solr",
+ "node_name":"node3",
+ "state":"active",
+ "type":"NRT",
+ "leader":"true"}}},
+ "shard2":{
+ "range":"0-7fffffff",
+ "state":"active",
+ "replicas":{
+ "core_node3":{
+ "core":"collection2_shard2_replica_n1",
+ "base_url":"http://127.0.0.1:51649/solr",
+ "node_name":"node2",
+ "state":"active",
+ "type":"NRT"},
+ "core_node4":{
+ "core":"collection2_shard2_replica_n2",
+ "base_url":"http://127.0.0.1:51651/solr",
+ "node_name":"node3",
+ "state":"active",
+ "type":"NRT",
+ "leader":"true"}}}},
+ "router":{
+ "name":"compositeId"},
+ "maxShardsPerNode":"2",
+ "autoAddReplicas":"true",
+ "nrtReplicas":"2",
+ "tlogReplicas":"0"}}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 9c5be7f..29e45ed 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -143,9 +143,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
" 'node_name':'node1'," +
" 'state':'active'}}}}}}";
- public static Map<String, Map<String, List<ReplicaInfo>>> getReplicaDetails(String node, String clusterState) {
+ public static Map<String, Map<String, List<ReplicaInfo>>> getReplicaDetails(String node, Map clusterState) {
ValidatingJsonMap m = ValidatingJsonMap
- .getDeepCopy((Map) Utils.fromJSONString(clusterState), 6, true);
+ .getDeepCopy(clusterState, 6, true);
Map<String, Map<String, List<ReplicaInfo>>> result = new LinkedHashMap<>();
m.forEach((collName, o) -> {
@@ -1426,101 +1426,14 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
- public void testMoveReplicasInMultipleCollections() {
+ public void testMoveReplicasInMultipleCollections() throws IOException {
Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
"node1:{cores:2}," +
"node3:{cores:4}" +
"}");
- String clusterState = "{\n" +
- "'collection1' : {\n" +
- " 'pullReplicas':'0',\n" +
- " 'replicationFactor':'2',\n" +
- " 'shards':{\n" +
- " 'shard1':{\n" +
- " 'range':'80000000-ffffffff',\n" +
- " 'state':'active',\n" +
- " 'replicas':{\n" +
- " 'core_node1':{\n" +
- " 'core':'collection1_shard1_replica_n1',\n" +
- " 'base_url':'http://127.0.0.1:51650/solr',\n" +
- " 'node_name':'node1',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT',\n" +
- " 'leader':'true'},\n" +
- " 'core_node6':{\n" +
- " 'core':'collection1_shard1_replica_n3',\n" +
- " 'base_url':'http://127.0.0.1:51651/solr',\n" +
- " 'node_name':'node3',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT'}}},\n" +
- " 'shard2':{\n" +
- " 'range':'0-7fffffff',\n" +
- " 'state':'active',\n" +
- " 'replicas':{\n" +
- " 'core_node3':{\n" +
- " 'core':'collection1_shard2_replica_n1',\n" +
- " 'base_url':'http://127.0.0.1:51650/solr',\n" +
- " 'node_name':'node1',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT',\n" +
- " 'leader':'true'},\n" +
- " 'core_node5':{\n" +
- " 'core':'collection1_shard2_replica_n3',\n" +
- " 'base_url':'http://127.0.0.1:51651/solr',\n" +
- " 'node_name':'node3',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT'}}}},\n" +
- " 'router':{'name':'compositeId'},\n" +
- " 'maxShardsPerNode':'2',\n" +
- " 'autoAddReplicas':'true',\n" +
- " 'nrtReplicas':'2',\n" +
- " 'tlogReplicas':'0'},\n" +
- "'collection2' : {\n" +
- " 'pullReplicas':'0',\n" +
- " 'replicationFactor':'2',\n" +
- " 'shards':{\n" +
- " 'shard1':{\n" +
- " 'range':'80000000-ffffffff',\n" +
- " 'state':'active',\n" +
- " 'replicas':{\n" +
- " 'core_node1':{\n" +
- " 'core':'collection2_shard1_replica_n1',\n" +
- " 'base_url':'http://127.0.0.1:51649/solr',\n" +
- " 'node_name':'node2',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT'},\n" +
- " 'core_node2':{\n" +
- " 'core':'collection2_shard1_replica_n2',\n" +
- " 'base_url':'http://127.0.0.1:51651/solr',\n" +
- " 'node_name':'node3',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT',\n" +
- " 'leader':'true'}}},\n" +
- " 'shard2':{\n" +
- " 'range':'0-7fffffff',\n" +
- " 'state':'active',\n" +
- " 'replicas':{\n" +
- " 'core_node3':{\n" +
- " 'core':'collection2_shard2_replica_n1',\n" +
- " 'base_url':'http://127.0.0.1:51649/solr',\n" +
- " 'node_name':'node2',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT'},\n" +
- " 'core_node4':{\n" +
- " 'core':'collection2_shard2_replica_n2',\n" +
- " 'base_url':'http://127.0.0.1:51651/solr',\n" +
- " 'node_name':'node3',\n" +
- " 'state':'active',\n" +
- " 'type':'NRT',\n" +
- " 'leader':'true'}}}},\n" +
- " 'router':{'name':'compositeId'},\n" +
- " 'maxShardsPerNode':'2',\n" +
- " 'autoAddReplicas':'true',\n" +
- " 'nrtReplicas':'2',\n" +
- " 'tlogReplicas':'0'}\n" +
- "}";
Policy policy = new Policy(new HashMap<>());
- Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues, clusterState))
+ Suggester suggester = policy.createSession(getSolrCloudManager(nodeValues,
+ (Map<String, Object>) TestPolicy2.loadFromResource("testMoveReplicasInMultipleCollections.json")))
.getSuggester(MOVEREPLICA)
.hint(Hint.COLL, "collection1")
.hint(Hint.COLL, "collection2")
@@ -2083,8 +1996,11 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(op);
assertEquals("node2", op.getNode());
}
+ static SolrCloudManager getSolrCloudManager(final Map<String, Map> nodeValues, String clusterS) {
+ return getSolrCloudManager(nodeValues,(Map) Utils.fromJSONString(clusterS));
- private SolrCloudManager getSolrCloudManager(final Map<String, Map> nodeValues, String clusterS) {
+ }
+ private static SolrCloudManager getSolrCloudManager(final Map<String, Map> nodeValues, Map clusterS) {
return new SolrCloudManager() {
ObjectCache objectCache = new ObjectCache();
@@ -2188,7 +2104,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
@Override
public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return getReplicaDetails(node, clusterState);
+ return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState));
}
};
}
@@ -2240,7 +2156,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
@Override
public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return getReplicaDetails(node, clusterState);
+ return getReplicaDetails(node, (Map)Utils.fromJSONString(clusterState));
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/86e00405/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index 615fd4d..07ce391 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -201,9 +201,9 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson)),
createCloudManager(state, metaData));
assertEquals(1, suggestions.size());
- assertEquals("node5", Utils.getObjectByPath(suggestions.get(0).operation, true, "command/move-replica/targetNode"));
+ assertEquals("node5", suggestions.get(0)._get("operation/command/move-replica/targetNode", null));
- String rName = (String) Utils.getObjectByPath(suggestions.get(0).operation, true, "command/move-replica/replica");
+ String rName = (String) suggestions.get(0)._get("operation/command/move-replica/replica", null);
found.set(false);
session.getNode("node1").forEachReplica(replicaInfo -> {
@@ -412,8 +412,6 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
assertEquals("10.0.0.79:8983_solr", suggestion._get("operation/command/move-replica/targetNode",null));
}
-
-
}
public static Object loadFromResource(String file) throws IOException {
[22/30] lucene-solr:jira/http2: SOLR-12814: Metrics history causing
"HttpParser URI is too large >8192" when many collections This fixes #461
Posted by da...@apache.org.
SOLR-12814: Metrics history causing "HttpParser URI is too large >8192" when many collections
This fixes #461
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5fb384c9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5fb384c9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5fb384c9
Branch: refs/heads/jira/http2
Commit: 5fb384c9898176d34fffe2b310a0a815d8aebecb
Parents: 36c6025
Author: Jan Høydahl <ja...@apache.org>
Authored: Thu Oct 4 18:32:38 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Thu Oct 4 18:32:38 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fb384c9/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 981881a..ae20fc3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -160,6 +160,8 @@ Bug Fixes
create inconsistencies between replicas of the same shard. min_rf parameter is now deprecated, and even if provided
replicas that don't ack an update from the leader will be marked for recovery. (Tomás Fernández Löbbe)
+* SOLR-12814: Metrics history causing "HttpParser URI is too large >8192" when many collections (janhoy)
+
Improvements
----------------------
* SOLR-12767: Solr now always includes in the response of update requests the achieved replication factor
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fb384c9/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index d148e21..5a9d5b0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -333,7 +333,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
throws IOException, SolrServerException {
String url = zkClientClusterStateProvider.getZkStateReader().getBaseUrlForNodeName(solrNode);
- GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.GET, path, params);
+ GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.POST, path, params);
try (HttpSolrClient client = new HttpSolrClient.Builder()
.withHttpClient(solrClient.getHttpClient())
.withBaseSolrUrl(url)
[02/30] lucene-solr:jira/http2: LUCENE-8513: Remove
MultiFields.getFields SlowCompositeReaderWrapper now works with MultiTerms
directly
Posted by da...@apache.org.
LUCENE-8513: Remove MultiFields.getFields
SlowCompositeReaderWrapper now works with MultiTerms directly
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fe844c73
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fe844c73
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fe844c73
Branch: refs/heads/jira/http2
Commit: fe844c739b1bd24090789acad099d3b3e5c12d4e
Parents: 86e0040
Author: David Smiley <ds...@apache.org>
Authored: Mon Oct 1 10:39:12 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Mon Oct 1 10:39:12 2018 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../benchmark/byTask/TestPerfTasksLogic.java | 9 +--
.../codecs/perfield/PerFieldPostingsFormat.java | 14 ++--
.../org/apache/lucene/index/MultiFields.java | 68 +-------------------
.../lucene/index/TestDirectoryReader.java | 11 ++--
.../org/apache/lucene/index/TestDocCount.java | 6 +-
.../index/TestParallelCompositeReader.java | 27 ++++----
.../apache/lucene/index/TestSegmentReader.java | 4 +-
.../lucene/index/TestStressIndexing2.java | 26 +++-----
.../org/apache/lucene/index/TestSumDocFreq.java | 7 +-
.../memory/TestMemoryIndexAgainstRAMDir.java | 2 +-
.../org/apache/lucene/misc/HighFreqTerms.java | 20 +++---
.../index/BaseIndexFileFormatTestCase.java | 22 ++++++-
.../org/apache/lucene/util/LuceneTestCase.java | 67 +++++++++----------
solr/CHANGES.txt | 2 +
.../solr/index/SlowCompositeReaderWrapper.java | 46 ++++++++-----
.../index/TestSlowCompositeReaderWrapper.java | 29 ++++++++-
17 files changed, 179 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3b10c16..3f23c1e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -84,6 +84,9 @@ API Changes
* LUCENE-8498: LowerCaseTokenizer has been removed, and CharTokenizer no longer
takes a normalizer function. (Alan Woodward)
+* LUCENE-8513: MultiFields.getFields is now removed. Please avoid this class,
+ and Fields in general, when possible. (David Smiley)
+
Changes in Runtime Behavior
* LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
----------------------------------------------------------------------
diff --git a/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java b/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
index 3d483f3..d0af909 100644
--- a/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
+++ b/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
@@ -21,6 +21,7 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.Collator;
+import java.util.Collection;
import java.util.List;
import java.util.Locale;
@@ -37,15 +38,15 @@ import org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTask;
import org.apache.lucene.collation.CollationKeyAnalyzer;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.Terms;
@@ -373,13 +374,13 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
int totalTokenCount2 = 0;
- Fields fields = MultiFields.getFields(reader);
+ Collection<String> fields = MultiFields.getIndexedFields(reader);
for (String fieldName : fields) {
if (fieldName.equals(DocMaker.ID_FIELD) || fieldName.equals(DocMaker.DATE_MSEC_FIELD) || fieldName.equals(DocMaker.TIME_SEC_FIELD)) {
continue;
}
- Terms terms = fields.terms(fieldName);
+ Terms terms = MultiFields.getTerms(reader, fieldName);
if (terms == null) {
continue;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
index 36f0358..9ac0fe2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
@@ -20,6 +20,7 @@ package org.apache.lucene.codecs.perfield;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -27,7 +28,7 @@ import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.ServiceLoader; // javadocs
+import java.util.ServiceLoader;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -41,13 +42,13 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FilterLeafReader.FilterFields;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.MergedIterator;
import org.apache.lucene.util.RamUsageEstimator;
/**
@@ -150,7 +151,10 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
@Override
public void merge(MergeState mergeState, NormsProducer norms) throws IOException {
- Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(new MultiFields(mergeState.fieldsProducers, null));
+ @SuppressWarnings("unchecked") Iterable<String> indexedFieldNames = () ->
+ new MergedIterator<>(true,
+ Arrays.stream(mergeState.fieldsProducers).map(FieldsProducer::iterator).toArray(Iterator[]::new));
+ Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(indexedFieldNames);
// Merge postings
PerFieldMergeState pfMergeState = new PerFieldMergeState(mergeState);
@@ -173,7 +177,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
}
}
- private Map<PostingsFormat, FieldsGroup> buildFieldsGroupMapping(Fields fields) {
+ private Map<PostingsFormat, FieldsGroup> buildFieldsGroupMapping(Iterable<String> indexedFieldNames) {
// Maps a PostingsFormat instance to the suffix it
// should use
Map<PostingsFormat,FieldsGroup> formatToGroups = new HashMap<>();
@@ -182,7 +186,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
Map<String,Integer> suffixes = new HashMap<>();
// Assign field -> PostingsFormat
- for(String field : fields) {
+ for(String field : indexedFieldNames) {
FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(field);
final PostingsFormat format = getPostingsFormatForField(field);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
index 32ce2fa..49a85a3 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -47,45 +46,13 @@ import org.apache.lucene.util.MergedIterator;
* atomic leaves and then operate per-LeafReader,
* instead of using this class.
*
- * @lucene.experimental
+ * @lucene.internal
*/
public final class MultiFields extends Fields {
private final Fields[] subs;
private final ReaderSlice[] subSlices;
private final Map<String,Terms> terms = new ConcurrentHashMap<>();
- /** Returns a single {@link Fields} instance for this
- * reader, merging fields/terms/docs/positions on the
- * fly. This method will return null if the reader
- * has no postings.
- *
- * <p><b>NOTE</b>: this is a slow way to access postings.
- * It's better to get the sub-readers and iterate through them
- * yourself. */
- public static Fields getFields(IndexReader reader) throws IOException {
- final List<LeafReaderContext> leaves = reader.leaves();
- switch (leaves.size()) {
- case 1:
- // already an atomic reader / reader with one leave
- return new LeafReaderFields(leaves.get(0).reader());
- default:
- final List<Fields> fields = new ArrayList<>(leaves.size());
- final List<ReaderSlice> slices = new ArrayList<>(leaves.size());
- for (final LeafReaderContext ctx : leaves) {
- final LeafReader r = ctx.reader();
- final Fields f = new LeafReaderFields(r);
- fields.add(f);
- slices.add(new ReaderSlice(ctx.docBase, r.maxDoc(), fields.size()-1));
- }
- if (fields.size() == 1) {
- return fields.get(0);
- } else {
- return new MultiFields(fields.toArray(Fields.EMPTY_ARRAY),
- slices.toArray(ReaderSlice.EMPTY_ARRAY));
- }
- }
- }
-
/** Returns a single {@link Bits} instance for this
* reader, merging live Documents on the
* fly. This method will return null if the reader
@@ -133,7 +100,7 @@ public final class MultiFields extends Fields {
Terms subTerms = ctx.reader().terms(field);
if (subTerms != null) {
termsPerLeaf.add(subTerms);
- slicePerLeaf.add(new ReaderSlice(ctx.docBase, r.maxDoc(), leafIdx - 1));
+ slicePerLeaf.add(new ReaderSlice(ctx.docBase, r.maxDoc(), leafIdx));
}
}
@@ -285,36 +252,5 @@ public final class MultiFields extends Fields {
.collect(Collectors.toSet());
}
- private static class LeafReaderFields extends Fields {
-
- private final LeafReader leafReader;
- private final List<String> indexedFields;
-
- LeafReaderFields(LeafReader leafReader) {
- this.leafReader = leafReader;
- this.indexedFields = new ArrayList<>();
- for (FieldInfo fieldInfo : leafReader.getFieldInfos()) {
- if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
- indexedFields.add(fieldInfo.name);
- }
- }
- Collections.sort(indexedFields);
- }
-
- @Override
- public Iterator<String> iterator() {
- return Collections.unmodifiableList(indexedFields).iterator();
- }
-
- @Override
- public int size() {
- return indexedFields.size();
- }
-
- @Override
- public Terms terms(String field) throws IOException {
- return leafReader.terms(field);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index 7afcf7a..dbc7ffa 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -27,6 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -618,19 +619,19 @@ public class TestDirectoryReader extends LuceneTestCase {
}
// check dictionary and posting lists
- Fields fields1 = MultiFields.getFields(index1);
- Fields fields2 = MultiFields.getFields(index2);
+ TreeSet<String> fields1 = new TreeSet<>(MultiFields.getIndexedFields(index1));
+ TreeSet<String> fields2 = new TreeSet<>(MultiFields.getIndexedFields(index2));
Iterator<String> fenum2 = fields2.iterator();
for (String field1 : fields1) {
assertEquals("Different fields", field1, fenum2.next());
- Terms terms1 = fields1.terms(field1);
+ Terms terms1 = MultiFields.getTerms(index1, field1);
if (terms1 == null) {
- assertNull(fields2.terms(field1));
+ assertNull(MultiFields.getTerms(index2, field1));
continue;
}
TermsEnum enum1 = terms1.iterator();
- Terms terms2 = fields2.terms(field1);
+ Terms terms2 = MultiFields.getTerms(index2, field1);
assertNotNull(terms2);
TermsEnum enum2 = terms2.iterator();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java b/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java
index 0f221e3..35771cf 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDocCount.java
@@ -17,6 +17,8 @@
package org.apache.lucene.index;
+import java.util.Collection;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.DocIdSetIterator;
@@ -57,9 +59,9 @@ public class TestDocCount extends LuceneTestCase {
}
private void verifyCount(IndexReader ir) throws Exception {
- Fields fields = MultiFields.getFields(ir);
+ final Collection<String> fields = MultiFields.getIndexedFields(ir);
for (String field : fields) {
- Terms terms = fields.terms(field);
+ Terms terms = MultiFields.getTerms(ir, field);
if (terms == null) {
continue;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
index d452306..bba57cd 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
@@ -280,11 +280,10 @@ public class TestParallelCompositeReader extends LuceneTestCase {
assertNull(pr.document(0).get("f3"));
assertNull(pr.document(0).get("f4"));
// check that fields are there
- Fields slow = MultiFields.getFields(pr);
- assertNotNull(slow.terms("f1"));
- assertNotNull(slow.terms("f2"));
- assertNotNull(slow.terms("f3"));
- assertNotNull(slow.terms("f4"));
+ assertNotNull(MultiFields.getTerms(pr, "f1"));
+ assertNotNull(MultiFields.getTerms(pr, "f2"));
+ assertNotNull(MultiFields.getTerms(pr, "f3"));
+ assertNotNull(MultiFields.getTerms(pr, "f4"));
pr.close();
// no stored fields at all
@@ -296,11 +295,10 @@ public class TestParallelCompositeReader extends LuceneTestCase {
assertNull(pr.document(0).get("f3"));
assertNull(pr.document(0).get("f4"));
// check that fields are there
- slow = MultiFields.getFields(pr);
- assertNull(slow.terms("f1"));
- assertNull(slow.terms("f2"));
- assertNotNull(slow.terms("f3"));
- assertNotNull(slow.terms("f4"));
+ assertNull(MultiFields.getTerms(pr, "f1"));
+ assertNull(MultiFields.getTerms(pr, "f2"));
+ assertNotNull(MultiFields.getTerms(pr, "f3"));
+ assertNotNull(MultiFields.getTerms(pr, "f4"));
pr.close();
// without overlapping
@@ -312,11 +310,10 @@ public class TestParallelCompositeReader extends LuceneTestCase {
assertNull(pr.document(0).get("f3"));
assertNull(pr.document(0).get("f4"));
// check that fields are there
- slow = MultiFields.getFields(pr);
- assertNull(slow.terms("f1"));
- assertNull(slow.terms("f2"));
- assertNotNull(slow.terms("f3"));
- assertNotNull(slow.terms("f4"));
+ assertNull(MultiFields.getTerms(pr, "f1"));
+ assertNull(MultiFields.getTerms(pr, "f2"));
+ assertNotNull(MultiFields.getTerms(pr, "f3"));
+ assertNotNull(MultiFields.getTerms(pr, "f4"));
pr.close();
// no main readers
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
index 5434956..35adf20 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
@@ -115,9 +115,9 @@ public class TestSegmentReader extends LuceneTestCase {
}
public void testTerms() throws IOException {
- Fields fields = MultiFields.getFields(reader);
+ final Collection<String> fields = MultiFields.getIndexedFields(reader);
for (String field : fields) {
- Terms terms = fields.terms(field);
+ Terms terms = MultiFields.getTerms(reader, field);
assertNotNull(terms);
TermsEnum termsEnum = terms.iterator();
while(termsEnum.next() != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
index d386f39..0334c5d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
@@ -310,16 +310,13 @@ public class TestStressIndexing2 extends LuceneTestCase {
int[] r2r1 = new int[r2.maxDoc()]; // r2 id to r1 id mapping
// create mapping from id2 space to id2 based on idField
- final Fields f1 = MultiFields.getFields(r1);
- if (f1 == null) {
- // make sure r2 is empty
- assertNull(MultiFields.getFields(r2));
+ if (MultiFields.getIndexedFields(r1).isEmpty()) {
+ assertTrue(MultiFields.getIndexedFields(r2).isEmpty());
return;
}
- final Terms terms1 = f1.terms(idField);
+ final Terms terms1 = MultiFields.getTerms(r1, idField);
if (terms1 == null) {
- assertTrue(MultiFields.getFields(r2) == null ||
- MultiFields.getFields(r2).terms(idField) == null);
+ assertTrue(MultiFields.getTerms(r2, idField) == null);
return;
}
final TermsEnum termsEnum = terms1.iterator();
@@ -327,9 +324,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
final Bits liveDocs1 = MultiFields.getLiveDocs(r1);
final Bits liveDocs2 = MultiFields.getLiveDocs(r2);
- Fields fields = MultiFields.getFields(r2);
- Terms terms2 = fields.terms(idField);
- if (fields.size() == 0 || terms2 == null) {
+ Terms terms2 = MultiFields.getTerms(r2, idField);
+ if (terms2 == null) {
// make sure r1 is in fact empty (eg has only all
// deleted docs):
Bits liveDocs = MultiFields.getLiveDocs(r1);
@@ -463,10 +459,8 @@ public class TestStressIndexing2 extends LuceneTestCase {
// Verify postings
//System.out.println("TEST: create te1");
- final Fields fields1 = MultiFields.getFields(r1);
- final Iterator<String> fields1Enum = fields1.iterator();
- final Fields fields2 = MultiFields.getFields(r2);
- final Iterator<String> fields2Enum = fields2.iterator();
+ final Iterator<String> fields1Enum = MultiFields.getIndexedFields(r1).stream().sorted().iterator();
+ final Iterator<String> fields2Enum = MultiFields.getIndexedFields(r2).stream().sorted().iterator();
String field1=null, field2=null;
@@ -490,7 +484,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
break;
}
field1 = fields1Enum.next();
- Terms terms = fields1.terms(field1);
+ Terms terms = MultiFields.getTerms(r1, field1);
if (terms == null) {
continue;
}
@@ -526,7 +520,7 @@ public class TestStressIndexing2 extends LuceneTestCase {
break;
}
field2 = fields2Enum.next();
- Terms terms = fields2.terms(field2);
+ Terms terms = MultiFields.getTerms(r2, field2);
if (terms == null) {
continue;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java b/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java
index 67063f6..9ef90ad 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSumDocFreq.java
@@ -17,6 +17,8 @@
package org.apache.lucene.index;
+import java.util.Collection;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
@@ -73,10 +75,9 @@ public class TestSumDocFreq extends LuceneTestCase {
private void assertSumDocFreq(IndexReader ir) throws Exception {
// compute sumDocFreq across all fields
- Fields fields = MultiFields.getFields(ir);
-
+ final Collection<String> fields = MultiFields.getIndexedFields(ir);
for (String f : fields) {
- Terms terms = fields.terms(f);
+ Terms terms = MultiFields.getTerms(ir, f);
long sumDocFreq = terms.getSumDocFreq();
if (sumDocFreq == -1) {
if (VERBOSE) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
----------------------------------------------------------------------
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
index a4215a1..1579594 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
@@ -171,7 +171,7 @@ public class TestMemoryIndexAgainstRAMDir extends BaseTokenStreamTestCase {
private void duellReaders(CompositeReader other, LeafReader memIndexReader)
throws IOException {
Fields memFields = memIndexReader.getTermVectors(0);
- for (String field : MultiFields.getFields(other)) {
+ for (String field : MultiFields.getIndexedFields(other)) {
Terms memTerms = memFields.terms(field);
Terms iwTerms = memIndexReader.terms(field);
if (iwTerms == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java b/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
index 01a2d9d..5a1cf04 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
@@ -16,23 +16,23 @@
*/
package org.apache.lucene.misc;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Locale;
+
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.SuppressForbidden;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.Comparator;
-import java.util.Locale;
-
/**
* <code>HighFreqTerms</code> class extracts the top n most frequent terms
* (by document frequency) from an existing Lucene index and reports their
@@ -107,13 +107,13 @@ public class HighFreqTerms {
tiq = new TermStatsQueue(numTerms, comparator);
tiq.fill(field, termsEnum);
} else {
- Fields fields = MultiFields.getFields(reader);
+ Collection<String> fields = MultiFields.getIndexedFields(reader);
if (fields.size() == 0) {
throw new RuntimeException("no fields found for this index");
}
tiq = new TermStatsQueue(numTerms, comparator);
for (String fieldName : fields) {
- Terms terms = fields.terms(fieldName);
+ Terms terms = MultiFields.getTerms(reader, fieldName);
if (terms != null) {
tiq.fill(fieldName, terms.iterator());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
index 83419de..03f01e7 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
@@ -25,9 +25,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -357,7 +359,25 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
};
try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState)) {
- consumer.write(MultiFields.getFields(oneDocReader), fakeNorms);
+ final Fields fields = new Fields() {
+ TreeSet<String> indexedFields = new TreeSet<>(MultiFields.getIndexedFields(oneDocReader));
+
+ @Override
+ public Iterator<String> iterator() {
+ return indexedFields.iterator();
+ }
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ return oneDocReader.terms(field);
+ }
+
+ @Override
+ public int size() {
+ return indexedFields.size();
+ }
+ };
+ consumer.write(fields, fakeNorms);
IOUtils.close(consumer);
IOUtils.close(consumer);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 293c344..68e7190 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -1950,7 +1950,7 @@ public abstract class LuceneTestCase extends Assert {
public void assertReaderEquals(String info, IndexReader leftReader, IndexReader rightReader) throws IOException {
assertReaderStatisticsEquals(info, leftReader, rightReader);
- assertFieldsEquals(info, leftReader, MultiFields.getFields(leftReader), MultiFields.getFields(rightReader), true);
+ assertTermsEquals(info, leftReader, rightReader, true);
assertNormsEquals(info, leftReader, rightReader);
assertStoredFieldsEquals(info, leftReader, rightReader);
assertTermVectorsEquals(info, leftReader, rightReader);
@@ -1974,33 +1974,13 @@ public abstract class LuceneTestCase extends Assert {
/**
* Fields api equivalency
*/
- public void assertFieldsEquals(String info, IndexReader leftReader, Fields leftFields, Fields rightFields, boolean deep) throws IOException {
- // Fields could be null if there are no postings,
- // but then it must be null for both
- if (leftFields == null || rightFields == null) {
- assertNull(info, leftFields);
- assertNull(info, rightFields);
- return;
- }
- assertFieldStatisticsEquals(info, leftFields, rightFields);
-
- Iterator<String> leftEnum = leftFields.iterator();
- Iterator<String> rightEnum = rightFields.iterator();
-
- while (leftEnum.hasNext()) {
- String field = leftEnum.next();
- assertEquals(info, field, rightEnum.next());
- assertTermsEquals(info, leftReader, leftFields.terms(field), rightFields.terms(field), deep);
- }
- assertFalse(rightEnum.hasNext());
- }
+ public void assertTermsEquals(String info, IndexReader leftReader, IndexReader rightReader, boolean deep) throws IOException {
+ Set<String> leftFields = new HashSet<>(MultiFields.getIndexedFields(leftReader));
+ Set<String> rightFields = new HashSet<>(MultiFields.getIndexedFields(rightReader));
+ assertEquals(info, leftFields, rightFields);
- /**
- * checks that top-level statistics on Fields are the same
- */
- public void assertFieldStatisticsEquals(String info, Fields leftFields, Fields rightFields) throws IOException {
- if (leftFields.size() != -1 && rightFields.size() != -1) {
- assertEquals(info, leftFields.size(), rightFields.size());
+ for (String field : leftFields) {
+ assertTermsEquals(info, leftReader, MultiFields.getTerms(leftReader, field), MultiFields.getTerms(rightReader, field), deep);
}
}
@@ -2331,15 +2311,9 @@ public abstract class LuceneTestCase extends Assert {
* checks that norms are the same across all fields
*/
public void assertNormsEquals(String info, IndexReader leftReader, IndexReader rightReader) throws IOException {
- Fields leftFields = MultiFields.getFields(leftReader);
- Fields rightFields = MultiFields.getFields(rightReader);
- // Fields could be null if there are no postings,
- // but then it must be null for both
- if (leftFields == null || rightFields == null) {
- assertNull(info, leftFields);
- assertNull(info, rightFields);
- return;
- }
+ Set<String> leftFields = new HashSet<>(MultiFields.getIndexedFields(leftReader));
+ Set<String> rightFields = new HashSet<>(MultiFields.getIndexedFields(rightReader));
+ assertEquals(info, leftFields, rightFields);
for (String field : leftFields) {
NumericDocValues leftNorms = MultiDocValues.getNormValues(leftReader, field);
@@ -2407,7 +2381,26 @@ public abstract class LuceneTestCase extends Assert {
for (int i = 0; i < leftReader.maxDoc(); i++) {
Fields leftFields = leftReader.getTermVectors(i);
Fields rightFields = rightReader.getTermVectors(i);
- assertFieldsEquals(info, leftReader, leftFields, rightFields, rarely());
+
+ // Fields could be null if there are no postings,
+ // but then it must be null for both
+ if (leftFields == null || rightFields == null) {
+ assertNull(info, leftFields);
+ assertNull(info, rightFields);
+ return;
+ }
+ if (leftFields.size() != -1 && rightFields.size() != -1) {
+ assertEquals(info, leftFields.size(), rightFields.size());
+ }
+
+ Iterator<String> leftEnum = leftFields.iterator();
+ Iterator<String> rightEnum = rightFields.iterator();
+ while (leftEnum.hasNext()) {
+ String field = leftEnum.next();
+ assertEquals(info, field, rightEnum.next());
+ assertTermsEquals(info, leftReader, leftFields.terms(field), rightFields.terms(field), rarely());
+ }
+ assertFalse(rightEnum.hasNext());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 490bcf2..ca1dacf 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -82,6 +82,8 @@ Other Changes
* SOLR-12652: Remove SolrMetricManager.overridableRegistryName method (Peter Somogyi via David Smiley)
+* LUCENE-8513: SlowCompositeReaderWrapper now uses MultiTerms directly instead of MultiFields (David Smiley)
+
================== 7.6.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
index 283d245..5b03ee8 100644
--- a/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
+++ b/solr/core/src/java/org/apache/solr/index/SlowCompositeReaderWrapper.java
@@ -19,6 +19,7 @@ package org.apache.solr.index;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.index.*;
import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
@@ -45,9 +46,15 @@ import org.apache.lucene.util.Version;
public final class SlowCompositeReaderWrapper extends LeafReader {
private final CompositeReader in;
- private final Fields fields;
private final LeafMetaData metaData;
-
+
+ final Map<String,Terms> cachedTerms = new ConcurrentHashMap<>();
+
+ // TODO: consider ConcurrentHashMap ?
+ // TODO: this could really be a weak map somewhere else on the coreCacheKey,
+ // but do we really need to optimize slow-wrapper any more?
+ final Map<String,OrdinalMap> cachedOrdMaps = new HashMap<>();
+
/** This method is sugar for getting an {@link LeafReader} from
* an {@link IndexReader} of any kind. If the reader is already atomic,
* it is returned unchanged, otherwise wrapped by this class.
@@ -62,9 +69,7 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
}
SlowCompositeReaderWrapper(CompositeReader reader) throws IOException {
- super();
in = reader;
- fields = MultiFields.getFields(in);
in.registerParentReader(this);
if (reader.leaves().isEmpty()) {
metaData = new LeafMetaData(Version.LATEST.major, Version.LATEST, null);
@@ -104,25 +109,38 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
@Override
public Terms terms(String field) throws IOException {
ensureOpen();
- return fields.terms(field);
+ try {
+ return cachedTerms.computeIfAbsent(field, f -> {
+ try {
+ return MultiFields.getTerms(in, f);
+ } catch (IOException e) { // yuck! ...sigh... checked exceptions with built-in lambdas are a pain
+ throw new RuntimeException("unwrapMe", e);
+ }
+ });
+ } catch (RuntimeException e) {
+ if (e.getMessage().equals("unwrapMe") && e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ }
+ throw e;
+ }
}
@Override
public NumericDocValues getNumericDocValues(String field) throws IOException {
ensureOpen();
- return MultiDocValues.getNumericValues(in, field);
+ return MultiDocValues.getNumericValues(in, field); // TODO cache?
}
@Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
ensureOpen();
- return MultiDocValues.getBinaryValues(in, field);
+ return MultiDocValues.getBinaryValues(in, field); // TODO cache?
}
@Override
public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException {
ensureOpen();
- return MultiDocValues.getSortedNumericValues(in, field);
+ return MultiDocValues.getSortedNumericValues(in, field); // TODO cache?
}
@Override
@@ -210,15 +228,11 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
starts[size] = maxDoc();
return new MultiDocValues.MultiSortedSetDocValues(values, starts, map, cost);
}
-
- // TODO: this could really be a weak map somewhere else on the coreCacheKey,
- // but do we really need to optimize slow-wrapper any more?
- final Map<String,OrdinalMap> cachedOrdMaps = new HashMap<>();
@Override
public NumericDocValues getNormValues(String field) throws IOException {
ensureOpen();
- return MultiDocValues.getNormValues(in, field);
+ return MultiDocValues.getNormValues(in, field); // TODO cache?
}
@Override
@@ -248,19 +262,19 @@ public final class SlowCompositeReaderWrapper extends LeafReader {
@Override
public Bits getLiveDocs() {
ensureOpen();
- return MultiFields.getLiveDocs(in);
+ return MultiFields.getLiveDocs(in); // TODO cache?
}
@Override
public PointValues getPointValues(String field) {
ensureOpen();
- return null;
+ return null; // because not supported. Throw UOE?
}
@Override
public FieldInfos getFieldInfos() {
ensureOpen();
- return MultiFields.getMergedFieldInfos(in);
+ return MultiFields.getMergedFieldInfos(in); // TODO cache?
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fe844c73/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
index 9907d59..3e693f2 100644
--- a/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
+++ b/solr/core/src/test/org/apache/solr/index/TestSlowCompositeReaderWrapper.java
@@ -22,15 +22,18 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
+import org.apache.lucene.index.MultiTerms;
import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -120,4 +123,28 @@ public class TestSlowCompositeReaderWrapper extends LuceneTestCase {
w.close();
dir.close();
}
+
+ public void testTermsAreCached() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
+ Document doc = new Document();
+ doc.add(new TextField("text", "hello world", Field.Store.NO));
+ w.addDocument(doc);
+ w.getReader().close();
+ doc = new Document();
+ doc.add(new TextField("text", "cruel world", Field.Store.NO));
+ w.addDocument(doc);
+
+ IndexReader reader = w.getReader();
+ assertTrue(reader.leaves().size() > 1);
+ SlowCompositeReaderWrapper slowWrapper = (SlowCompositeReaderWrapper) SlowCompositeReaderWrapper.wrap(reader);
+ assertEquals(0, slowWrapper.cachedTerms.size());
+ assertEquals(MultiTerms.class, slowWrapper.terms("text").getClass());
+ assertEquals(1, slowWrapper.cachedTerms.size());
+ assertNull(slowWrapper.terms("bogusField"));
+ assertEquals(1, slowWrapper.cachedTerms.size());//bogus field isn't cached
+ reader.close();
+ w.close();
+ dir.close();
+ }
}
[06/30] lucene-solr:jira/http2: SOLR-12080: Disable this test until
it's fixed.
Posted by da...@apache.org.
SOLR-12080: Disable this test until it's fixed.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e6877484
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e6877484
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e6877484
Branch: refs/heads/jira/http2
Commit: e687748405f5ef439969b542aaec014abdfbfaec
Parents: 2201b65
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Oct 3 07:56:26 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Oct 3 07:56:26 2018 +0200
----------------------------------------------------------------------
.../src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e6877484/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
index 989d39b..61a5530 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
@@ -76,6 +76,12 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
testFailedMove();
}
+ @Test
+ @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12080") // added 03-Oct-2018
+ public void testFailedMove() throws Exception {
+ super.testFailedMove();
+ }
+
public static class ForkJoinThreadsFilter implements ThreadFilter {
@Override
[24/30] lucene-solr:jira/http2: SOLR-12835: Document statistics
exposed by the Query Result Cache when maxRamMB is configured
Posted by da...@apache.org.
SOLR-12835: Document statistics exposed by the Query Result Cache when maxRamMB is configured
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ace0db7a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ace0db7a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ace0db7a
Branch: refs/heads/jira/http2
Commit: ace0db7a0a02e441055badca1687afd25b037f23
Parents: 14e6eb2
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Oct 5 08:20:31 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Oct 5 08:20:31 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../src/performance-statistics-reference.adoc | 12 +++++++++++-
2 files changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ace0db7a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ae20fc3..93faa36 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -143,6 +143,8 @@ Other Changes
changed from {collectionDefaults: {nrtReplicas : 2}} to {defaults : {collection : {nrtReplicas : 2}}}.
(ab, shalin)
+* SOLR-12835: Document statistics exposed by the Query Result Cache when maxRamMB is configured. (shalin)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ace0db7a/solr/solr-ref-guide/src/performance-statistics-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/performance-statistics-reference.adoc b/solr/solr-ref-guide/src/performance-statistics-reference.adoc
index cf948c7..8b5cbd0 100644
--- a/solr/solr-ref-guide/src/performance-statistics-reference.adoc
+++ b/solr/solr-ref-guide/src/performance-statistics-reference.adoc
@@ -192,8 +192,18 @@ The following statistics are available for each of the caches mentioned above:
|hits |Number of hits for the current index searcher.
|inserts |Number of inserts into the cache.
|lookups |Number of lookups against the cache.
-|size |Size of the cache at that particular instance (in KBs).
+|size |Number of entries in the cache at that particular instance.
|warmupTime |Warm-up time for the registered index searcher. This time is taken in account for the “auto-warming” of caches.
|===
+When eviction by heap usage is enabled, the following additional statistics are available for the Query Result Cache:
+
+[cols="25,75",options="header"]
+|===
+|Attribute |Description
+|maxRamMB |Maximum heap that should be used by the cache beyond which keys will be evicted.
+|ramBytesUsed| Actual heap usage of the cache at that particular instance.
+|evictionsRamUsage| Number of cache evictions for the current index searcher because heap usage exceeded maxRamMB.
+|===
+
More information on Solr caches is available in the section <<query-settings-in-solrconfig.adoc#query-settings-in-solrconfig,Query Settings in SolrConfig>>.
[18/30] lucene-solr:jira/http2: SOLR-12827: Fix blurb in ref guide to
say that the key is deprecated instead of saying that it is no longer
supported.
Posted by da...@apache.org.
SOLR-12827: Fix blurb in ref guide to say that the key is deprecated instead of saying that it is no longer supported.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/793a677d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/793a677d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/793a677d
Branch: refs/heads/jira/http2
Commit: 793a677d0f62e2bcfeae7e0abb42b5f7ab40126e
Parents: 452c2da
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Thu Oct 4 16:21:42 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Oct 4 16:22:29 2018 +0530
----------------------------------------------------------------------
solr/solr-ref-guide/src/collections-api.adoc | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/793a677d/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index a5b4e56..f320cdb 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -1247,9 +1247,10 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
}' http://localhost:8983/api/cluster
----
-NOTE: Until Solr 7.5, cluster properties supported a "collectionDefaults" key which is no longer supported. Using the API
-structure for Solr 7.4 or Solr 7.5 will continue to work but the format of the properties will automatically be converted
-to the new nested structure. The old "collectionDefaults" key is deprecated and will be removed in Solr 9.
+NOTE: Until Solr 7.5, cluster properties supported a "collectionDefaults" key which is now deprecated. Using the API
+structure for Solr 7.4 or Solr 7.5 with "collectionDefaults" will continue to work but the format of the properties
+will automatically be converted to the new nested structure. Support for the "collectionDefaults" key will be
+removed in Solr 9.
[[collectionprop]]
== COLLECTIONPROP: Collection Properties
[12/30] lucene-solr:jira/http2: SOLR-12828: Add oscillate Stream
Evaluator to support sine wave analysis
Posted by da...@apache.org.
SOLR-12828: Add oscillate Stream Evaluator to support sine wave analysis
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/751bf7db
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/751bf7db
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/751bf7db
Branch: refs/heads/jira/http2
Commit: 751bf7db208d505ecb48779b432ee3e52f00fdfc
Parents: d8e4079
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Oct 3 12:24:48 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Oct 3 12:25:35 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/Lang.java | 6 ++-
.../solrj/io/eval/GetAmplitudeEvaluator.java | 42 +++++++++++++++
.../io/eval/GetAngularFrequencyEvaluator.java | 42 +++++++++++++++
.../client/solrj/io/eval/GetPhaseEvaluator.java | 42 +++++++++++++++
.../solrj/io/eval/HarmonicFitEvaluator.java | 11 +++-
.../solrj/io/eval/OscillateEvaluator.java | 57 ++++++++++++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 5 +-
.../solrj/io/stream/MathExpressionTest.java | 42 ++++++++++++++-
8 files changed, 242 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index 17f9d57..02968b2 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -174,6 +174,7 @@ public class Lang {
.withFunctionName("betaDistribution", BetaDistributionEvaluator.class)
.withFunctionName("polyfit", PolyFitEvaluator.class)
.withFunctionName("harmonicFit", HarmonicFitEvaluator.class)
+ .withFunctionName("harmfit", HarmonicFitEvaluator.class)
.withFunctionName("loess", LoessEvaluator.class)
.withFunctionName("matrix", MatrixEvaluator.class)
.withFunctionName("transpose", TransposeEvaluator.class)
@@ -261,7 +262,10 @@ public class Lang {
.withFunctionName("getBaryCenter", GetBaryCenterEvaluator.class)
.withFunctionName("getArea", GetAreaEvaluator.class)
.withFunctionName("getBoundarySize", GetBoundarySizeEvaluator.class)
-
+ .withFunctionName("oscillate", OscillateEvaluator.class)
+ .withFunctionName("getAmplitude", GetAmplitudeEvaluator.class)
+ .withFunctionName("getPhase", GetPhaseEvaluator.class)
+ .withFunctionName("getAngularFrequency", GetAngularFrequencyEvaluator.class)
// Boolean Stream Evaluators
.withFunctionName("and", AndEvaluator.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAmplitudeEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAmplitudeEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAmplitudeEvaluator.java
new file mode 100644
index 0000000..131029b
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAmplitudeEvaluator.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetAmplitudeEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetAmplitudeEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof VectorFunction)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting a Vector Function",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ VectorFunction vectorFunction = (VectorFunction)value;
+ return vectorFunction.getFromContext("amplitude");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAngularFrequencyEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAngularFrequencyEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAngularFrequencyEvaluator.java
new file mode 100644
index 0000000..001e95e
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetAngularFrequencyEvaluator.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetAngularFrequencyEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetAngularFrequencyEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof VectorFunction)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting a Vector Function",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ VectorFunction vectorFunction = (VectorFunction)value;
+ return vectorFunction.getFromContext("angularFrequency");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetPhaseEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetPhaseEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetPhaseEvaluator.java
new file mode 100644
index 0000000..546ef20
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetPhaseEvaluator.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetPhaseEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetPhaseEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof VectorFunction)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting a Vector Function",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ VectorFunction vectorFunction = (VectorFunction)value;
+ return vectorFunction.getFromContext("phase");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HarmonicFitEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HarmonicFitEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HarmonicFitEvaluator.java
index 04c0b0c..4e7ac6e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HarmonicFitEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/HarmonicFitEvaluator.java
@@ -70,6 +70,9 @@ public class HarmonicFitEvaluator extends RecursiveNumericEvaluator implements M
points.add(x[i], y[i]);
}
+ double[] guess = new HarmonicCurveFitter.ParameterGuesser(points.toList()).guess();
+ curveFitter = curveFitter.withStartPoint(guess);
+
double[] coef = curveFitter.fit(points.toList());
HarmonicOscillator pf = new HarmonicOscillator(coef[0], coef[1], coef[2]);
@@ -79,6 +82,12 @@ public class HarmonicFitEvaluator extends RecursiveNumericEvaluator implements M
list.add(yvalue);
}
- return list;
+ VectorFunction vectorFunction = new VectorFunction(pf, list);
+ vectorFunction.addToContext("amplitude", coef[0]);
+ vectorFunction.addToContext("angularFrequency", coef[1]);
+ vectorFunction.addToContext("phase", coef[2]);
+
+ return vectorFunction;
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OscillateEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OscillateEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OscillateEvaluator.java
new file mode 100644
index 0000000..6fdd3c5
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/OscillateEvaluator.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.commons.math3.analysis.function.HarmonicOscillator;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class OscillateEvaluator extends RecursiveNumericEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public OscillateEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object... objects) throws IOException{
+
+ if(objects.length != 3) {
+ throw new IOException("The oscillate function takes 3 arguments.");
+ }
+
+ double amp = ((Number)objects[0]).doubleValue();
+ double om = ((Number)objects[1]).doubleValue();
+ double phase = ((Number)objects[2]).doubleValue();
+
+
+ HarmonicOscillator pf = new HarmonicOscillator(amp, om, phase);
+
+ List list = new ArrayList();
+ for(int i=0; i<128; i++) {
+ double yvalue= pf.value(i);
+ list.add(yvalue);
+ }
+
+ return new VectorFunction(pf, list);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index bc15e02..169fead 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -52,7 +52,7 @@ public class TestLang extends LuceneTestCase {
"poissonDistribution", "enumeratedDistribution", "probability", "sumDifference", "meanDifference",
"primes", "factorial", "movingMedian", "binomialCoefficient", "expMovingAvg", "monteCarlo", "constantDistribution",
"weibullDistribution", "mean", "mode", "logNormalDistribution", "zipFDistribution", "gammaDistribution",
- "betaDistribution", "polyfit", "harmonicFit", "loess", "matrix", "transpose", "unitize",
+ "betaDistribution", "polyfit", "harmonicFit", "harmfit", "loess", "matrix", "transpose", "unitize",
"triangularDistribution", "precision", "minMaxScale", "markovChain", "grandSum",
"scalarAdd", "scalarSubtract", "scalarMultiply", "scalarDivide", "sumRows",
"sumColumns", "diff", "corrPValues", "normalizeSum", "geometricDistribution", "olsRegress",
@@ -71,7 +71,8 @@ public class TestLang extends LuceneTestCase {
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft", "euclidean","manhattan",
"earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue", "knnRegress", "gaussfit",
"outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "latlonVectors",
- "convexHull", "getVertices", "getBaryCenter", "getArea", "getBoundarySize"};
+ "convexHull", "getVertices", "getBaryCenter", "getArea", "getBoundarySize","oscillate",
+ "getAmplitude", "getPhase", "getAngularFrequency"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/751bf7db/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 67b0e08..73aa0c9 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -342,7 +342,6 @@ public class MathExpressionTest extends SolrCloudTestCase {
List<Tuple> tuples = getTuples(solrStream);
assertTrue(tuples.size() == 1);
List<List<Number>>locVectors = (List<List<Number>>)tuples.get(0).get("b");
- System.out.println(locVectors);
int v=1;
for(List<Number> row : locVectors) {
double lat = row.get(0).doubleValue();
@@ -2360,6 +2359,46 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
@Test
+ public void testOscillate() throws Exception {
+ String cexpr = "let(echo=true," +
+ " a=oscillate(10, .3, 2.9)," +
+ " b=describe(a)," +
+ " c=getValue(b, min)," +
+ " d=getValue(b, max)," +
+ " e=harmfit(a)," +
+ " f=getAmplitude(e)," +
+ " g=getAngularFrequency(e)," +
+ " h=getPhase(e))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> wave = (List<Number>)tuples.get(0).get("a");
+ assertEquals(wave.size(), 128);
+ Map desc = (Map)tuples.get(0).get("b");
+ Number min = (Number)tuples.get(0).get("c");
+ Number max = (Number)tuples.get(0).get("d");
+ assertEquals(min.doubleValue(), -9.9, .1);
+ assertEquals(max.doubleValue(), 9.9, .1);
+
+ List<Number> wave1 = (List<Number>)tuples.get(0).get("e");
+ assertEquals(wave1.size(), 128);
+
+ Number amp = (Number)tuples.get(0).get("f");
+ Number freq = (Number)tuples.get(0).get("g");
+ Number pha = (Number)tuples.get(0).get("h");
+
+ assertEquals(amp.doubleValue(), 10, .1);
+ assertEquals(freq.doubleValue(), .3, .1);
+ assertEquals(pha.doubleValue(), 2.9, .1);
+ }
+
+ @Test
public void testEbeAdd() throws Exception {
String cexpr = "let(echo=true," +
" a=array(2, 4, 6, 8, 10, 12)," +
@@ -2410,6 +2449,7 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
+
@Test
public void testSetAndGetValue() throws Exception {
String cexpr = "let(echo=true," +
[28/30] lucene-solr:jira/http2: SOLR-12803: Ensure CUSC routes docs
to right cores
Posted by da...@apache.org.
SOLR-12803: Ensure CUSC routes docs to right cores
ConcurrentUpdateSolrClient can batch together many documents when making
an indexing request to Solr. When adding an update request to the
current batch being made, it checks that the query-parameters of the
docs being added match those already in the batch. But prior to this
commit it never checked that the collections/cores were the same.
This could result in documents being sent to the wrong collection if the
same client is used to index documents to two different
cores/collections simultaneously.
This commit addresses this problem, ensuring that documents aren't added
to a batch directed at a different core/collection.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/367bdf7f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/367bdf7f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/367bdf7f
Branch: refs/heads/jira/http2
Commit: 367bdf7f749a4386071f4444e2d6a09591f38daf
Parents: e2b8bec
Author: Jason Gerlowski <ge...@apache.org>
Authored: Sun Oct 7 09:51:43 2018 -0400
Committer: Jason Gerlowski <ge...@apache.org>
Committed: Sun Oct 7 10:08:50 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solrj/impl/ConcurrentUpdateSolrClient.java | 6 +-
.../org/apache/solr/common/StringUtils.java | 10 +++
...rentUpdateSolrClientMultiCollectionTest.java | 94 ++++++++++++++++++++
4 files changed, 110 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/367bdf7f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4bd8158..4fea2ff 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -148,6 +148,8 @@ Other Changes
Bug Fixes
----------------------
+* SOLR-12803: Ensure ConcurrentUpdateSolrClient sends documents to correct collection (Jason Gerlowski)
+
* SOLR-11836: FacetStream works with bucketSizeLimit of -1 which will fetch all the buckets.
(Alfonso Muñoz-Pomer Fuentes, Amrit Sarkar via Varun Thacker)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/367bdf7f/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
index 5845e7f..26af757 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
@@ -47,6 +47,7 @@ import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.StringUtils;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
@@ -243,6 +244,7 @@ public class ConcurrentUpdateSolrClient extends SolrClient {
final boolean isXml = ClientUtils.TEXT_XML.equals(contentType);
final ModifiableSolrParams origParams = new ModifiableSolrParams(update.getRequest().getParams());
+ final String origTargetCollection = update.getCollection();
EntityTemplate template = new EntityTemplate(new ContentProducer() {
@@ -256,8 +258,8 @@ public class ConcurrentUpdateSolrClient extends SolrClient {
while (upd != null) {
UpdateRequest req = upd.getRequest();
SolrParams currentParams = new ModifiableSolrParams(req.getParams());
- if (!origParams.toNamedList().equals(currentParams.toNamedList())) {
- queue.add(upd); // params are different, push back to queue
+ if (!origParams.toNamedList().equals(currentParams.toNamedList()) || !StringUtils.equals(origTargetCollection, upd.getCollection())) {
+ queue.add(upd); // Request has different params or destination core/collection, return to queue
break;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/367bdf7f/solr/solrj/src/java/org/apache/solr/common/StringUtils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/StringUtils.java b/solr/solrj/src/java/org/apache/solr/common/StringUtils.java
index 9b8e042..58382a7 100644
--- a/solr/solrj/src/java/org/apache/solr/common/StringUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/StringUtils.java
@@ -22,4 +22,14 @@ public class StringUtils {
return (s == null) || s.isEmpty();
}
+ /**
+ * A "null-safe" equals method. Returns true if the two provided references are both null, or if they are string-equal.
+ */
+ public static boolean equals(String first, String second) {
+ if (first == null) {
+ return second == null;
+ }
+ return first.equals(second);
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/367bdf7f/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientMultiCollectionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientMultiCollectionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientMultiCollectionTest.java
new file mode 100644
index 0000000..9823740
--- /dev/null
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientMultiCollectionTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * {@link ConcurrentUpdateSolrClient} reuses the same HTTP connection to send multiple requests. These tests ensure
+ * that this connection-reuse never results in documents being sent to the wrong collection. See SOLR-12803
+ */
+public class ConcurrentUpdateSolrClientMultiCollectionTest extends SolrCloudTestCase {
+
+ private static final String COLLECTION_ONE_NAME = "collection1";
+ private static final String COLLECTION_TWO_NAME = "collection2";
+
+ private String solrUrl;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(1)
+ .addConfig("conf", new File(ExternalPaths.TECHPRODUCTS_CONFIGSET).toPath())
+ .configure();
+ }
+
+ @Before
+ public void createCollections() throws Exception {
+ solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();
+
+ CollectionAdminRequest.createCollection(COLLECTION_ONE_NAME, "conf", 1, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection(COLLECTION_TWO_NAME, "conf", 1, 1).process(cluster.getSolrClient());
+ }
+
+ @After
+ public void deleteCollections() throws Exception {
+ cluster.deleteAllCollections();
+ }
+
+ @Test
+ public void testEnsureDocumentsSentToCorrectCollection() throws Exception {
+ int numTotalDocs = 1000;
+ int numExpectedPerCollection = numTotalDocs / 2;
+ try (SolrClient client = new ConcurrentUpdateSolrClient.Builder(solrUrl)
+ .withQueueSize(numTotalDocs).build()) {
+ splitDocumentsAcrossCollections(client, numTotalDocs);
+
+ assertEquals(numExpectedPerCollection, client.query(COLLECTION_ONE_NAME, new SolrQuery("*:*")).getResults().getNumFound());
+ assertEquals(numExpectedPerCollection, client.query(COLLECTION_TWO_NAME, new SolrQuery("*:*")).getResults().getNumFound());
+ }
+
+ }
+
+ private void splitDocumentsAcrossCollections(SolrClient client, int numTotalDocs) throws IOException, SolrServerException {
+ for (int docNum = 0; docNum < numTotalDocs; docNum++) {
+ final SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", "value" + docNum);
+
+ if (docNum %2 == 0) {
+ client.add(COLLECTION_ONE_NAME, doc);
+ } else {
+ client.add(COLLECTION_TWO_NAME, doc);
+ }
+ }
+
+ client.commit(COLLECTION_ONE_NAME);
+ client.commit(COLLECTION_TWO_NAME);
+ }
+}
[08/30] lucene-solr:jira/http2: SOLR-12750: Migrate API should lock
the collection instead of shard
Posted by da...@apache.org.
SOLR-12750: Migrate API should lock the collection instead of shard
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/84683b6a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/84683b6a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/84683b6a
Branch: refs/heads/jira/http2
Commit: 84683b6aa072459b364c62e71abd526cdcf8e117
Parents: 65105aa
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Oct 3 16:24:26 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Oct 3 16:24:26 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../src/java/org/apache/solr/common/params/CollectionParams.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84683b6a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 576140c..28736cb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -131,6 +131,8 @@ Bug Fixes
* SOLR-12648: Autoscaling framework based replica placement is not used unless a policy is specified or
non-empty cluster policy exists. (shalin)
+* SOLR-12750: Migrate API should lock the collection instead of shard. (shalin)
+
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84683b6a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index 6fb348f..dee2f5f 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -86,7 +86,7 @@ public interface CollectionParams {
CREATESHARD(true, LockLevel.COLLECTION),
DELETEREPLICA(true, LockLevel.SHARD),
FORCELEADER(true, LockLevel.SHARD),
- MIGRATE(true, LockLevel.SHARD),
+ MIGRATE(true, LockLevel.COLLECTION),
ADDROLE(true, LockLevel.NONE),
REMOVEROLE(true, LockLevel.NONE),
CLUSTERPROP(true, LockLevel.NONE),
[09/30] lucene-solr:jira/http2: SOLR-12725:
ParseDateFieldUpdateProcessorFactory should reuse ParsePosition.
Posted by da...@apache.org.
SOLR-12725: ParseDateFieldUpdateProcessorFactory should reuse ParsePosition.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eba7beda
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eba7beda
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eba7beda
Branch: refs/heads/jira/http2
Commit: eba7bedadf65cda114544d2495cf264065efa7ab
Parents: 84683b6
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Oct 3 13:58:31 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Oct 3 13:59:02 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../ParseDateFieldUpdateProcessorFactory.java | 28 ++++++++++++++++----
2 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eba7beda/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 28736cb..d9ee5fc 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -428,6 +428,8 @@ Optimizations
* SOLR-12766: When retrying internal requests, backoff only once for the full batch of retries (Tomás Fernández Löbbe)
+* SOLR-12725: ParseDateFieldUpdateProcessorFactory should reuse ParsePosition (ab)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eba7beda/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
index c7aaaaf..08aa973 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
@@ -17,6 +17,7 @@
package org.apache.solr.update.processor;
import java.lang.invoke.MethodHandles;
+import java.text.ParsePosition;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
@@ -119,13 +120,15 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
private static final String DEFAULT_TIME_ZONE_PARAM = "defaultTimeZone";
private static final String LOCALE_PARAM = "locale";
- private Map<String,DateTimeFormatter> formats = new LinkedHashMap<>();
+ private Map<String, DateTimeFormatter> formats = new LinkedHashMap<>();
@Override
public UpdateRequestProcessor getInstance(SolrQueryRequest req,
SolrQueryResponse rsp,
UpdateRequestProcessor next) {
return new AllValuesOrNoneFieldMutatingUpdateProcessor(getSelector(), next) {
+ final ParsePosition parsePosition = new ParsePosition(0);
+
@Override
protected Object mutateValue(Object srcVal) {
if (srcVal instanceof CharSequence) {
@@ -143,7 +146,7 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
for (Map.Entry<String,DateTimeFormatter> format : formats.entrySet()) {
DateTimeFormatter parser = format.getValue();
try {
- return Date.from(parseInstant(parser, srcStringVal));
+ return Date.from(parseInstant(parser, srcStringVal, parsePosition));
} catch (DateTimeParseException e) {
log.debug("value '{}' is not parseable with format '{}'",
new Object[] { srcStringVal, format.getKey() });
@@ -208,7 +211,7 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
public static void validateFormatter(DateTimeFormatter formatter) {
// check it's valid via round-trip
try {
- parseInstant(formatter, formatter.format(Instant.now()));
+ parseInstant(formatter, formatter.format(Instant.now()), new ParsePosition(0));
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Bad or unsupported pattern: " + formatter.toFormat().toString(), e);
@@ -220,8 +223,23 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
// the input string contains a timezone/offset that differs from the "override zone"
// (which we configure in DEFAULT_TIME_ZONE). Besides, we need the code below which handles
// the optionality of time. Were it not for that, we truly could do formatter.parse(Instant::from).
- private static Instant parseInstant(DateTimeFormatter formatter, String dateStr) {
- final TemporalAccessor temporal = formatter.parse(dateStr);
+ private static Instant parseInstant(DateTimeFormatter formatter, String dateStr, ParsePosition parsePosition) {
+ // prepare for reuse
+ parsePosition.setIndex(0);
+ parsePosition.setErrorIndex(-1);
+ final TemporalAccessor temporal = formatter.parse(dateStr, parsePosition);
+ // check that all content has been parsed
+ if (parsePosition.getIndex() < dateStr.length()) {
+ final String abbr;
+ if (dateStr.length() > 64) {
+ abbr = dateStr.subSequence(0, 64).toString() + "...";
+ } else {
+ abbr = dateStr;
+ }
+ throw new DateTimeParseException("Text '" + abbr + "' could not be parsed, unparsed text found at index " +
+ parsePosition.getIndex(), dateStr, parsePosition.getIndex());
+ }
+
// Get Date; mandatory
LocalDate date = temporal.query(TemporalQueries.localDate());//mandatory
if (date == null) {
[27/30] lucene-solr:jira/http2: SOLR-12782: UninvertingReader avoids
FieldInfo creation. Wrapping is now a bit more lightweight: Does not create
FieldInfo for fields that can't be uninverted (saves mem) and can avoid
wrapping the reader altogether if the
Posted by da...@apache.org.
SOLR-12782: UninvertingReader avoids FieldInfo creation.
Wrapping is now a bit more lightweight: Does not create FieldInfo for fields that
can't be uninverted (saves mem) and can avoid wrapping the reader altogether if there's nothing to uninvert.
IndexSchema.getUninversionMap refactored to getUninversionMapper and no longer merges FieldInfos.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e2b8becc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e2b8becc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e2b8becc
Branch: refs/heads/jira/http2
Commit: e2b8beccb03b6f9068c00e4711eac2500d120d44
Parents: eb47099
Author: David Smiley <ds...@apache.org>
Authored: Fri Oct 5 20:40:39 2018 -0700
Committer: David Smiley <ds...@apache.org>
Committed: Fri Oct 5 20:40:39 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 5 +
.../solr/handler/component/ExpandComponent.java | 17 ++-
.../UninvertDocValuesMergePolicyFactory.java | 5 +-
.../org/apache/solr/schema/IndexSchema.java | 53 ++++------
.../solr/search/CollapsingQParserPlugin.java | 18 ++--
.../java/org/apache/solr/search/Insanity.java | 5 +-
.../apache/solr/search/SolrIndexSearcher.java | 17 +--
.../solr/uninverting/UninvertingReader.java | 103 +++++++++++--------
.../solr/update/DeleteByQueryWrapper.java | 2 +-
.../solr/uninverting/TestUninvertingReader.java | 18 ++--
10 files changed, 120 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 41f8735..4bd8158 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -169,9 +169,14 @@ Bug Fixes
Improvements
----------------------
+
* SOLR-12767: Solr now always includes in the response of update requests the achieved replication factor
(Tomás Fernández Löbbe)
+* SOLR-12782: UninvertingReader wrapping is now a bit more lightweight: Does not create FieldInfo for fields that
+ can't be uninverted (saves mem) and can avoid wrapping the reader altogether if there's nothing to uninvert.
+ IndexSchema.getUninversionMap refactored to getUninversionMapper and no longer merges FieldInfos. (David Smiley)
+
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
index f0870aa..e79161d 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
@@ -19,10 +19,9 @@ package org.apache.solr.handler.component;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.hppc.IntObjectHashMap;
@@ -215,10 +214,9 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
if(fieldType instanceof StrField) {
//Get The Top Level SortedDocValues
if(CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
- Map<String, UninvertingReader.Type> mapping = new HashMap();
- mapping.put(field, UninvertingReader.Type.SORTED);
- @SuppressWarnings("resource")
- UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), field), mapping);
+ @SuppressWarnings("resource") LeafReader uninvertingReader = UninvertingReader.wrap(
+ new ReaderWrapper(searcher.getSlowAtomicReader(), field),
+ Collections.singletonMap(field, UninvertingReader.Type.SORTED)::get);
values = uninvertingReader.getSortedDocValues(field);
} else {
values = DocValues.getSorted(reader, field);
@@ -386,10 +384,9 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
if(values != null) {
//Get The Top Level SortedDocValues again so we can re-iterate:
if(CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
- Map<String, UninvertingReader.Type> mapping = new HashMap();
- mapping.put(field, UninvertingReader.Type.SORTED);
- @SuppressWarnings("resource")
- UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), field), mapping);
+ @SuppressWarnings("resource") LeafReader uninvertingReader = UninvertingReader.wrap(
+ new ReaderWrapper(searcher.getSlowAtomicReader(), field),
+ Collections.singletonMap(field, UninvertingReader.Type.SORTED)::get);
values = uninvertingReader.getSortedDocValues(field);
} else {
values = DocValues.getSorted(reader, field);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/index/UninvertDocValuesMergePolicyFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/index/UninvertDocValuesMergePolicyFactory.java b/solr/core/src/java/org/apache/solr/index/UninvertDocValuesMergePolicyFactory.java
index b6bfbed..46f03d3 100644
--- a/solr/core/src/java/org/apache/solr/index/UninvertDocValuesMergePolicyFactory.java
+++ b/solr/core/src/java/org/apache/solr/index/UninvertDocValuesMergePolicyFactory.java
@@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterCodecReader;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.OneMergeWrappingMergePolicy;
@@ -134,13 +135,13 @@ public class UninvertDocValuesMergePolicyFactory extends WrapperMergePolicyFacto
*/
private class UninvertingFilterCodecReader extends FilterCodecReader {
- private final UninvertingReader uninvertingReader;
+ private final LeafReader uninvertingReader;
private final DocValuesProducer docValuesProducer;
public UninvertingFilterCodecReader(CodecReader in, Map<String,UninvertingReader.Type> uninversionMap) {
super(in);
- this.uninvertingReader = new UninvertingReader(in, uninversionMap);
+ this.uninvertingReader = UninvertingReader.wrap(in, uninversionMap::get);
this.docValuesProducer = new DocValuesProducer() {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index f233717..1129a2c 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -44,11 +44,7 @@ import java.util.stream.Stream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.MultiFields;
import org.apache.lucene.queries.payloads.PayloadDecoder;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Version;
@@ -150,7 +146,7 @@ public class IndexSchema {
protected Map<String, List<CopyField>> copyFieldsMap = new HashMap<>();
public Map<String,List<CopyField>> getCopyFieldsMap() { return Collections.unmodifiableMap(copyFieldsMap); }
-
+
protected DynamicCopy[] dynamicCopyFields;
public DynamicCopy[] getDynamicCopyFields() { return dynamicCopyFields; }
@@ -183,14 +179,14 @@ public class IndexSchema {
throw new RuntimeException(e);
}
}
-
+
/**
* @since solr 1.4
*/
public SolrResourceLoader getResourceLoader() {
return loader;
}
-
+
/** Gets the name of the resource used to instantiate this schema. */
public String getResourceName() {
return resourceName;
@@ -205,7 +201,7 @@ public class IndexSchema {
public String getSchemaName() {
return name;
}
-
+
/** The Default Lucene Match Version for this IndexSchema */
public Version getDefaultLuceneMatchVersion() {
return solrConfig.luceneMatchVersion;
@@ -223,7 +219,7 @@ public class IndexSchema {
* <p>
* Modifying this Map (or any item in it) will affect the real schema
* </p>
- *
+ *
* <p>
* NOTE: this function is not thread safe. However, it is safe to use within the standard
* <code>inform( SolrCore core )</code> function for <code>SolrCoreAware</code> classes.
@@ -237,11 +233,11 @@ public class IndexSchema {
* in the index, keyed on field type name.
*
* <p>
- * Modifying this Map (or any item in it) will affect the real schema. However if you
+ * Modifying this Map (or any item in it) will affect the real schema. However if you
* make any modifications, be sure to call {@link IndexSchema#refreshAnalyzers()} to
* update the Analyzers for the registered fields.
* </p>
- *
+ *
* <p>
* NOTE: this function is not thread safe. However, it is safe to use within the standard
* <code>inform( SolrCore core )</code> function for <code>SolrCoreAware</code> classes.
@@ -270,7 +266,7 @@ public class IndexSchema {
if (null == similarity) {
similarity = similarityFactory.getSimilarity();
}
- return similarity;
+ return similarity;
}
protected SimilarityFactory similarityFactory;
@@ -279,7 +275,7 @@ public class IndexSchema {
/** Returns the SimilarityFactory that constructed the Similarity for this index */
public SimilarityFactory getSimilarityFactory() { return similarityFactory; }
-
+
/**
* Returns the Analyzer used when indexing documents for this index
*
@@ -300,7 +296,7 @@ public class IndexSchema {
*/
public Analyzer getQueryAnalyzer() { return queryAnalyzer; }
-
+
protected SchemaField uniqueKeyField;
/**
@@ -342,35 +338,28 @@ public class IndexSchema {
}
return f;
}
-
+
/**
* This will re-create the Analyzers. If you make any modifications to
* the Field map ({@link IndexSchema#getFields()}, this function is required
* to synch the internally cached field analyzers.
- *
+ *
* @since solr 1.3
*/
public void refreshAnalyzers() {
indexAnalyzer = new SolrIndexAnalyzer();
queryAnalyzer = new SolrQueryAnalyzer();
}
-
- public Map<String,UninvertingReader.Type> getUninversionMap(IndexReader reader) {
- final Map<String,UninvertingReader.Type> map = new HashMap<>();
- for (FieldInfo f : MultiFields.getMergedFieldInfos(reader)) {
- if (f.getDocValuesType() == DocValuesType.NONE) {
- // we have a field (of some kind) in the reader w/o DocValues
- // if we have an equivalent indexed=true field in the schema, trust it's uninversion type (if any)
- final SchemaField sf = getFieldOrNull(f.name);
- if (sf != null && sf.indexed()) {
- final UninvertingReader.Type type = sf.getType().getUninversionType(sf);
- if (type != null) {
- map.put(f.name, type);
- }
- }
+
+ /** @see UninvertingReader */
+ public Function<String, UninvertingReader.Type> getUninversionMapper() {
+ return name -> {
+ SchemaField sf = getFieldOrNull(name);
+ if (sf == null) {
+ return null;
}
- }
- return map;
+ return sf.getType().getUninversionType(sf);
+ };
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 13417ab..8875afc 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -19,8 +19,8 @@ package org.apache.solr.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -1256,15 +1256,13 @@ public class CollapsingQParserPlugin extends QParserPlugin {
if(collapseFieldType instanceof StrField) {
if(HINT_TOP_FC.equals(hint)) {
- /*
- * This hint forces the use of the top level field cache for String fields.
- * This is VERY fast at query time but slower to warm and causes insanity.
- */
-
- Map<String, UninvertingReader.Type> mapping = new HashMap();
- mapping.put(collapseField, UninvertingReader.Type.SORTED);
- @SuppressWarnings("resource") final UninvertingReader uninvertingReader =
- new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
+ /*
+ * This hint forces the use of the top level field cache for String fields.
+ * This is VERY fast at query time but slower to warm and causes insanity.
+ */
+ @SuppressWarnings("resource") final LeafReader uninvertingReader = UninvertingReader.wrap(
+ new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField),
+ Collections.singletonMap(collapseField, UninvertingReader.Type.SORTED)::get);
docValuesProducer = new EmptyDocValuesProducer() {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/search/Insanity.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/Insanity.java b/solr/core/src/java/org/apache/solr/search/Insanity.java
index 8fe081f..67d1320 100644
--- a/solr/core/src/java/org/apache/solr/search/Insanity.java
+++ b/solr/core/src/java/org/apache/solr/search/Insanity.java
@@ -49,8 +49,9 @@ public class Insanity {
* instead of a numeric.
*/
public static LeafReader wrapInsanity(LeafReader sane, String insaneField) {
- return new UninvertingReader(new InsaneReader(sane, insaneField),
- Collections.singletonMap(insaneField, UninvertingReader.Type.SORTED));
+ return UninvertingReader.wrap(
+ new InsaneReader(sane, insaneField),
+ Collections.singletonMap(insaneField, UninvertingReader.Type.SORTED)::get);
}
/** Hides the proper numeric dv type for the field */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index ae9e485..9b36c97 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -157,26 +157,11 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
private static DirectoryReader wrapReader(SolrCore core, DirectoryReader reader) throws IOException {
assert reader != null;
return ExitableDirectoryReader.wrap(
- wrapUninvertingReaderPerSegment(core, reader),
+ UninvertingReader.wrap(reader, core.getLatestSchema().getUninversionMapper()),
SolrQueryTimeoutImpl.getInstance());
}
/**
- * If docvalues are enabled or disabled after data has already been indexed for a field, such that
- * only some segments have docvalues, uninverting on the top level reader will cause
- * IllegalStateException to be thrown when trying to use a field with such mixed data. This is because
- * the {@link IndexSchema#getUninversionMap(IndexReader)} method decides to put a field
- * into the uninverteding map only if *NO* segment in the index contains docvalues for that field.
- *
- * Therefore, this class provides a uninverting map per segment such that for any field,
- * DocValues are used from segments if they exist and uninversion of the field is performed on the rest
- * of the segments.
- */
- private static DirectoryReader wrapUninvertingReaderPerSegment(SolrCore core, DirectoryReader reader) throws IOException {
- return UninvertingReader.wrap(reader, r -> core.getLatestSchema().getUninversionMap(r));
- }
-
- /**
* Builds the necessary collector chain (via delegate wrapping) and executes the query against it. This method takes
* into consideration both the explicitly provided collector and postFilter as well as any needed collector wrappers
* for dealing with options specified in the QueryCommand.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
index 9f0f527..176795d 100644
--- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
+++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
@@ -21,11 +21,11 @@ import java.util.ArrayList;
import java.util.Map;
import java.util.function.Function;
-import org.apache.lucene.document.BinaryDocValuesField; // javadocs
-import org.apache.lucene.document.NumericDocValuesField; // javadocs
-import org.apache.lucene.document.SortedDocValuesField; // javadocs
-import org.apache.lucene.document.SortedSetDocValuesField; // javadocs
-import org.apache.lucene.document.StringField; // javadocs
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValuesType;
@@ -53,7 +53,7 @@ import org.apache.solr.uninverting.FieldCache.CacheEntry;
* based on the core cache key of the wrapped LeafReader.
*/
public class UninvertingReader extends FilterLeafReader {
-
+
/**
* Specifies the type of uninversion to apply for the field.
*/
@@ -173,33 +173,33 @@ public class UninvertingReader extends FilterLeafReader {
SORTED_SET_DOUBLE
}
-
+
+ /** @see #wrap(DirectoryReader, Function) */
+ public static DirectoryReader wrap(DirectoryReader reader, Map<String, Type> mapping) throws IOException {
+ return wrap(reader, mapping::get);
+ }
+
/**
- *
- * Wraps a provided DirectoryReader. Note that for convenience, the returned reader
+ * Wraps a provided {@link DirectoryReader}. Note that for convenience, the returned reader
* can be used normally (e.g. passed to {@link DirectoryReader#openIfChanged(DirectoryReader)})
* and so on.
*
* @param in input directory reader
- * @param perSegmentMapper function to map a segment reader to a mapping of fields to their uninversion type
+ * @param mapper function to map a field name to an uninversion type. A Null result means to not uninvert.
* @return a wrapped directory reader
*/
- public static DirectoryReader wrap(DirectoryReader in, final Function<LeafReader, Map<String,Type>> perSegmentMapper) throws IOException {
- return new UninvertingDirectoryReader(in, perSegmentMapper);
- }
-
- public static DirectoryReader wrap(DirectoryReader in, final Map<String,Type> mapping) throws IOException {
- return UninvertingReader.wrap(in, (r) -> mapping);
+ public static DirectoryReader wrap(DirectoryReader in, Function<String, Type> mapper) throws IOException {
+ return new UninvertingDirectoryReader(in, mapper);
}
-
+
static class UninvertingDirectoryReader extends FilterDirectoryReader {
- final Function<LeafReader, Map<String,Type>> mapper;
+ final Function<String, Type> mapper;
- public UninvertingDirectoryReader(DirectoryReader in, final Function<LeafReader, Map<String,Type>> mapper) throws IOException {
+ public UninvertingDirectoryReader(DirectoryReader in, final Function<String, Type> mapper) throws IOException {
super(in, new FilterDirectoryReader.SubReaderWrapper() {
@Override
public LeafReader wrap(LeafReader reader) {
- return new UninvertingReader(reader, mapper.apply(reader));
+ return UninvertingReader.wrap(reader, mapper);
}
});
this.mapper = mapper;
@@ -219,26 +219,26 @@ public class UninvertingReader extends FilterLeafReader {
return in.getReaderCacheHelper();
}
}
-
- final Map<String,Type> mapping;
- final FieldInfos fieldInfos;
-
- /**
- * Create a new UninvertingReader with the specified mapping
+
+ /**
+ * Create a new UninvertingReader with the specified mapping, wrapped around the input. It may be deemed that there
+ * is no mapping to do, in which case the input is returned.
* <p>
- * Expert: This should almost never be used. Use {@link #wrap(DirectoryReader, Function)}
- * instead.
- *
+ * Expert: This should almost never be used. Use {@link #wrap(DirectoryReader, Function)} instead.
+ *
* @lucene.internal
*/
- public UninvertingReader(LeafReader in, Map<String,Type> mapping) {
- super(in);
- this.mapping = mapping;
- ArrayList<FieldInfo> filteredInfos = new ArrayList<>();
+ public static LeafReader wrap(LeafReader in, Function<String, Type> mapping) {
+ boolean wrap = false;
+
+ // Calculate a new FieldInfos that has DocValuesType where we didn't before
+ ArrayList<FieldInfo> newFieldInfos = new ArrayList<>(in.getFieldInfos().size());
for (FieldInfo fi : in.getFieldInfos()) {
DocValuesType type = fi.getDocValuesType();
- if (type == DocValuesType.NONE) {
- Type t = mapping.get(fi.name);
+ // fields which currently don't have docValues, but are uninvertable (indexed or points data present)
+ if (type == DocValuesType.NONE &&
+ (fi.getIndexOptions() != IndexOptions.NONE || (fi.getPointNumBytes() > 0 && fi.getPointDimensionCount() == 1))) {
+ Type t = mapping.apply(fi.name); // could definitely return null, thus still can't uninvert it
if (t != null) {
if (t == Type.INTEGER_POINT || t == Type.LONG_POINT || t == Type.FLOAT_POINT || t == Type.DOUBLE_POINT) {
// type uses points
@@ -280,11 +280,30 @@ public class UninvertingReader extends FilterLeafReader {
}
}
}
- filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
- fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(),
- fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
+ if (type != fi.getDocValuesType()) { // we changed it
+ wrap = true;
+ newFieldInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(),
+ fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes(),
+ fi.getPointDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()));
+ } else {
+ newFieldInfos.add(fi);
+ }
+ }
+ if (!wrap) {
+ return in;
+ } else {
+ FieldInfos fieldInfos = new FieldInfos(newFieldInfos.toArray(new FieldInfo[newFieldInfos.size()]));
+ return new UninvertingReader(in, mapping, fieldInfos);
}
- fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
+ }
+
+ final Function<String, Type> mapping;
+ final FieldInfos fieldInfos;
+
+ private UninvertingReader(LeafReader in, Function<String, Type> mapping, FieldInfos fieldInfos) {
+ super(in);
+ this.mapping = mapping;
+ this.fieldInfos = fieldInfos;
}
@Override
@@ -388,11 +407,7 @@ public class UninvertingReader extends FilterLeafReader {
* if the field doesn't exist or doesn't have a mapping.
*/
private Type getType(String field) {
- FieldInfo info = fieldInfos.fieldInfo(field);
- if (info == null || info.getDocValuesType() == DocValuesType.NONE) {
- return null;
- }
- return mapping.get(field);
+ return mapping.apply(field);
}
// NOTE: delegating the cache helpers is wrong since this wrapper alters the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java
index 1c87a39..9a16f60 100644
--- a/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java
+++ b/solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java
@@ -50,7 +50,7 @@ final class DeleteByQueryWrapper extends Query {
}
LeafReader wrap(LeafReader reader) {
- return new UninvertingReader(reader, schema.getUninversionMap(reader));
+ return UninvertingReader.wrap(reader, schema.getUninversionMapper());
}
// we try to be well-behaved, but we are not (and IW's applyQueryDeletes isn't much better...)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e2b8becc/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
index f140ce2..9ec1234 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestUninvertingReader.java
@@ -36,6 +36,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -367,24 +368,29 @@ public class TestUninvertingReader extends LuceneTestCase {
uninvertingMap.put("dv", Type.LEGACY_INTEGER);
uninvertingMap.put("dint", Type.INTEGER_POINT);
- DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir),
- uninvertingMap);
+ DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), uninvertingMap);
LeafReader leafReader = ir.leaves().get(0).reader();
+ FieldInfos fieldInfos = leafReader.getFieldInfos();
+ LeafReader originalLeafReader = ((UninvertingReader)leafReader).getDelegate();
+
+ assertNotSame(originalLeafReader.getFieldInfos(), fieldInfos);
+ assertSame("do not rebuild FieldInfo for unaffected fields",
+ originalLeafReader.getFieldInfos().fieldInfo("id"), fieldInfos.fieldInfo("id"));
- FieldInfo intFInfo = leafReader.getFieldInfos().fieldInfo("int");
+ FieldInfo intFInfo = fieldInfos.fieldInfo("int");
assertEquals(DocValuesType.NUMERIC, intFInfo.getDocValuesType());
assertEquals(0, intFInfo.getPointDimensionCount());
assertEquals(0, intFInfo.getPointNumBytes());
- FieldInfo dintFInfo = leafReader.getFieldInfos().fieldInfo("dint");
+ FieldInfo dintFInfo = fieldInfos.fieldInfo("dint");
assertEquals(DocValuesType.NUMERIC, dintFInfo.getDocValuesType());
assertEquals(1, dintFInfo.getPointDimensionCount());
assertEquals(4, dintFInfo.getPointNumBytes());
- FieldInfo dvFInfo = leafReader.getFieldInfos().fieldInfo("dv");
+ FieldInfo dvFInfo = fieldInfos.fieldInfo("dv");
assertEquals(DocValuesType.NUMERIC, dvFInfo.getDocValuesType());
- FieldInfo storedFInfo = leafReader.getFieldInfos().fieldInfo("stored");
+ FieldInfo storedFInfo = fieldInfos.fieldInfo("stored");
assertEquals(DocValuesType.NONE, storedFInfo.getDocValuesType());
TestUtil.checkReader(ir);
[04/30] lucene-solr:jira/http2: SOLR-12824: fixed test failure
Posted by da...@apache.org.
SOLR-12824: fixed test failure
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4ca885ac
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4ca885ac
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4ca885ac
Branch: refs/heads/jira/http2
Commit: 4ca885ac9eb3c68e31205df6d7456f59b2c08c70
Parents: 961d565
Author: Noble Paul <no...@apache.org>
Authored: Tue Oct 2 21:16:15 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Tue Oct 2 21:17:22 2018 +1000
----------------------------------------------------------------------
.../cloud/autoscaling/sim/TestSimTriggerIntegration.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4ca885ac/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index 744155e..50769e8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -36,14 +36,15 @@ import com.google.common.util.concurrent.AtomicDouble;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.CloudTestUtils;
import org.apache.solr.cloud.autoscaling.ActionContext;
+import org.apache.solr.cloud.autoscaling.CapturedEvent;
import org.apache.solr.cloud.autoscaling.ComputePlanAction;
import org.apache.solr.cloud.autoscaling.ExecutePlanAction;
import org.apache.solr.cloud.autoscaling.NodeLostTrigger;
@@ -53,8 +54,8 @@ import org.apache.solr.cloud.autoscaling.TriggerActionBase;
import org.apache.solr.cloud.autoscaling.TriggerEvent;
import org.apache.solr.cloud.autoscaling.TriggerEventQueue;
import org.apache.solr.cloud.autoscaling.TriggerListenerBase;
-import org.apache.solr.cloud.autoscaling.CapturedEvent;
import org.apache.solr.cloud.autoscaling.TriggerValidationException;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.LiveNodesListener;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
@@ -1315,11 +1316,11 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
assertTrue(totalReplicaRate.get() > 100.0);
// check operations
- List<Map<String, Object>> ops = (List<Map<String, Object>>)ev.context.get("properties.operations");
+ List<MapWriter> ops = (List<MapWriter>)ev.context.get("properties.operations");
assertNotNull(ops);
assertTrue(ops.size() > 1);
- for (Map<String, Object> m : ops) {
- assertEquals("ADDREPLICA", m.get("params.action"));
+ for (MapWriter m : ops) {
+ assertEquals("ADDREPLICA", m._get("params.action", null));
}
}
}
[03/30] lucene-solr:jira/http2: SOLR-12824: NamedList to implement
MapWriter interface
Posted by da...@apache.org.
SOLR-12824: NamedList to implement MapWriter interface
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/961d5659
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/961d5659
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/961d5659
Branch: refs/heads/jira/http2
Commit: 961d565955ed9b2acd8f44319c16b627d6192714
Parents: fe844c7
Author: Noble Paul <no...@apache.org>
Authored: Tue Oct 2 17:34:58 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Tue Oct 2 17:34:58 2018 +1000
----------------------------------------------------------------------
.../solr/cloud/CollectionsAPISolrJTest.java | 11 ++--
.../solr/cloud/autoscaling/CapturedEvent.java | 2 +-
.../SearchRateTriggerIntegrationTest.java | 7 +-
.../test/org/apache/solr/util/TestUtils.java | 67 +++++++++++++++++---
.../client/solrj/response/SolrResponseBase.java | 12 +++-
.../java/org/apache/solr/common/MapWriter.java | 7 +-
.../org/apache/solr/common/MapWriterMap.java | 39 ++++++++++++
.../java/org/apache/solr/common/util/Utils.java | 49 ++++++++++----
8 files changed, 162 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 5390704..cd9087d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -189,15 +189,14 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
- String nodeName = ((NamedList) response.getResponse().get("success")).getName(0);
- String corename = (String) response.getResponse()._get(asList("success", nodeName,"core"),null);
+ String nodeName = (String) response._get("success[0]/key", null);
+ String corename = (String) response._get(asList("success", nodeName, "core"), null);
try (HttpSolrClient coreclient = getHttpSolrClient(cluster.getSolrClient().getZkStateReader().getBaseUrlForNodeName(nodeName))) {
CoreAdminResponse status = CoreAdminRequest.getStatus(corename, coreclient);
- NamedList m = status.getResponse();
- assertEquals(collectionName, m._get(asList("status", corename, "cloud", "collection"), null));
- assertNotNull(m._get(asList("status", corename, "cloud", "shard"), null));
- assertNotNull(m._get(asList("status", corename, "cloud", "replica"), null));
+ assertEquals(collectionName, status._get(asList("status", corename, "cloud", "collection"), null));
+ assertNotNull(status._get(asList("status", corename, "cloud", "shard"), null));
+ assertNotNull(status._get(asList("status", corename, "cloud", "replica"), null));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java
index 462e948..d5c3127 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java
@@ -38,7 +38,7 @@ public class CapturedEvent {
public CapturedEvent(long timestamp, ActionContext context, AutoScalingConfig.TriggerListenerConfig config, TriggerEventProcessorStage stage, String actionName,
TriggerEvent event, String message) {
if (context != null) {
- context.toMap(this.context);
+ context._forEachEntry((o, o2) -> CapturedEvent.this.context.put((String) o, o2));
TriggerEvent.fixOps("properties." + TriggerEvent.REQUESTED_OPS, this.context);
TriggerEvent.fixOps("properties." + TriggerEvent.UNSUPPORTED_OPS, this.context);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
index 5d7ce99..106ec17 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
@@ -38,6 +38,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.CloudTestUtils;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -264,11 +265,11 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
assertEquals(collectionRate, totalReplicaRate.get(), 5.0);
// check operations
- List<Map<String, Object>> ops = (List<Map<String, Object>>) ev.context.get("properties.operations");
+ List<MapWriter> ops = (List<MapWriter>) ev.context.get("properties.operations");
assertNotNull(ops);
assertTrue(ops.size() > 1);
- for (Map<String, Object> m : ops) {
- assertEquals("ADDREPLICA", m.get("params.action"));
+ for (MapWriter m : ops) {
+ assertEquals("ADDREPLICA", m._get("params.action",null));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/core/src/test/org/apache/solr/util/TestUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.java
index 88dbba2..27db2d6 100644
--- a/solr/core/src/test/org/apache/solr/util/TestUtils.java
+++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java
@@ -19,7 +19,6 @@ package org.apache.solr.util;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.StringReader;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -40,6 +39,7 @@ import org.apache.solr.common.util.Utils;
import org.junit.Assert;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Arrays.asList;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
@@ -51,12 +51,12 @@ import static org.apache.solr.common.util.Utils.fromJSONString;
public class TestUtils extends SolrTestCaseJ4 {
public void testJoin() {
- assertEquals("a|b|c", StrUtils.join(Arrays.asList("a","b","c"), '|'));
- assertEquals("a,b,c", StrUtils.join(Arrays.asList("a","b","c"), ','));
- assertEquals("a\\,b,c", StrUtils.join(Arrays.asList("a,b","c"), ','));
- assertEquals("a,b|c", StrUtils.join(Arrays.asList("a,b","c"), '|'));
+ assertEquals("a|b|c", StrUtils.join(asList("a","b","c"), '|'));
+ assertEquals("a,b,c", StrUtils.join(asList("a","b","c"), ','));
+ assertEquals("a\\,b,c", StrUtils.join(asList("a,b","c"), ','));
+ assertEquals("a,b|c", StrUtils.join(asList("a,b","c"), '|'));
- assertEquals("a\\\\b|c", StrUtils.join(Arrays.asList("a\\b","c"), '|'));
+ assertEquals("a\\\\b|c", StrUtils.join(asList("a\\b","c"), '|'));
}
public void testEscapeTextWithSeparator() {
@@ -191,8 +191,8 @@ public class TestUtils extends SolrTestCaseJ4 {
jbc.marshal((MapWriter) ew -> {
ew.put("set-user", fromJSONString("{x:y}"));
ew.put("set-user", fromJSONString("{x:y,x1:y1}"));
- ew.put("single", Arrays.asList(fromJSONString("[{x:y,x1:y1},{x2:y2}]"), fromJSONString( "{x2:y2}")));
- ew.put("multi", Arrays.asList(fromJSONString("{x:y,x1:y1}"), fromJSONString( "{x2:y2}")));
+ ew.put("single", asList(fromJSONString("[{x:y,x1:y1},{x2:y2}]"), fromJSONString( "{x2:y2}")));
+ ew.put("multi", asList(fromJSONString("{x:y,x1:y1}"), fromJSONString( "{x2:y2}")));
}, baos);
}
@@ -252,6 +252,57 @@ public class TestUtils extends SolrTestCaseJ4 {
assertEquals("x-update", Utils.getObjectByPath(m,false, "authorization/permissions[1]/name"));
}
+
+ public void testMapWriterIdx(){
+ String json = "{" +
+ " 'responseHeader':{" +
+ " 'status':0," +
+ " 'QTime':6752}," +
+ " 'success':{" +
+ " '127.0.0.1:56443_solr':{" +
+ " 'responseHeader':{" +
+ " 'status':0," +
+ " 'QTime':4276}," +
+ " 'core':'corestatus_test_shard2_replica_n5'}," +
+ " '127.0.0.1:56445_solr':{" +
+ " 'responseHeader':{" +
+ " 'status':0," +
+ " 'QTime':4271}," +
+ " 'core':'corestatus_test_shard1_replica_n1'}," +
+ " '127.0.0.1:56446_solr':{" +
+ " 'responseHeader':{" +
+ " 'status':0," +
+ " 'QTime':5015}," +
+ " 'core':'corestatus_test_shard1_replica_n2'}," +
+ " '127.0.0.1:56444_solr':{" +
+ " 'responseHeader':{" +
+ " 'status':0," +
+ " 'QTime':5033}," +
+ " 'core':'corestatus_test_shard2_replica_n3'}}}";
+ Map m = (Map) fromJSONString(json);
+
+ assertEquals("127.0.0.1:56443_solr", Utils.getObjectByPath(m,false, "success[0]/key"));
+ assertEquals("corestatus_test_shard2_replica_n5", Utils.getObjectByPath(m, false,asList("success[0]", "value", "core") ));
+ assertEquals(4276L, Utils.getObjectByPath(m, false,asList("success[0]", "value", "responseHeader", "QTime") ));
+
+ assertEquals("127.0.0.1:56444_solr", Utils.getObjectByPath(m,false, "success[3]/key"));
+ assertEquals("corestatus_test_shard2_replica_n3", Utils.getObjectByPath(m, false,asList("success[3]", "value", "core") ));
+ assertEquals(5033L, Utils.getObjectByPath(m, false,asList("success[3]", "value", "responseHeader", "QTime") ));
+
+ Map nodes = (Map) m.get("success");
+ m.put("success", (MapWriter) ew -> nodes.forEach((o, o2) -> ew.putNoEx((String) o,o2)));
+ assertEquals("127.0.0.1:56443_solr", Utils.getObjectByPath(m,false, "success[0]/key"));
+ assertEquals("corestatus_test_shard2_replica_n5", Utils.getObjectByPath(m, false,asList("success[0]", "value", "core") ));
+ assertEquals(4276L, Utils.getObjectByPath(m, false,asList("success[0]", "value", "responseHeader", "QTime") ));
+
+ assertEquals("127.0.0.1:56444_solr", Utils.getObjectByPath(m,false, "success[3]/key"));
+ assertEquals("corestatus_test_shard2_replica_n3", Utils.getObjectByPath(m, false,asList("success[3]", "value", "core") ));
+ assertEquals(5033L, Utils.getObjectByPath(m, false,asList("success[3]", "value", "responseHeader", "QTime") ));
+ final int[] count = {0};
+ NamedList nl = new NamedList(m);
+ nl._forEachEntry("success", (o, o2) -> count[0]++);
+ assertEquals(count[0], 4);
+ }
public void testMergeJson() {
Map<String, Object> sink = (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java
index d20481c..ffadb38 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java
@@ -16,7 +16,10 @@
*/
package org.apache.solr.client.solrj.response;
+import java.io.IOException;
+
import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.NamedList;
/**
@@ -24,12 +27,17 @@ import org.apache.solr.common.util.NamedList;
*
* @since solr 1.3
*/
-public class SolrResponseBase extends SolrResponse
+public class SolrResponseBase extends SolrResponse implements MapWriter
{
private long elapsedTime = -1;
private NamedList<Object> response = null;
private String requestUrl = null;
-
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ if (response != null) response.writeMap(ew);
+ }
+
@Override
public long getElapsedTime() {
return elapsedTime;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index 3ec37e7..3256b6a 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -91,12 +91,17 @@ public interface MapWriter extends MapSerializable {
Object v = Utils.getObjectByPath(this, false, path);
return v == null ? def : v;
}
+
default void _forEachEntry(String path, BiConsumer fun) {
Utils.forEachMapEntry(this, path, fun);
}
+ default void _forEachEntry(List<String> path, BiConsumer fun) {
+ Utils.forEachMapEntry(this, path, fun);
+ }
+
default void _forEachEntry(BiConsumer fun) {
- Utils.forEachMapEntry(this, null, fun);
+ Utils.forEachMapEntry(this, fun);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
new file mode 100644
index 0000000..f6580b2
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class MapWriterMap implements MapWriter {
+ private final Map delegate;
+
+ public MapWriterMap(Map delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ delegate.forEach((k, v) -> ew.putNoEx(k == null ? null : k.toString(), v));
+ }
+
+ @Override
+ public Map toMap(Map<String, Object> map) {
+ return delegate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/961d5659/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 2d1df42..c8274f3 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -30,6 +30,7 @@ import java.net.URL;
import java.net.URLDecoder;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -54,6 +55,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.MapWriterMap;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SpecProvider;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -104,7 +106,16 @@ public class Utils {
}
public static void forEachMapEntry(MapWriter mw, String path, BiConsumer fun) {
- Object o = path == null ? mw : Utils.getObjectByPath(mw, false, path);
+ Object o = Utils.getObjectByPath(mw, false, path);
+ forEachMapEntry(o, fun);
+ }
+
+ public static void forEachMapEntry(MapWriter mw, List<String> path, BiConsumer fun) {
+ Object o = Utils.getObjectByPath(mw, false, path);
+ forEachMapEntry(o, fun);
+ }
+
+ public static void forEachMapEntry(Object o, BiConsumer fun) {
if (o instanceof MapWriter) {
MapWriter m = (MapWriter) o;
try {
@@ -365,7 +376,7 @@ public class Utils {
for (int i = 0; i < hierarchy.size(); i++) {
int idx = -1;
String s = hierarchy.get(i);
- if (s.endsWith("]")) {
+ if (s != null && s.endsWith("]")) {
Matcher matcher = ARRAY_ELEMENT_INDEX.matcher(s);
if (matcher.find()) {
s = matcher.group(1);
@@ -376,8 +387,14 @@ public class Utils {
Object o = getVal(obj, s, -1);
if (o == null) return null;
if (idx > -1) {
- List l = (List) o;
- o = idx < l.size() ? l.get(idx) : null;
+ if (o instanceof MapWriter) {
+ o = getVal(o, null, idx);
+ } else if (o instanceof Map) {
+ o = getVal(new MapWriterMap((Map) o), null, idx);
+ } else {
+ List l = (List) o;
+ o = idx < l.size() ? l.get(idx) : null;
+ }
}
if (!isMapLike(o)) return null;
obj = o;
@@ -385,10 +402,7 @@ public class Utils {
Object val = getVal(obj, s, -1);
if (val == null) return null;
if (idx > -1) {
- if (val instanceof MapWriter) {
- val = getVal((MapWriter) val, null, idx);
-
- } else if (val instanceof IteratorWriter) {
+ if (val instanceof IteratorWriter) {
val = getValueAt((IteratorWriter) val, idx);
} else {
List l = (List) val;
@@ -427,6 +441,19 @@ public class Utils {
}
+ static class MapWriterEntry<V> extends AbstractMap.SimpleEntry<String, V> implements MapWriter, Map.Entry<String, V> {
+ MapWriterEntry(String key, V value) {
+ super(key, value);
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("key", getKey());
+ ew.put("value", getValue());
+ }
+
+ }
+
private static boolean isMapLike(Object o) {
return o instanceof Map || o instanceof NamedList || o instanceof MapWriter;
}
@@ -440,10 +467,10 @@ public class Utils {
@Override
public MapWriter.EntryWriter put(String k, Object v) {
if (result[0] != null) return this;
- if (k != null) {
- if (key.equals(k)) result[0] = v;
+ if (idx < 0) {
+ if (k.equals(key)) result[0] = v;
} else {
- if (++count == idx) result[0] = v;
+ if (++count == idx) result[0] = new MapWriterEntry(k, v);
}
return this;
}
[20/30] lucene-solr:jira/http2: SOLR-12811: Add enclosingDisk and
associated geometric Stream Evaluators
Posted by da...@apache.org.
SOLR-12811: Add enclosingDisk and associated geometric Stream Evaluators
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a0487b04
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a0487b04
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a0487b04
Branch: refs/heads/jira/http2
Commit: a0487b04ea0b676e3732e49de1ca0e38a91aab3c
Parents: 0f10000
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Oct 4 10:27:32 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Oct 4 10:36:27 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/Lang.java | 4 ++
.../solrj/io/eval/EnclosingDiskEvaluator.java | 64 ++++++++++++++++++++
.../solrj/io/eval/GetCenterEvaluator.java | 52 ++++++++++++++++
.../solrj/io/eval/GetRadiusEvaluator.java | 44 ++++++++++++++
.../io/eval/GetSupportPointsEvaluator.java | 56 +++++++++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 3 +-
.../solrj/io/stream/MathExpressionTest.java | 43 +++++++++++++
7 files changed, 265 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a0487b04/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index 02968b2..28c4c66 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -266,6 +266,10 @@ public class Lang {
.withFunctionName("getAmplitude", GetAmplitudeEvaluator.class)
.withFunctionName("getPhase", GetPhaseEvaluator.class)
.withFunctionName("getAngularFrequency", GetAngularFrequencyEvaluator.class)
+ .withFunctionName("enclosingDisk", EnclosingDiskEvaluator.class)
+ .withFunctionName("getCenter", GetCenterEvaluator.class)
+ .withFunctionName("getRadius", GetRadiusEvaluator.class)
+ .withFunctionName("getSupportPoints", GetSupportPointsEvaluator.class)
// Boolean Stream Evaluators
.withFunctionName("and", AndEvaluator.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a0487b04/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnclosingDiskEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnclosingDiskEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnclosingDiskEvaluator.java
new file mode 100644
index 0000000..eb5d33f
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/EnclosingDiskEvaluator.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.commons.math3.geometry.enclosing.EnclosingBall;
+import org.apache.commons.math3.geometry.enclosing.WelzlEncloser;
+import org.apache.commons.math3.geometry.euclidean.twod.DiskGenerator;
+import org.apache.commons.math3.geometry.euclidean.twod.Euclidean2D;
+import org.apache.commons.math3.geometry.euclidean.twod.Vector2D;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class EnclosingDiskEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public EnclosingDiskEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object... objects) throws IOException{
+
+ if(objects[0] instanceof Matrix) {
+ return getEnclosingDisk((Matrix)objects[0]);
+ } else {
+ throw new IOException("The enclosingDisk function operates on a matrix of 2D vectors");
+ }
+ }
+
+ public static EnclosingBall getEnclosingDisk(Matrix matrix) throws IOException {
+ double[][] data = matrix.getData();
+ List<Vector2D> points = new ArrayList(data.length);
+ if(data[0].length == 2) {
+ for(double[] row : data) {
+ points.add(new Vector2D(row[0], row[1]));
+ }
+
+ WelzlEncloser<Euclidean2D, Vector2D> welzlEncloser = new WelzlEncloser(.001, new DiskGenerator());
+ EnclosingBall enclosingBall = welzlEncloser.enclose(points);
+ return enclosingBall;
+ } else {
+ throw new IOException("The enclosingDisk function operates on a matrix of 2D vectors");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a0487b04/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetCenterEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetCenterEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetCenterEvaluator.java
new file mode 100644
index 0000000..45ecea1
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetCenterEvaluator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import java.util.Locale;
+
+import org.apache.commons.math3.geometry.enclosing.EnclosingBall;
+import org.apache.commons.math3.geometry.euclidean.twod.Vector2D;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetCenterEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetCenterEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof EnclosingBall)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting an EnclosingBall",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ EnclosingBall enclosingBall = (EnclosingBall)value;
+ Vector2D vec = (Vector2D)enclosingBall.getCenter();
+ List<Number> center = new ArrayList();
+ center.add(vec.getX());
+ center.add(vec.getY());
+ return center;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a0487b04/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRadiusEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRadiusEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRadiusEvaluator.java
new file mode 100644
index 0000000..1fae9eb
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetRadiusEvaluator.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.commons.math3.geometry.enclosing.EnclosingBall;
+
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetRadiusEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetRadiusEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof EnclosingBall)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting an EnclosingBall",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ EnclosingBall enclosingBall = (EnclosingBall)value;
+ return enclosingBall.getRadius();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a0487b04/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetSupportPointsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetSupportPointsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetSupportPointsEvaluator.java
new file mode 100644
index 0000000..2248490
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/GetSupportPointsEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Locale;
+
+import org.apache.commons.math3.geometry.enclosing.EnclosingBall;
+
+import org.apache.commons.math3.geometry.Point;
+import org.apache.commons.math3.geometry.euclidean.twod.Vector2D;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class GetSupportPointsEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public GetSupportPointsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(!(value instanceof EnclosingBall)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting an EnclosingBall",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ } else {
+ EnclosingBall enclosingBall = (EnclosingBall)value;
+ Point[] points = enclosingBall.getSupport();
+ double[][] data = new double[points.length][2];
+ int i=0;
+ for(Point point : points) {
+ Vector2D eu = (Vector2D)point;
+ data[i][0] = eu.getX();
+ data[i][1] = eu.getY();
+ ++i;
+ }
+
+ return new Matrix(data);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a0487b04/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 169fead..3ae5547 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -72,7 +72,8 @@ public class TestLang extends LuceneTestCase {
"earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue", "knnRegress", "gaussfit",
"outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "latlonVectors",
"convexHull", "getVertices", "getBaryCenter", "getArea", "getBoundarySize","oscillate",
- "getAmplitude", "getPhase", "getAngularFrequency"};
+ "getAmplitude", "getPhase", "getAngularFrequency", "enclosingDisk", "getCenter", "getRadius",
+ "getSupportPoints"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a0487b04/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 73aa0c9..deb4522 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -472,6 +472,49 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
@Test
+ public void testEnclosingDisk() throws Exception {
+ String expr = "let(echo=true," +
+ " x=array(96.42894739701268, 99.11076410926444, 95.71563821370013,101.4356840561301, 96.17912865782684, 113.430677406492, 109.5927785287056, 87.26561260238425, 103.3122002816537, 100.4959815617706, 92.78972440872515, 92.98815024042877, 89.1448359089767, 104.9410622701036, 106.5546761317927, 102.0132643274808, 119.6726096270366, 97.61388415294184, 106.7928221374049, 94.31369945729962, 87.37098859879977, 82.8015657665458, 88.84342877874248, 94.58797342988339, 92.38720473619748)," +
+ " y=array(97.43395922838836, 109.5441846957560, 78.82698890096127, 96.67181538737611,95.52423701473863, 85.3391529394878, 87.01956497912255, 111.5289690656729,86.41034184809114, 84.11696923489203, 109.3874354244069, 102.3391063812790,109.0604436531823,102.7957014900897,114.4376483055848,107.4387578165579,106.2490201384653,103.4490197583837,93.8201540211101,101.6060721649409, 115.3512636715722,119.1046170610335,99.74910277836263,104.2116724112481, 86.02222520549304)," +
+ " c=transpose(matrix(x, y))," +
+ " d=enclosingDisk(c)," +
+ " e=getCenter(d)," +
+ " f=getRadius(d)," +
+ " g=getSupportPoints(d))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+
+ List<Number> center = (List<Number>)tuples.get(0).get("e");
+ assertEquals(center.get(0).doubleValue(), 97.40659699625388, 0.0);
+ assertEquals(center.get(1).doubleValue(), 101.57826559647323, 0.0);
+
+ double radius =tuples.get(0).getDouble("f");
+ assertEquals(radius, 22.814029299535, 0.0);
+
+ List<List<Number>> supportPoints = (List<List<Number>>)tuples.get(0).get("g");
+ List<Number> support1 = supportPoints.get(0);
+ assertEquals(support1.get(0).doubleValue(), 95.71563821370013, 0.0);
+ assertEquals(support1.get(1).doubleValue(), 78.82698890096127, 0.0);
+
+ List<Number> support2 = supportPoints.get(1);
+ assertEquals(support2.get(0).doubleValue(), 82.8015657665458, 0.0);
+ assertEquals(support2.get(1).doubleValue(), 119.1046170610335, 0.0);
+
+ List<Number> support3 = supportPoints.get(2);
+ assertEquals(support3.get(0).doubleValue(), 113.430677406492, 0.0);
+ assertEquals(support3.get(1).doubleValue(), 85.3391529394878, 0.0);
+ }
+
+ @Test
public void testCumulativeProbability() throws Exception {
String expr = "cumulativeProbability(normalDistribution(500, 40), 500)";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
[25/30] lucene-solr:jira/http2: LUCENE-8520: Fix test by running
query so it count total hits
Posted by da...@apache.org.
LUCENE-8520: Fix test by running query so it count total hits
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/98b057c9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/98b057c9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/98b057c9
Branch: refs/heads/jira/http2
Commit: 98b057c93a79e11f60cc6e1feeb16743030a576b
Parents: ace0db7
Author: iverase <iv...@apache.org>
Authored: Fri Oct 5 08:59:34 2018 +0200
Committer: iverase <iv...@apache.org>
Committed: Fri Oct 5 08:59:34 2018 +0200
----------------------------------------------------------------------
.../test/org/apache/lucene/index/TestIndexWriterMaxDocs.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98b057c9/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java
index 5d41459..29f885a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMaxDocs.java
@@ -29,6 +29,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.MockDirectoryWrapper;
@@ -67,7 +68,9 @@ public class TestIndexWriterMaxDocs extends LuceneTestCase {
assertEquals(IndexWriter.MAX_DOCS, ir.maxDoc());
assertEquals(IndexWriter.MAX_DOCS, ir.numDocs());
IndexSearcher searcher = new IndexSearcher(ir);
- TopDocs hits = searcher.search(new TermQuery(new Term("field", "text")), 10);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(10, Integer.MAX_VALUE);
+ searcher.search(new TermQuery(new Term("field", "text")), collector);
+ TopDocs hits = collector.topDocs();
assertEquals(IndexWriter.MAX_DOCS, hits.totalHits.value);
// Sort by docID reversed:
[11/30] lucene-solr:jira/http2: SOLR-12822: /autoscaling/suggestions
to include suggestion to add-replica for lost replicas
Posted by da...@apache.org.
SOLR-12822: /autoscaling/suggestions to include suggestion to add-replica for lost replicas
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d8e40796
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d8e40796
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d8e40796
Branch: refs/heads/jira/http2
Commit: d8e40796e224987f1db696a57569b712aaf1e503
Parents: abace29
Author: Noble Paul <no...@apache.org>
Authored: Wed Oct 3 23:37:58 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Wed Oct 3 23:37:58 2018 +1000
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../solrj/cloud/autoscaling/PolicyHelper.java | 53 ++++++--
.../solrj/cloud/autoscaling/ReplicaCount.java | 13 +-
.../solrj/cloud/autoscaling/Suggester.java | 6 +-
.../solrj/cloud/autoscaling/Suggestion.java | 7 +-
.../apache/solr/common/cloud/DocCollection.java | 9 ++
.../java/org/apache/solr/common/util/Utils.java | 21 +++-
.../solr/autoscaling/testAddMissingReplica.json | 123 +++++++++++++++++++
.../solrj/cloud/autoscaling/TestPolicy.java | 5 +
.../solrj/cloud/autoscaling/TestPolicy2.java | 34 +++++
10 files changed, 259 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index af95736..36a63d2 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -115,6 +115,8 @@ New Features
* SOLR-11522: /autoscaling/suggestions now include rebalance options as well even if there are no violations (noble)
+* SOLR-12822: /autoscaling/suggestions to include suggestion to add-replica for lost replicas (noble)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 123d144..0f4af70 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -226,7 +226,7 @@ public class PolicyHelper {
public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
SolrCloudManager cloudManager) {
- return getSuggestions(autoScalingConf, cloudManager, 20);
+ return getSuggestions(autoScalingConf, cloudManager, 50);
}
public static List<Suggester.SuggestionInfo> getSuggestions(AutoScalingConfig autoScalingConf,
@@ -244,13 +244,52 @@ public class PolicyHelper {
tagType.getSuggestions(ctx.setViolation(violation));
ctx.violation = null;
}
- if (ctx.getSuggestions().size() < max) {
- suggestOptimizations(ctx);
+
+ if (ctx.needMore()) {
+ try {
+ addMissingReplicas(cloudManager, ctx);
+ } catch (IOException e) {
+ log.error("Unable to fetch cluster state", e);
+ }
+ }
+
+ if (ctx.needMore()) {
+ suggestOptimizations(ctx, Math.min(ctx.max - ctx.getSuggestions().size(), 10));
}
return ctx.getSuggestions();
}
- private static void suggestOptimizations(Suggestion.Ctx ctx) {
+ private static void addMissingReplicas(SolrCloudManager cloudManager, Suggestion.Ctx ctx) throws IOException {
+ cloudManager.getClusterStateProvider().getClusterState().forEachCollection(coll -> coll.forEach(slice -> {
+ ReplicaCount replicaCount = new ReplicaCount();
+ slice.forEach(replica -> {
+ if (replica.getState() == Replica.State.ACTIVE || replica.getState() == Replica.State.RECOVERING) {
+ replicaCount.increment(replica.getType());
+ }
+ });
+ addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.NRT, ctx);
+ addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.PULL, ctx);
+ addMissingReplicas(replicaCount, coll, slice.getName(), Replica.Type.TLOG, ctx);
+ }
+ ));
+ }
+
+ private static void addMissingReplicas(ReplicaCount count, DocCollection coll, String shard, Replica.Type type, Suggestion.Ctx ctx) {
+ int delta = count.delta(coll.getExpectedReplicaCount(type, 0), type);
+ for (; ; ) {
+ if (delta >= 0) break;
+ SolrRequest suggestion = ctx.addSuggestion(
+ ctx.session.getSuggester(ADDREPLICA)
+ .hint(Hint.REPLICATYPE, type)
+ .hint(Hint.COLL_SHARD, new Pair(coll.getName(), shard)), "repair");
+ if (suggestion == null) return;
+ delta++;
+ }
+ }
+
+
+ private static void suggestOptimizations(Suggestion.Ctx ctx, int count) {
+ int maxTotalSuggestions = ctx.getSuggestions().size() + count;
List<Row> matrix = ctx.session.matrix;
if (matrix.isEmpty()) return;
for (int i = 0; i < matrix.size(); i++) {
@@ -261,13 +300,13 @@ public class PolicyHelper {
e.setValue(FreeDiskVariable.getSortedShards(Collections.singletonList(row), e.getValue(), e.getKey()));
}
for (Map.Entry<String, Collection<String>> e : collVsShards.entrySet()) {
- if (!ctx.needMore()) break;
+ if (ctx.getSuggestions().size() >= maxTotalSuggestions) break;
for (String shard : e.getValue()) {
- if (!ctx.needMore()) break;
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
.hint(Hint.COLL_SHARD, new Pair<>(e.getKey(), shard))
.hint(Hint.SRC_NODE, row.node);
- ctx.addSuggestion(suggester);
+ ctx.addSuggestion(suggester, "improvement");
+ if (ctx.getSuggestions().size() >= maxTotalSuggestions) break;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
index 87fcf5a..e168ff9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
@@ -70,7 +70,11 @@ class ReplicaCount implements MapWriter {
}
void increment(ReplicaInfo info) {
- switch (info.getType()) {
+ increment(info.getType());
+ }
+
+ public void increment(Replica.Type type) {
+ switch (type) {
case NRT:
nrt++;
break;
@@ -97,4 +101,11 @@ class ReplicaCount implements MapWriter {
public void reset() {
nrt = tlog = pull = 0;
}
+
+ public int delta(int expectedReplicaCount, Replica.Type type) {
+ if (type == Replica.Type.NRT) return (int) (nrt - expectedReplicaCount);
+ if (type == Replica.Type.PULL) return (int) (pull - expectedReplicaCount);
+ if (type == Replica.Type.TLOG) return (int) (tlog - expectedReplicaCount);
+ throw new RuntimeException("NO type");
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index 9f42b9f..0071978 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -189,12 +189,14 @@ public abstract class Suggester implements MapWriter {
}
public static class SuggestionInfo implements MapWriter {
+ String type;
Violation violation;
SolrRequest operation;
- public SuggestionInfo(Violation violation, SolrRequest op) {
+ public SuggestionInfo(Violation violation, SolrRequest op, String type) {
this.violation = violation;
this.operation = op;
+ this.type = type;
}
public SolrRequest getOperation() {
@@ -207,7 +209,7 @@ public abstract class Suggester implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put("type", violation == null ? "improvement" : "violation");
+ ew.put("type", type);
if(violation!= null) ew.put("violation",
new ConditionalMapWriter(violation,
(k, v) -> !"violatingReplicas".equals(k)));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
index 8f120e2..45d4582 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggestion.java
@@ -36,13 +36,16 @@ public class Suggestion {
public Policy.Session session;
public Violation violation;
private List<Suggester.SuggestionInfo> suggestions = new ArrayList<>();
-
SolrRequest addSuggestion(Suggester suggester) {
+ return addSuggestion(suggester, "violation");
+ }
+
+ SolrRequest addSuggestion(Suggester suggester, String type) {
SolrRequest op = suggester.getSuggestion();
if (op != null) {
session = suggester.getSession();
suggestions.add(new Suggester.SuggestionInfo(violation,
- ((V2RequestSupport) op.setUseV2(true)).getV2Request()));
+ ((V2RequestSupport) op.setUseV2(true)).getV2Request(), type));
}
return op;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
index ab250a6..adf0211 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
@@ -406,4 +406,13 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice> {
public String getPolicyName() {
return policy;
}
+
+ public int getExpectedReplicaCount(Replica.Type type, int def) {
+ Integer result = null;
+ if (type == Replica.Type.NRT) result = numNrtReplicas;
+ if (type == Replica.Type.PULL) result = numPullReplicas;
+ if (type == Replica.Type.TLOG) result = numTlogReplicas;
+ return result == null ? def : result;
+
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index c8274f3..ac3f120 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -45,6 +45,7 @@ import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
+import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -265,7 +266,23 @@ public class Utils {
public static Object fromJSON(InputStream is){
try {
- return new ObjectBuilder(getJSONParser((new InputStreamReader(is, StandardCharsets.UTF_8)))).getVal();
+ return STANDARDOBJBUILDER.apply(getJSONParser((new InputStreamReader(is, StandardCharsets.UTF_8)))).getVal();
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error", e);
+ }
+ }
+
+ public static final Function<JSONParser, ObjectBuilder> STANDARDOBJBUILDER = jsonParser -> {
+ try {
+ return new ObjectBuilder(jsonParser);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ };
+
+ public static Object fromJSON(InputStream is, Function<JSONParser, ObjectBuilder> objBuilderProvider) {
+ try {
+ return objBuilderProvider.apply(getJSONParser((new InputStreamReader(is, StandardCharsets.UTF_8)))).getVal();
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error", e);
}
@@ -293,7 +310,7 @@ public class Utils {
public static Object fromJSONString(String json) {
try {
- return new ObjectBuilder(getJSONParser(new StringReader(json))).getVal();
+ return STANDARDOBJBUILDER.apply(getJSONParser(new StringReader(json))).getVal();
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Parse error", e);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddMissingReplica.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddMissingReplica.json b/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddMissingReplica.json
new file mode 100644
index 0000000..6046945
--- /dev/null
+++ b/solr/solrj/src/test-files/solrj/solr/autoscaling/testAddMissingReplica.json
@@ -0,0 +1,123 @@
+{
+ "responseHeader":{
+ "status":0,
+ "QTime":23},
+ "diagnostics":{
+ "sortedNodes":[{
+ "node":"10.0.0.80:7575_solr",
+ "isLive":true,
+ "cores":1.0,
+ "freedisk":673.2483978271484,
+ "totaldisk":1037.938980102539,
+ "replicas":{"gettingstarted":{"shard1":[{
+ "core_node4":{
+ "core":"gettingstarted_shard1_replica_n1",
+ "shard":"shard1",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.80:7575_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.80:7575/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]}}}
+ ,{
+ "node":"10.0.0.80:8983_solr",
+ "isLive":true,
+ "cores":1.0,
+ "freedisk":673.2483940124512,
+ "totaldisk":1037.938980102539,
+ "replicas":{"gettingstarted":{"shard2":[{
+ "core_node8":{
+ "core":"gettingstarted_shard2_replica_n5",
+ "shard":"shard2",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.80:8983_solr",
+ "type":"NRT",
+ "leader":"true",
+ "base_url":"http://10.0.0.80:8983/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]}}}
+ ,{
+ "node":"10.0.0.80:8984_solr",
+ "isLive":true,
+ "cores":1.0,
+ "freedisk":673.2483901977539,
+ "totaldisk":1037.938980102539,
+ "replicas":{"gettingstarted":{"shard1":[{
+ "core_node6":{
+ "core":"gettingstarted_shard1_replica_n2",
+ "shard":"shard1",
+ "collection":"gettingstarted",
+ "node_name":"10.0.0.80:8984_solr",
+ "type":"NRT",
+ "base_url":"http://10.0.0.80:8984/solr",
+ "state":"active",
+ "force_set_state":"false",
+ "INDEX.sizeInGB":6.426125764846802E-8}}]}}}],
+ "liveNodes":["10.0.0.80:7575_solr",
+ "10.0.0.80:8983_solr",
+ "10.0.0.80:8984_solr"],
+ "violations":[],
+ "config":{
+ "cluster-preferences":[{
+ "minimize":"cores",
+ "precision":1}
+ ,{
+ "maximize":"freedisk"}]}},
+
+ "cluster":{
+ "collections":{
+ "gettingstarted":{
+ "pullReplicas":"0",
+ "replicationFactor":"2",
+ "shards":{
+ "shard1":{
+ "range":"80000000-ffffffff",
+ "state":"active",
+ "replicas":{
+ "core_node4":{
+ "core":"gettingstarted_shard1_replica_n1",
+ "base_url":"http://10.0.0.80:7575/solr",
+ "node_name":"10.0.0.80:7575_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "leader":"true"},
+ "core_node6":{
+ "core":"gettingstarted_shard1_replica_n2",
+ "base_url":"http://10.0.0.80:8984/solr",
+ "node_name":"10.0.0.80:8984_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false"}}},
+ "shard2":{
+ "range":"0-7fffffff",
+ "state":"active",
+ "replicas":{
+ "core_node7":{
+ "core":"gettingstarted_shard2_replica_n3",
+ "base_url":"http://10.0.0.80:7574/solr",
+ "node_name":"10.0.0.80:7574_solr",
+ "state":"down",
+ "type":"NRT",
+ "force_set_state":"false"},
+ "core_node8":{
+ "core":"gettingstarted_shard2_replica_n5",
+ "base_url":"http://10.0.0.80:8983/solr",
+ "node_name":"10.0.0.80:8983_solr",
+ "state":"active",
+ "type":"NRT",
+ "force_set_state":"false",
+ "leader":"true"}}}},
+ "router":{"name":"compositeId"},
+ "maxShardsPerNode":"-1",
+ "autoAddReplicas":"false",
+ "nrtReplicas":"2",
+ "tlogReplicas":"0",
+ "znodeVersion":12,
+ "configName":"gettingstarted"}},
+ "live_nodes":["10.0.0.80:8983_solr",
+ "10.0.0.80:7575_solr",
+ "10.0.0.80:8984_solr"]}}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 23184a0..ec3c56c 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -1345,6 +1345,11 @@ public class TestPolicy extends SolrTestCaseJ4 {
public ClusterStateProvider getClusterStateProvider() {
return new DelegatingClusterStateProvider(null) {
@Override
+ public ClusterState getClusterState() throws IOException {
+ return ClusterState.load(0,new HashMap<>(), getLiveNodes(),"/clusterstate.json");
+ }
+
+ @Override
public Set<String> getLiveNodes() {
return new HashSet<>((Collection<String>) m.get("liveNodes"));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d8e40796/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index 07ce391..391b210 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
@@ -338,6 +339,7 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
};
return new DelegatingCloudManager(null) {
+ ClusterState clusterState = null;
@Override
public NodeStateProvider getNodeStateProvider() {
return nodeStateProvider;
@@ -345,7 +347,24 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
@Override
public ClusterStateProvider getClusterStateProvider() {
+ if (clusterState == null) {
+ Map map = (Map) Utils.getObjectByPath (m, false, "cluster/collections");
+ if (map == null) map = new HashMap<>();
+ clusterState = ClusterState.load(0, map, liveNodes, "/clusterstate.json");
+ }
+
return new DelegatingClusterStateProvider(null) {
+
+ @Override
+ public ClusterState getClusterState() throws IOException {
+ return clusterState;
+ }
+
+ @Override
+ public ClusterState.CollectionRef getState(String collection) {
+ return clusterState.getCollectionRef(collection);
+ }
+
@Override
public Set<String> getLiveNodes() {
return liveNodes;
@@ -414,6 +433,21 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
}
+ public void testAddMissingReplica() throws IOException {
+ Map<String, Object> m = (Map<String, Object>) loadFromResource("testAddMissingReplica.json");
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+ AutoScalingConfig autoScalingConfig = new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
+
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(autoScalingConfig, cloudManagerFromDiagnostics);
+
+ assertEquals(1, suggestions.size());
+ assertEquals("repair", suggestions.get(0)._get("type",null));
+ assertEquals("add-replica", suggestions.get(0)._get("operation/command[0]/key",null));
+ assertEquals("shard2", suggestions.get(0)._get("operation/command/add-replica/shard",null));
+ assertEquals("NRT", suggestions.get(0)._get("operation/command/add-replica/type",null));
+
+ }
+
public static Object loadFromResource(String file) throws IOException {
try (InputStream is = TestPolicy2.class.getResourceAsStream("/solrj/solr/autoscaling/" + file)) {
return Utils.fromJSON(is);
[26/30] lucene-solr:jira/http2: SOLR-12836: ZkController creates a
cloud solr client with no connection or read timeouts.
Posted by da...@apache.org.
SOLR-12836: ZkController creates a cloud solr client with no connection or read timeouts.
This changes ZkController to use the http client created by the update shard handler instead of creating a custom one.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eb47099e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eb47099e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eb47099e
Branch: refs/heads/jira/http2
Commit: eb47099ee22669926e6fdd4fc39cce7d043e3e97
Parents: 98b057c
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Oct 5 15:18:11 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Oct 5 15:18:11 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
solr/core/src/java/org/apache/solr/cloud/ZkController.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb47099e/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 93faa36..41f8735 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -164,6 +164,9 @@ Bug Fixes
* SOLR-12814: Metrics history causing "HttpParser URI is too large >8192" when many collections (janhoy)
+* SOLR-12836: ZkController creates a cloud solr client with no connection or read timeouts. Now the http client
+ created by the update shard handler is used instead. (shalin)
+
Improvements
----------------------
* SOLR-12767: Solr now always includes in the response of update requests the achieved replication factor
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb47099e/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 8318277..d5de5dd 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -671,7 +671,8 @@ public class ZkController {
if (cloudManager != null) {
return cloudManager;
}
- cloudSolrClient = new CloudSolrClient.Builder(Collections.singletonList(zkServerAddress), Optional.empty()).build();
+ cloudSolrClient = new CloudSolrClient.Builder(Collections.singletonList(zkServerAddress), Optional.empty())
+ .withHttpClient(cc.getUpdateShardHandler().getDefaultHttpClient()).build();
cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), cloudSolrClient);
}
return cloudManager;
[29/30] lucene-solr:jira/http2: Do not set Conscrypt provider if not
needed
Posted by da...@apache.org.
Do not set Conscrypt provider if not needed
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/75e91255
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/75e91255
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/75e91255
Branch: refs/heads/jira/http2
Commit: 75e912559d8fb5f651c207685462b7ed84d440c7
Parents: bb0b9a1
Author: Cao Manh Dat <da...@apache.org>
Authored: Mon Oct 8 09:05:09 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Mon Oct 8 09:05:09 2018 +0700
----------------------------------------------------------------------
.../test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/75e91255/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 9a6bb27..fa5ff65 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -285,7 +285,10 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
ignoreException("ignore_exception");
newRandomConfig();
- Security.insertProviderAt(new OpenSSLProvider(), 1);
+ // Set Conscrypt as default OpenSSLProvider for all clients
+ if (Security.getProvider("Conscrypt") == null) {
+ Security.insertProviderAt(new OpenSSLProvider(), 1);
+ }
sslConfig = buildSSLConfig();
// based on randomized SSL config, set SchemaRegistryProvider appropriately
[05/30] lucene-solr:jira/http2: SOLR-12709: Several fixes to the
simulator and its .system collection auto-creation.
Posted by da...@apache.org.
SOLR-12709: Several fixes to the simulator and its .system collection auto-creation.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2201b652
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2201b652
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2201b652
Branch: refs/heads/jira/http2
Commit: 2201b6526635597aba170bc8f39a4bb438d16b93
Parents: 4ca885a
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Oct 2 17:35:45 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Oct 2 17:35:45 2018 +0200
----------------------------------------------------------------------
.../org/apache/solr/cloud/CloudTestUtils.java | 4 +-
.../SearchRateTriggerIntegrationTest.java | 12 +-
.../cloud/autoscaling/sim/SimCloudManager.java | 3 +-
.../sim/SimClusterStateProvider.java | 112 +++++++++++--------
.../autoscaling/sim/SimSolrCloudTestCase.java | 6 +-
.../autoscaling/sim/TestSimLargeCluster.java | 2 +-
.../sim/TestSimTriggerIntegration.java | 14 ++-
7 files changed, 90 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2201b652/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java b/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
index eb50b96..e93cd58 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
@@ -97,14 +97,14 @@ public class CloudTestUtils {
// due to the way we manage collections in SimClusterStateProvider a null here
// can mean that a collection is still being created but has no replicas
if (coll == null) { // does not yet exist?
- timeout.sleep(50);
+ timeout.sleep(100);
continue;
}
if (predicate.matches(state.getLiveNodes(), coll)) {
log.trace("-- predicate matched with state {}", state);
return timeout.timeElapsed(TimeUnit.MILLISECONDS);
}
- timeout.sleep(50);
+ timeout.sleep(100);
if (timeout.timeLeft(TimeUnit.MILLISECONDS) < timeWarn) {
log.trace("-- still not matching predicate: {}", state);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2201b652/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
index 106ec17..6febdd3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
@@ -678,8 +678,16 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
if (m.get("success") != null) {
replicas.incrementAndGet();
} else if (m.get("status") != null) {
- NamedList<Object> status = (NamedList<Object>)m.get("status");
- if ("completed".equals(status.get("state"))) {
+ Object status = m.get("status");
+ String state;
+ if (status instanceof Map) {
+ state = (String)((Map)status).get("state");
+ } else if (status instanceof NamedList) {
+ state = (String)((NamedList)status).get("state");
+ } else {
+ throw new IllegalArgumentException("unsupported status format: " + status.getClass().getName() + ", " + status);
+ }
+ if ("completed".equals(state)) {
nodes.incrementAndGet();
} else {
fail("unexpected DELETENODE status: " + m);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2201b652/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
index 53e2c7e..9a6b73f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
@@ -542,6 +542,7 @@ public class SimCloudManager implements SolrCloudManager {
triggerThread.interrupt();
IOUtils.closeQuietly(triggerThread);
if (killNodeId != null) {
+ log.info(" = killing node " + killNodeId);
simRemoveNode(killNodeId, false);
}
objectCache.clear();
@@ -746,7 +747,7 @@ public class SimCloudManager implements SolrCloudManager {
if (!"autoscaling_event".equals(d.getFieldValue("type"))) {
continue;
}
- eventCounts.computeIfAbsent((String)d.getFieldValue("event.source_s"), s -> new TreeMap<>())
+ eventCounts.computeIfAbsent((String)d.getFieldValue("event.source_s"), s -> new ConcurrentHashMap<>())
.computeIfAbsent((String)d.getFieldValue("stage_s"), s -> new AtomicInteger())
.incrementAndGet();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2201b652/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 8b14682..ee39666 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -155,7 +155,6 @@ public class SimClusterStateProvider implements ClusterStateProvider {
private Map<String, Object> lastSavedProperties = null;
private AtomicReference<Map<String, DocCollection>> collectionsStatesRef = new AtomicReference<>();
- private AtomicBoolean saveClusterState = new AtomicBoolean();
private Random bulkUpdateRandom = new Random(0);
@@ -619,14 +618,15 @@ public class SimClusterStateProvider implements ClusterStateProvider {
* Save clusterstate.json to {@link DistribStateManager}.
* @return saved state
*/
- private synchronized ClusterState saveClusterState(ClusterState state) throws IOException {
+ private ClusterState saveClusterState(ClusterState state) throws IOException {
+ ensureNotClosed();
byte[] data = Utils.toJSON(state);
try {
VersionedData oldData = stateManager.getData(ZkStateReader.CLUSTER_STATE);
int version = oldData != null ? oldData.getVersion() : -1;
Assert.assertEquals(clusterStateVersion, version + 1);
stateManager.setData(ZkStateReader.CLUSTER_STATE, data, version);
- log.debug("** saved cluster state version " + version);
+ log.debug("** saved cluster state version " + (version + 1));
clusterStateVersion++;
} catch (Exception e) {
throw new IOException(e);
@@ -754,7 +754,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
synchronized (ri) {
ri.getVariables().put(ZkStateReader.LEADER_PROP, "true");
}
- log.debug("-- elected new leader for {} / {}: {}", collection, s.getName(), ri);
+ log.debug("-- elected new leader for {} / {} (currentVersion={}): {}", collection,
+ s.getName(), clusterStateVersion, ri);
stateChanged.set(true);
}
} else {
@@ -762,7 +763,6 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
if (stateChanged.get() || saveState) {
collectionsStatesRef.set(null);
- saveClusterState.set(true);
}
}
@@ -778,6 +778,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
boolean waitForFinalState = props.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false);
final String collectionName = props.getStr(NAME);
+ log.debug("-- simCreateCollection {}, currentVersion={}", collectionName, clusterStateVersion);
String router = props.getStr("router.name", DocRouter.DEFAULT_NAME);
String policy = props.getStr(Policy.POLICY);
@@ -808,11 +809,46 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
final String withCollectionShard = wcShard;
- ZkWriteCommand cmd = new ClusterStateMutator(cloudManager).createCollection(clusterState, props);
- if (cmd.noop) {
- log.warn("Collection {} already exists. exit", collectionName);
- results.add("success", "no-op");
- return;
+ lock.lockInterruptibly();
+ ZkWriteCommand cmd = ZkWriteCommand.noop();
+ try {
+ cmd = new ClusterStateMutator(cloudManager).createCollection(clusterState, props);
+ if (cmd.noop) {
+ log.warn("Collection {} already exists. exit", collectionName);
+ log.debug("-- collection: {}, clusterState: {}", collectionName, clusterState);
+ results.add("success", "no-op");
+ return;
+ }
+ // add collection props
+ DocCollection coll = cmd.collection;
+ collProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()).putAll(coll.getProperties());
+ colShardReplicaMap.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>());
+ // add slice props
+ coll.getSlices().forEach(s -> {
+ Map<String, Object> sliceProps = sliceProperties.computeIfAbsent(coll.getName(), c -> new ConcurrentHashMap<>())
+ .computeIfAbsent(s.getName(), slice -> new ConcurrentHashMap<>());
+ s.getProperties().forEach((k, v) -> {
+ if (k != null && v != null) {
+ sliceProps.put(k, v);
+ }
+ });
+ colShardReplicaMap.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>())
+ .computeIfAbsent(s.getName(), sh -> new ArrayList<>());
+ });
+
+ // modify the `withCollection` and store this new collection's name with it
+ if (withCollection != null) {
+ ZkNodeProps message = new ZkNodeProps(
+ Overseer.QUEUE_OPERATION, MODIFYCOLLECTION.toString(),
+ ZkStateReader.COLLECTION_PROP, withCollection,
+ CollectionAdminParams.COLOCATED_WITH, collectionName);
+ cmd = new CollectionMutator(cloudManager).modifyCollection(clusterState,message);
+ }
+ // force recreation of collection states
+ collectionsStatesRef.set(null);
+
+ } finally {
+ lock.unlock();
}
opDelays.computeIfAbsent(collectionName, c -> new HashMap<>()).putAll(defaultOpDelays);
@@ -883,28 +919,6 @@ public class SimClusterStateProvider implements ClusterStateProvider {
throw new RuntimeException(e);
}
});
- // add collection props
- DocCollection coll = cmd.collection;
- collProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>()).putAll(coll.getProperties());
- // add slice props
- coll.getSlices().forEach(s -> {
- Map<String, Object> sliceProps = sliceProperties.computeIfAbsent(coll.getName(), c -> new ConcurrentHashMap<>())
- .computeIfAbsent(s.getName(), slice -> new ConcurrentHashMap<>());
- s.getProperties().forEach((k, v) -> {
- if (k != null && v != null) {
- sliceProps.put(k, v);
- }
- });
- });
-
- // modify the `withCollection` and store this new collection's name with it
- if (withCollection != null) {
- ZkNodeProps message = new ZkNodeProps(
- Overseer.QUEUE_OPERATION, MODIFYCOLLECTION.toString(),
- ZkStateReader.COLLECTION_PROP, withCollection,
- CollectionAdminParams.COLOCATED_WITH, collectionName);
- cmd = new CollectionMutator(cloudManager).modifyCollection(clusterState,message);
- }
// force recreation of collection states
collectionsStatesRef.set(null);
@@ -918,6 +932,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
results.add("success", "");
+ log.debug("-- finished createCollection {}, currentVersion={}", collectionName, clusterStateVersion);
}
/**
@@ -963,7 +978,6 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
});
collectionsStatesRef.set(null);
- saveClusterState.set(true);
results.add("success", "");
} catch (Exception e) {
log.warn("Exception", e);
@@ -977,18 +991,20 @@ public class SimClusterStateProvider implements ClusterStateProvider {
*/
public void simDeleteAllCollections() throws Exception {
lock.lockInterruptibly();
+ collectionsStatesRef.set(null);
try {
- nodeReplicaMap.clear();
- colShardReplicaMap.clear();
collProperties.clear();
sliceProperties.clear();
leaderThrottles.clear();
+ nodeReplicaMap.clear();
+ colShardReplicaMap.clear();
cloudManager.getSimNodeStateProvider().simGetAllNodeValues().forEach((n, values) -> {
values.put(ImplicitSnitch.CORES, 0);
- values.put(ImplicitSnitch.DISK, 1000);
+ values.put(ImplicitSnitch.DISK, SimCloudManager.DEFAULT_FREE_DISK);
+ values.put(Variable.Type.TOTALDISK.tagName, SimCloudManager.DEFAULT_TOTAL_DISK);
+ values.put(ImplicitSnitch.SYSLOADAVG, 1.0);
+ values.put(ImplicitSnitch.HEAPUSAGE, 123450000);
});
- collectionsStatesRef.set(null);
- saveClusterState.set(true);
} finally {
lock.unlock();
}
@@ -1326,20 +1342,21 @@ public class SimClusterStateProvider implements ClusterStateProvider {
}
}
- public void createSystemCollection() throws IOException {
+ public synchronized void createSystemCollection() throws IOException {
try {
if (colShardReplicaMap.containsKey(CollectionAdminParams.SYSTEM_COLL)) {
return;
}
+ String repFactor = String.valueOf(Math.min(3, liveNodes.size()));
ZkNodeProps props = new ZkNodeProps(
NAME, CollectionAdminParams.SYSTEM_COLL,
- REPLICATION_FACTOR, "1",
+ REPLICATION_FACTOR, repFactor,
OverseerCollectionMessageHandler.NUM_SLICES, "1",
CommonAdminParams.WAIT_FOR_FINAL_STATE, "true"
);
simCreateCollection(props, new NamedList());
CloudTestUtils.waitForState(cloudManager, CollectionAdminParams.SYSTEM_COLL, 20, TimeUnit.SECONDS,
- CloudTestUtils.clusterShape(1, 1, false, true));
+ CloudTestUtils.clusterShape(1, Integer.parseInt(repFactor), false, true));
} catch (Exception e) {
throw new IOException(e);
}
@@ -1372,6 +1389,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
if (!colShardReplicaMap.containsKey(collection)) {
if (CollectionAdminParams.SYSTEM_COLL.equals(collection)) {
// auto-create
+ log.trace("-- auto-create .system when req=" + req);
createSystemCollection();
} else {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection '" + collection + "' doesn't exist");
@@ -2041,25 +2059,22 @@ public class SimClusterStateProvider implements ClusterStateProvider {
@Override
public ClusterState getClusterState() throws IOException {
+ ensureNotClosed();
Map<String, DocCollection> states = getCollectionStates();
ClusterState state = new ClusterState(clusterStateVersion, liveNodes.get(), states);
- if (saveClusterState.getAndSet(false)) {
- saveClusterState(state);
- }
return state;
}
// this method uses a simple cache in collectionsStatesRef. Operations that modify
// cluster state should always reset this cache so that the changes become visible
- private Map<String, DocCollection> getCollectionStates() {
+ private Map<String, DocCollection> getCollectionStates() throws IOException {
Map<String, DocCollection> collectionStates = collectionsStatesRef.get();
if (collectionStates != null) {
return collectionStates;
}
lock.lock();
collectionsStatesRef.set(null);
- saveClusterState.set(true);
- log.debug("** creating new collection states");
+ log.debug("** creating new collection states, currentVersion={}", clusterStateVersion);
try {
Map<String, Map<String, Map<String, Replica>>> collMap = new HashMap<>();
nodeReplicaMap.forEach((n, replicas) -> {
@@ -2101,9 +2116,10 @@ public class SimClusterStateProvider implements ClusterStateProvider {
Map<String, Object> collProps = collProperties.computeIfAbsent(coll, c -> new ConcurrentHashMap<>());
Map<String, Object> routerProp = (Map<String, Object>) collProps.getOrDefault(DocCollection.DOC_ROUTER, Collections.singletonMap("name", DocRouter.DEFAULT_NAME));
DocRouter router = DocRouter.getDocRouter((String)routerProp.getOrDefault("name", DocRouter.DEFAULT_NAME));
- DocCollection dc = new DocCollection(coll, slices, collProps, router, clusterStateVersion + 1, ZkStateReader.CLUSTER_STATE);
+ DocCollection dc = new DocCollection(coll, slices, collProps, router, clusterStateVersion, ZkStateReader.CLUSTER_STATE);
res.put(coll, dc);
});
+ saveClusterState(new ClusterState(clusterStateVersion, liveNodes.get(), res));
collectionsStatesRef.set(res);
return res;
} finally {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2201b652/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
index 270e7e7..3d41ea4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimSolrCloudTestCase.java
@@ -89,8 +89,6 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 {
// clear any persisted configuration
cluster.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
cluster.getDistribStateManager().setData(ZkStateReader.ROLES, Utils.toJSON(new HashMap<>()), -1);
- cluster.getSimClusterStateProvider().simDeleteAllCollections();
- cluster.simClearSystemCollection();
cluster.getSimNodeStateProvider().simRemoveDeadNodes();
cluster.getSimClusterStateProvider().simRemoveDeadNodes();
// restore the expected number of nodes
@@ -110,7 +108,9 @@ public class SimSolrCloudTestCase extends SolrTestCaseJ4 {
removeChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
cluster.getSimClusterStateProvider().simResetLeaderThrottles();
cluster.simRestartOverseer(null);
- cluster.getTimeSource().sleep(5000);
+ cluster.getSimClusterStateProvider().simDeleteAllCollections();
+ cluster.simClearSystemCollection();
+ cluster.getTimeSource().sleep(10000);
cluster.simResetOpCounts();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2201b652/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
index 4a0c362..42ddcc1 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
@@ -136,7 +136,7 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
if (!cluster.getSimClusterStateProvider().simListCollections().contains(CollectionAdminParams.SYSTEM_COLL)) {
cluster.getSimClusterStateProvider().createSystemCollection();
CloudTestUtils.waitForState(cluster, CollectionAdminParams.SYSTEM_COLL, 120, TimeUnit.SECONDS,
- CloudTestUtils.clusterShape(1, 1, false, true));
+ CloudTestUtils.clusterShape(1, 3, false, true));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2201b652/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index 50769e8..36beeae 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -76,7 +76,7 @@ import static org.apache.solr.cloud.autoscaling.ScheduledTriggers.DEFAULT_SCHEDU
/**
* An end-to-end integration test for triggers
*/
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;")
public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -152,12 +152,11 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
// lets start a node
cluster.simAddNode();
}
+ cluster.getTimeSource().sleep(10000);
// do this in advance if missing
- if (!cluster.getSimClusterStateProvider().simListCollections().contains(CollectionAdminParams.SYSTEM_COLL)) {
- cluster.getSimClusterStateProvider().createSystemCollection();
- CloudTestUtils.waitForState(cluster, CollectionAdminParams.SYSTEM_COLL, 120, TimeUnit.SECONDS,
- CloudTestUtils.clusterShape(1, 1, false, true));
- }
+ cluster.getSimClusterStateProvider().createSystemCollection();
+ CloudTestUtils.waitForState(cluster, CollectionAdminParams.SYSTEM_COLL, 120, TimeUnit.SECONDS,
+ CloudTestUtils.clusterShape(1, 2, false, true));
}
@Test
@@ -661,6 +660,9 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
fail("The TriggerAction should have been created by now");
}
+ // wait for the trigger to run at least once
+ cluster.getTimeSource().sleep(2 * waitForSeconds * 1000);
+
// add node to generate the event
String newNode = cluster.simAddNode();
boolean await = actionStarted.await(60000 / SPEED, TimeUnit.MILLISECONDS);
[10/30] lucene-solr:jira/http2: Fix the CHANGES entry - this is
applicable only to 8.0 codebase.
Posted by da...@apache.org.
Fix the CHANGES entry - this is applicable only to 8.0 codebase.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/abace298
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/abace298
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/abace298
Branch: refs/heads/jira/http2
Commit: abace2987a00e565068a92d288967f6b9e2efd6f
Parents: eba7bed
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Oct 3 14:11:43 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Oct 3 14:11:43 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/abace298/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d9ee5fc..af95736 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -67,6 +67,11 @@ SOLR-12591: Expand the set of recognized date format patterns of schemaless mode
SOLR-12593: The default configSet now includes an "ignored_*" dynamic field. (David Smiley)
+Optimizations
+----------------------
+
+* SOLR-12725: ParseDateFieldUpdateProcessorFactory should reuse ParsePosition. (ab)
+
Other Changes
----------------------
@@ -428,8 +433,6 @@ Optimizations
* SOLR-12766: When retrying internal requests, backoff only once for the full batch of retries (Tomás Fernández Löbbe)
-* SOLR-12725: ParseDateFieldUpdateProcessorFactory should reuse ParsePosition (ab)
-
Other Changes
----------------------
[30/30] lucene-solr:jira/http2: Merge branch 'master' into jira/http2
Posted by da...@apache.org.
Merge branch 'master' into jira/http2
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2d51180b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2d51180b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2d51180b
Branch: refs/heads/jira/http2
Commit: 2d51180bb32081eafc8eca79302356186ea112e0
Parents: 75e9125 367bdf7
Author: Cao Manh Dat <da...@apache.org>
Authored: Mon Oct 8 09:29:58 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Mon Oct 8 09:29:58 2018 +0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 10 +-
.../benchmark/byTask/TestPerfTasksLogic.java | 9 +-
.../codecs/perfield/PerFieldPostingsFormat.java | 14 +-
.../org/apache/lucene/index/MultiFields.java | 68 +---
.../org/apache/lucene/util/QueryBuilder.java | 10 +
.../lucene/index/TestDirectoryReader.java | 11 +-
.../org/apache/lucene/index/TestDocCount.java | 6 +-
.../lucene/index/TestIndexWriterMaxDocs.java | 5 +-
.../index/TestParallelCompositeReader.java | 27 +-
.../apache/lucene/index/TestSegmentReader.java | 4 +-
.../lucene/index/TestStressIndexing2.java | 26 +-
.../org/apache/lucene/index/TestSumDocFreq.java | 7 +-
.../apache/lucene/util/TestQueryBuilder.java | 28 ++
.../memory/TestMemoryIndexAgainstRAMDir.java | 2 +-
.../org/apache/lucene/misc/HighFreqTerms.java | 20 +-
.../index/BaseIndexFileFormatTestCase.java | 22 +-
.../org/apache/lucene/util/LuceneTestCase.java | 67 ++--
solr/CHANGES.txt | 53 +++
.../java/org/apache/solr/cloud/CloudUtil.java | 48 ++-
.../org/apache/solr/cloud/ZkController.java | 3 +-
.../solr/cloud/api/collections/Assign.java | 3 +-
.../api/collections/CreateCollectionCmd.java | 4 +-
.../cloud/autoscaling/IndexSizeTrigger.java | 34 +-
.../solr/handler/admin/CollectionsHandler.java | 10 +-
.../solr/handler/component/ExpandComponent.java | 17 +-
.../solr/index/SlowCompositeReaderWrapper.java | 46 ++-
.../UninvertDocValuesMergePolicyFactory.java | 5 +-
.../org/apache/solr/schema/IndexSchema.java | 53 ++-
.../solr/search/CollapsingQParserPlugin.java | 18 +-
.../java/org/apache/solr/search/Insanity.java | 5 +-
.../apache/solr/search/SolrIndexSearcher.java | 17 +-
.../solr/uninverting/UninvertingReader.java | 103 +++---
.../solr/update/DeleteByQueryWrapper.java | 2 +-
.../apache/solr/update/SolrCmdDistributor.java | 2 +-
.../processor/DistributedUpdateProcessor.java | 106 ++----
.../ParseDateFieldUpdateProcessorFactory.java | 28 +-
.../org/apache/solr/cloud/CloudTestUtils.java | 4 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 141 +++++++-
.../apache/solr/cloud/HttpPartitionTest.java | 103 +-----
.../apache/solr/cloud/MoveReplicaHDFSTest.java | 6 +
.../solr/cloud/ReplicationFactorTest.java | 61 +++-
.../org/apache/solr/cloud/TestCryptoKeys.java | 13 +-
.../solr/cloud/autoscaling/CapturedEvent.java | 2 +-
.../cloud/autoscaling/IndexSizeTriggerTest.java | 141 ++++++++
.../SearchRateTriggerIntegrationTest.java | 19 +-
.../cloud/autoscaling/sim/SimCloudManager.java | 3 +-
.../sim/SimClusterStateProvider.java | 112 +++---
.../autoscaling/sim/SimSolrCloudTestCase.java | 6 +-
.../autoscaling/sim/TestSimLargeCluster.java | 2 +-
.../sim/TestSimTriggerIntegration.java | 25 +-
.../solr/cloud/cdcr/CdcrBidirectionalTest.java | 1 +
.../apache/solr/core/TestDynamicLoading.java | 22 +-
.../apache/solr/core/TestSolrConfigHandler.java | 175 +++++-----
.../apache/solr/handler/TestBlobHandler.java | 45 +--
.../apache/solr/handler/TestConfigReload.java | 19 +-
.../handler/TestSolrConfigHandlerCloud.java | 3 +-
.../TestSolrConfigHandlerConcurrent.java | 44 ++-
.../index/TestSlowCompositeReaderWrapper.java | 29 +-
.../solr/uninverting/TestUninvertingReader.java | 18 +-
.../solr/update/SolrCmdDistributorTest.java | 4 +-
.../test/org/apache/solr/util/TestUtils.java | 73 +++-
solr/solr-ref-guide/src/collections-api.adoc | 41 ++-
.../src/performance-statistics-reference.adoc | 12 +-
.../src/solrcloud-autoscaling-triggers.adoc | 6 +
...olrcloud-recoveries-and-write-tolerance.adoc | 6 +-
.../client/solrj/cloud/autoscaling/Policy.java | 27 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 53 ++-
.../solrj/cloud/autoscaling/Preference.java | 8 +
.../solrj/cloud/autoscaling/ReplicaCount.java | 13 +-
.../solrj/cloud/autoscaling/Suggester.java | 6 +-
.../solrj/cloud/autoscaling/Suggestion.java | 7 +-
.../solrj/impl/ConcurrentUpdateSolrClient.java | 6 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 2 +-
.../org/apache/solr/client/solrj/io/Lang.java | 10 +-
.../solrj/io/eval/EnclosingDiskEvaluator.java | 64 ++++
.../solrj/io/eval/GetAmplitudeEvaluator.java | 42 +++
.../io/eval/GetAngularFrequencyEvaluator.java | 42 +++
.../solrj/io/eval/GetCenterEvaluator.java | 52 +++
.../client/solrj/io/eval/GetPhaseEvaluator.java | 42 +++
.../solrj/io/eval/GetRadiusEvaluator.java | 44 +++
.../io/eval/GetSupportPointsEvaluator.java | 56 +++
.../solrj/io/eval/HarmonicFitEvaluator.java | 11 +-
.../solrj/io/eval/OscillateEvaluator.java | 57 +++
.../client/solrj/request/UpdateRequest.java | 4 +
.../client/solrj/response/SolrResponseBase.java | 12 +-
.../org/apache/solr/common/IteratorWriter.java | 9 +
.../apache/solr/common/LinkedHashMapWriter.java | 60 ++++
.../java/org/apache/solr/common/MapWriter.java | 38 +-
.../org/apache/solr/common/MapWriterMap.java | 53 +++
.../org/apache/solr/common/StringUtils.java | 10 +
.../solr/common/cloud/ClusterProperties.java | 41 ++-
.../apache/solr/common/cloud/DocCollection.java | 9 +
.../apache/solr/common/cloud/ZkStateReader.java | 7 +-
.../common/params/CollectionAdminParams.java | 5 +
.../solr/common/params/CollectionParams.java | 2 +-
.../org/apache/solr/common/util/NamedList.java | 10 +-
.../java/org/apache/solr/common/util/Utils.java | 117 ++++++-
.../src/resources/apispec/cluster.Commands.json | 34 +-
.../solr/autoscaling/testAddMissingReplica.json | 123 +++++++
.../testMoveReplicasInMultipleCollections.json | 88 +++++
.../solrj/cloud/autoscaling/TestPolicy.java | 348 ++++++++++++++-----
.../solrj/cloud/autoscaling/TestPolicy2.java | 36 +-
...rentUpdateSolrClientMultiCollectionTest.java | 94 +++++
.../apache/solr/client/solrj/io/TestLang.java | 6 +-
.../solrj/io/stream/MathExpressionTest.java | 85 ++++-
.../solr/BaseDistributedSearchTestCase.java | 2 +
.../cloud/AbstractFullDistribZkTestBase.java | 1 -
107 files changed, 2807 insertions(+), 958 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d51180b/solr/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d51180b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index c8ad92f,9536f9d..66a251f
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@@ -298,10 -386,10 +298,10 @@@ public class SolrCmdDistributor
// we need to add the data to the rollup tracker.
//
// In the case of a leaderTracker and rollupTracker both being present, then we need to take care when assembling
- // the final response to check both the rollup and leader trackers on the aggregator node.
- public void trackRequestResult(HttpResponse resp, boolean success) {
-
+ // the final response to check both the rollup and leader trackers on the aggrator node.
+ public void trackRequestResult(NamedList resp, boolean success) {
+
- // Returing Integer.MAX_VALUE here means there was no "rf" on the response, therefore we just need to increment
+ // Returning Integer.MAX_VALUE here means there was no "rf" on the response, therefore we just need to increment
// our achieved rf if we are a leader, i.e. have a leaderTracker.
int rfFromResp = getRfFromResponse(resp);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d51180b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index 2cb287e,bab8607..0d1bbeb
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@@ -2148,8 -2140,6 +2124,7 @@@ public class DistributedUpdateProcesso
// Since we only allocate one of these on the leader and, by definition, the leader has been found and is running,
// we have a replication factor of one by default.
private int achievedRf = 1;
- private final int requestedRf;
+ private Set<String> nodes = new HashSet<>();
private final String myShardId;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d51180b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d51180b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d51180b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
[19/30] lucene-solr:jira/http2: LUCENE-8479:
QueryBuilder#analyzeGraphPhrase now throws TooManyClause exception if the
number of expanded path reaches the BooleanQuery#maxClause limit.
Posted by da...@apache.org.
LUCENE-8479: QueryBuilder#analyzeGraphPhrase now throws TooManyClause exception if the number of expanded path reaches the BooleanQuery#maxClause limit.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0f100004
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0f100004
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0f100004
Branch: refs/heads/jira/http2
Commit: 0f100004bcd8ffc15cb5bd9281e1c9a61c17eb86
Parents: 793a677
Author: Jim Ferenczi <ji...@apache.org>
Authored: Thu Oct 4 13:04:53 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Thu Oct 4 13:04:53 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 7 ++++-
.../org/apache/lucene/util/QueryBuilder.java | 10 +++++++
.../apache/lucene/util/TestQueryBuilder.java | 29 ++++++++++++++++++++
3 files changed, 45 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f100004/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3f23c1e..ad83ff9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -181,12 +181,17 @@ Optimizations
======================= Lucene 7.6.0 =======================
-Build
+Build:
* LUCENE-8504: Upgrade forbiddenapis to version 2.6. (Uwe Schindler)
* LUCENE-8493: Stop publishing insecure .sha1 files with releases (janhoy)
+Bug fixes:
+
+* LUCENE-8479: QueryBuilder#analyzeGraphPhrase now throws TooManyClause exception
+ if the number of expanded path reaches the BooleanQuery#maxClause limit. (Jim Ferenczi)
+
======================= Lucene 7.5.1 =======================
Bug Fixes:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f100004/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java b/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
index 697e3bb..37b7e3e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
@@ -552,6 +552,7 @@ public class QueryBuilder {
List<SpanQuery> clauses = new ArrayList<>();
int[] articulationPoints = graph.articulationPoints();
int lastState = 0;
+ int maxClauseCount = BooleanQuery.getMaxClauseCount();
for (int i = 0; i <= articulationPoints.length; i++) {
int start = lastState;
int end = -1;
@@ -567,6 +568,9 @@ public class QueryBuilder {
TokenStream ts = it.next();
SpanQuery q = createSpanQuery(ts, field);
if (q != null) {
+ if (queries.size() >= maxClauseCount) {
+ throw new BooleanQuery.TooManyClauses();
+ }
queries.add(q);
}
}
@@ -581,6 +585,9 @@ public class QueryBuilder {
if (terms.length == 1) {
queryPos = new SpanTermQuery(terms[0]);
} else {
+ if (terms.length >= maxClauseCount) {
+ throw new BooleanQuery.TooManyClauses();
+ }
SpanTermQuery[] orClauses = new SpanTermQuery[terms.length];
for (int idx = 0; idx < terms.length; idx++) {
orClauses[idx] = new SpanTermQuery(terms[idx]);
@@ -591,6 +598,9 @@ public class QueryBuilder {
}
if (queryPos != null) {
+ if (clauses.size() >= maxClauseCount) {
+ throw new BooleanQuery.TooManyClauses();
+ }
clauses.add(queryPos);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0f100004/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java b/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
index fece166..4af077b 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
@@ -18,8 +18,10 @@ package org.apache.lucene.util;
import java.io.IOException;
+import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CannedBinaryTokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockSynonymFilter;
import org.apache.lucene.analysis.MockTokenizer;
@@ -443,4 +445,31 @@ public class TestQueryBuilder extends LuceneTestCase {
QueryBuilder builder = new QueryBuilder(analyzer);
assertNull(builder.createBooleanQuery("field", "whatever"));
}
+
+ public void testMaxBooleanClause() throws Exception {
+ int size = 34;
+ CannedBinaryTokenStream.BinaryToken[] tokens = new CannedBinaryTokenStream.BinaryToken[size];
+ BytesRef term1 = new BytesRef("ff");
+ BytesRef term2 = new BytesRef("f");
+ for (int i = 0; i < size;) {
+ if (i % 2 == 0) {
+ tokens[i] = new CannedBinaryTokenStream.BinaryToken(term2, 1, 1);
+ tokens[i + 1] = new CannedBinaryTokenStream.BinaryToken(term1, 0, 2);
+ i += 2;
+ } else {
+ tokens[i] = new CannedBinaryTokenStream.BinaryToken(term2, 1, 1);
+ i ++;
+ }
+ }
+ QueryBuilder qb = new QueryBuilder(null);
+ try (TokenStream ts = new CannedBinaryTokenStream(tokens)) {
+ expectThrows(BooleanQuery.TooManyClauses.class, () -> qb.analyzeGraphBoolean("", ts, BooleanClause.Occur.MUST));
+ }
+ try (TokenStream ts = new CannedBinaryTokenStream(tokens)) {
+ expectThrows(BooleanQuery.TooManyClauses.class, () -> qb.analyzeGraphBoolean("", ts, BooleanClause.Occur.SHOULD));
+ }
+ try (TokenStream ts = new CannedBinaryTokenStream(tokens)) {
+ expectThrows(BooleanQuery.TooManyClauses.class, () -> qb.analyzeGraphPhrase(ts, "", 0));
+ }
+ }
}
[17/30] lucene-solr:jira/http2: SOLR-12815: Implement maxOps limit
for IndexSizeTrigger.
Posted by da...@apache.org.
SOLR-12815: Implement maxOps limit for IndexSizeTrigger.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/452c2dab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/452c2dab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/452c2dab
Branch: refs/heads/jira/http2
Commit: 452c2dabf08b0c27b1f1504096ddace0ddf126a5
Parents: 152fd96
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Oct 3 20:39:58 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Oct 4 12:48:25 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../cloud/autoscaling/IndexSizeTrigger.java | 34 ++++-
.../cloud/autoscaling/IndexSizeTriggerTest.java | 141 +++++++++++++++++++
.../src/solrcloud-autoscaling-triggers.adoc | 6 +
4 files changed, 182 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/452c2dab/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9bb39e9..981881a 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -127,6 +127,8 @@ New Features
* SOLR-12822: /autoscaling/suggestions to include suggestion to add-replica for lost replicas (noble)
+* SOLR-12815: Implement maxOps limit for IndexSizeTrigger. (ab)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/452c2dab/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 6129cc7..25083ae 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -62,6 +62,7 @@ public class IndexSizeTrigger extends TriggerBase {
public static final String BELOW_DOCS_PROP = "belowDocs";
public static final String BELOW_OP_PROP = "belowOp";
public static final String COLLECTIONS_PROP = "collections";
+ public static final String MAX_OPS_PROP = "maxOps";
public static final String BYTES_SIZE_PROP = "__bytes__";
public static final String DOCS_SIZE_PROP = "__docs__";
@@ -69,9 +70,12 @@ public class IndexSizeTrigger extends TriggerBase {
public static final String BELOW_SIZE_PROP = "belowSize";
public static final String VIOLATION_PROP = "violationType";
+ public static final int DEFAULT_MAX_OPS = 10;
+
public enum Unit { bytes, docs }
private long aboveBytes, aboveDocs, belowBytes, belowDocs;
+ private int maxOps;
private CollectionParams.CollectionAction aboveOp, belowOp;
private final Set<String> collections = new HashSet<>();
private final Map<String, Long> lastAboveEventMap = new ConcurrentHashMap<>();
@@ -80,7 +84,8 @@ public class IndexSizeTrigger extends TriggerBase {
public IndexSizeTrigger(String name) {
super(TriggerEventType.INDEXSIZE, name);
TriggerUtils.validProperties(validProperties,
- ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, BELOW_BYTES_PROP, BELOW_DOCS_PROP, COLLECTIONS_PROP);
+ ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, BELOW_BYTES_PROP, BELOW_DOCS_PROP,
+ COLLECTIONS_PROP, MAX_OPS_PROP);
}
@Override
@@ -151,6 +156,15 @@ public class IndexSizeTrigger extends TriggerBase {
if (belowOp == null) {
throw new TriggerValidationException(getName(), BELOW_OP_PROP, "unrecognized value of: '" + belowOpStr + "'");
}
+ String maxOpsStr = String.valueOf(properties.getOrDefault(MAX_OPS_PROP, DEFAULT_MAX_OPS));
+ try {
+ maxOps = Integer.parseInt(maxOpsStr);
+ if (maxOps < 1) {
+ throw new Exception("must be > 1");
+ }
+ } catch (Exception e) {
+ throw new TriggerValidationException(getName(), MAX_OPS_PROP, "invalid value: '" + maxOpsStr + "': " + e.getMessage());
+ }
}
@Override
@@ -351,7 +365,22 @@ public class IndexSizeTrigger extends TriggerBase {
// calculate ops
final List<TriggerEvent.Op> ops = new ArrayList<>();
aboveSize.forEach((coll, replicas) -> {
+ // sort by decreasing size to first split the largest ones
+ // XXX see the comment below about using DOCS_SIZE_PROP in lieu of BYTES_SIZE_PROP
+ replicas.sort((r1, r2) -> {
+ long delta = (Long) r1.getVariable(DOCS_SIZE_PROP) - (Long) r2.getVariable(DOCS_SIZE_PROP);
+ if (delta > 0) {
+ return -1;
+ } else if (delta < 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+ });
replicas.forEach(r -> {
+ if (ops.size() >= maxOps) {
+ return;
+ }
TriggerEvent.Op op = new TriggerEvent.Op(aboveOp);
op.addHint(Suggester.Hint.COLL_SHARD, new Pair<>(coll, r.getShard()));
ops.add(op);
@@ -365,6 +394,9 @@ public class IndexSizeTrigger extends TriggerBase {
if (replicas.size() < 2) {
return;
}
+ if (ops.size() >= maxOps) {
+ return;
+ }
// sort by increasing size
replicas.sort((r1, r2) -> {
// XXX this is not quite correct - if BYTES_SIZE_PROP decided that replica got here
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/452c2dab/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index fd93d03..996532a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -681,6 +681,147 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
unsupportedOps.forEach(op -> assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction()));
}
+ @Test
+ public void testMaxOps() throws Exception {
+ String collectionName = "testMaxOps_collection";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
+ "conf", 5, 2).setMaxShardsPerNode(10);
+ create.process(solrClient);
+ CloudTestUtils.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
+ CloudTestUtils.clusterShape(5, 2, false, true));
+
+ long waitForSeconds = 3 + random().nextInt(5);
+ // add disabled trigger
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'index_size_trigger5'," +
+ "'event' : 'indexSize'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'aboveDocs' : 10," +
+ "'enabled' : false," +
+ "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'capturing5'," +
+ "'trigger' : 'index_size_trigger5'," +
+ "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," +
+ "'beforeAction' : ['compute_plan']," +
+ "'afterAction' : ['compute_plan']," +
+ "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'finished'," +
+ "'trigger' : 'index_size_trigger5'," +
+ "'stage' : ['SUCCEEDED']," +
+ "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+
+ for (int i = 0; i < 200; i++) {
+ SolrInputDocument doc = new SolrInputDocument("id", "id-" + i);
+ solrClient.add(collectionName, doc);
+ }
+ solrClient.commit(collectionName);
+
+ // enable the trigger
+ String resumeTriggerCommand = "{" +
+ "'resume-trigger' : {" +
+ "'name' : 'index_size_trigger5'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+
+ boolean await = finished.await(60000 / SPEED, TimeUnit.MILLISECONDS);
+ assertTrue("did not finish processing in time", await);
+
+ // suspend the trigger
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : 'index_size_trigger5'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ assertEquals(1, listenerEvents.size());
+ List<CapturedEvent> events = listenerEvents.get("capturing5");
+ assertNotNull("'capturing5' events not found", events);
+ assertEquals("events: " + events, 4, events.size());
+ assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
+ assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
+ assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
+ assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(3).stage);
+ // check ops
+ List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) events.get(2).event.getProperty(TriggerEvent.REQUESTED_OPS);
+ assertNotNull("should contain requestedOps", ops);
+ assertEquals("number of ops: " + ops, 5, ops.size());
+
+ listenerEvents.clear();
+ finished = new CountDownLatch(1);
+
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'index_size_trigger5'," +
+ "'event' : 'indexSize'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'aboveDocs' : 10," +
+ "'maxOps' : 3," +
+ "'enabled' : true," +
+ "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}]" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ await = finished.await(60000 / SPEED, TimeUnit.MILLISECONDS);
+ assertTrue("did not finish processing in time", await);
+
+ // suspend the trigger
+ suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : 'index_size_trigger5'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ assertEquals(1, listenerEvents.size());
+ events = listenerEvents.get("capturing5");
+ assertNotNull("'capturing5' events not found", events);
+ assertEquals("events: " + events, 4, events.size());
+ assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
+ assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
+ assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
+ assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(3).stage);
+ // check ops
+ ops = (List<TriggerEvent.Op>) events.get(2).event.getProperty(TriggerEvent.REQUESTED_OPS);
+ assertNotNull("should contain requestedOps", ops);
+ assertEquals("number of ops: " + ops, 3, ops.size());
+ }
+
private Map<String, Object> createTriggerProps(long waitForSeconds) {
Map<String, Object> props = new HashMap<>();
props.put("event", "indexSize");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/452c2dab/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index 0b41e21..97b9dd7 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -298,6 +298,12 @@ the default value is `MERGESHARDS` (but see the note above).
A comma-separated list of collection names that this trigger should monitor. If not
specified or empty all collections are monitored.
+`maxOps`::
+Maximum number of operations requested in a single event. This property limits the speed of
+changes in a highly dynamic situation, which may lead to more serious threshold violations,
+but it also limits the maximum load on the cluster that the large number of requested
+operations may cause. The default value is 10.
+
Events generated by this trigger contain additional details about the shards
that exceeded thresholds and the types of violations (upper / lower bounds, bytes / docs metrics).
[15/30] lucene-solr:jira/http2: SOLR-12648: Use the information
whether custom preferences were specified in the CloudUtil.usePolicyFramework
method instead of comparing directly against default preferences. Comparing
against default preferences was wron
Posted by da...@apache.org.
SOLR-12648: Use the information whether custom preferences were specified in the CloudUtil.usePolicyFramework method instead of comparing directly against default preferences. Comparing against default preferences was wrong because it ignores the case where the user explicitly added the same preferences as the default ones.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/05949a32
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/05949a32
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/05949a32
Branch: refs/heads/jira/http2
Commit: 05949a32c9a1a20cce1623e6b76bc8397c4bc325
Parents: 554ac64
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Oct 3 16:58:45 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Oct 4 13:26:51 2018 +0530
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/cloud/CloudUtil.java | 3 +--
.../apache/solr/client/solrj/cloud/autoscaling/Policy.java | 7 +++++++
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/05949a32/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
index 26ba1b8..14bebef 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
@@ -29,7 +29,6 @@ import java.util.Optional;
import org.apache.commons.io.FileUtils;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
@@ -174,7 +173,7 @@ public class CloudUtil {
// if no autoscaling configuration exists then obviously we cannot use the policy framework
if (autoScalingConfig.getPolicy().isEmpty()) return false;
// do custom preferences exist
- if (!autoScalingConfig.getPolicy().getClusterPreferences().equals(Policy.DEFAULT_PREFERENCES)) return true;
+ if (!autoScalingConfig.getPolicy().isEmptyPreferences()) return true;
// does a cluster policy exist
if (!autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) return true;
// finally we check if the current collection has a policy
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/05949a32/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 78043df..cfe9455 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -478,6 +478,13 @@ public class Policy implements MapWriter {
return empty;
}
+ /**
+ * @return true if no preferences were specified by the user, false otherwise
+ */
+ public boolean isEmptyPreferences() {
+ return emptyPreferences;
+ }
+
/*This stores the logical state of the system, given a policy and
* a cluster state.
*
[16/30] lucene-solr:jira/http2: SOLR-12827: Migrate cluster wide
defaults syntax in cluster properties to a nested structure
Posted by da...@apache.org.
SOLR-12827: Migrate cluster wide defaults syntax in cluster properties to a nested structure
The cluster wide defaults structure has changed from {collectionDefaults: {nrtReplicas : 2}} to {defaults : {collection : {nrtReplicas : 2}}}. The old format continues to be supported and can be read from ZK as well as written using the V2 set-obj-property syntax but it is deprecated and will be removed in Solr 9. We recommend that users change their API calls to use the new format going forward.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/152fd966
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/152fd966
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/152fd966
Branch: refs/heads/jira/http2
Commit: 152fd966a7a23b4a5379d9b24ae731ef4fe58766
Parents: 05949a3
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Thu Oct 4 16:12:55 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Oct 4 16:12:55 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 9 ++
.../solr/handler/admin/CollectionsHandler.java | 10 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 127 ++++++++++++++++++-
.../test/org/apache/solr/util/TestUtils.java | 6 +-
solr/solr-ref-guide/src/collections-api.adoc | 40 +++---
.../solr/common/cloud/ClusterProperties.java | 41 +++++-
.../apache/solr/common/cloud/ZkStateReader.java | 7 +-
.../common/params/CollectionAdminParams.java | 5 +
.../src/resources/apispec/cluster.Commands.json | 34 ++++-
9 files changed, 240 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7337a23..9bb39e9 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -106,6 +106,11 @@ Upgrade Notes
* SOLR-12767: The min_rf parameter is no longer needed, Solr will always return the achieved replication factor (rf)
in the response header.
+* SOLR-12827: The cluster wide defaults structure has changed from {collectionDefaults: {nrtReplicas : 2}} to
+ {defaults : {collection : {nrtReplicas : 2}}}. The old format continues to be supported and can be read from
+ ZK as well as written using the V2 set-obj-property syntax but it is deprecated and will be removed in Solr 9.
+ We recommend that users change their API calls to use the new format going forward.
+
New Features
----------------------
@@ -132,6 +137,10 @@ Other Changes
of a builder class instead of calling a method with large number of arguments. The number of special cases that had
to be handled have been cut down as well. (shalin)
+* SOLR-12827: Migrate cluster wide defaults syntax in cluster properties to a nested structure. The structure has
+ changed from {collectionDefaults: {nrtReplicas : 2}} to {defaults : {collection : {nrtReplicas : 2}}}.
+ (ab, shalin)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index aef2448..93181a4 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -123,7 +123,6 @@ import static org.apache.solr.common.cloud.DocCollection.RULE;
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
@@ -213,7 +212,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
protected void copyFromClusterProp(Map<String, Object> props, String prop) throws IOException {
if (props.get(prop) != null) return;//if it's already specified , return
Object defVal = new ClusterProperties(coreContainer.getZkController().getZkStateReader().getZkClient())
- .getClusterProperty(ImmutableList.of(COLLECTION_DEF, prop), null);
+ .getClusterProperty(ImmutableList.of(CollectionAdminParams.DEFAULTS, CollectionAdminParams.COLLECTION, prop), null);
if (defVal != null) props.put(prop, String.valueOf(defVal));
}
@@ -457,13 +456,6 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}
public enum CollectionOperation implements CollectionOp {
- /**
- * very simple currently, you can pass a template collection, and the new collection is created on
- * every node the template collection is on
- * there is a lot more to add - you should also be able to create with an explicit server list
- * we might also want to think about error handling (add the request to a zk queue and involve overseer?)
- * as well as specific replicas= options
- */
CREATE_OP(CREATE, (req, rsp, h) -> {
Map<String, Object> props = copy(req.getParams().required(), null, NAME);
props.put("fromApi", "true");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index cd9087d..d9826eb 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -56,6 +56,8 @@ import static java.util.Arrays.asList;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.NUM_SHARDS_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
+import static org.apache.solr.common.params.CollectionAdminParams.DEFAULTS;
@LuceneTestCase.Slow
public class CollectionsAPISolrJTest extends SolrCloudTestCase {
@@ -99,7 +101,7 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
}
@Test
- public void testCreateCollWithDefaultClusterProperties() throws Exception {
+ public void testCreateCollWithDefaultClusterPropertiesOldFormat() throws Exception {
String COLL_NAME = "CollWithDefaultClusterProperties";
try {
V2Response rsp = new V2Request.Builder("/cluster")
@@ -113,9 +115,9 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
if (m != null) break;
Thread.sleep(10);
}
- Object clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP), null);
+ Object clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NUM_SHARDS_PROP), null);
assertEquals("2", String.valueOf(clusterProperty));
- clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS), null);
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NRT_REPLICAS), null);
assertEquals("2", String.valueOf(clusterProperty));
CollectionAdminResponse response = CollectionAdminRequest
.createCollection(COLL_NAME, "conf", null, null, null, null)
@@ -130,12 +132,131 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertEquals(2, slice.getReplicas().size());
}
CollectionAdminRequest.deleteCollection(COLL_NAME).process(cluster.getSolrClient());
+
+ // unset only a single value using old format
+ rsp = new V2Request.Builder("/cluster")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{\n" +
+ " \"set-obj-property\": {\n" +
+ " \"collectionDefaults\": {\n" +
+ " \"nrtReplicas\": null\n" +
+ " }\n" +
+ " }\n" +
+ "}")
+ .build()
+ .process(cluster.getSolrClient());
+ // assert that it is really gone in both old and new paths
+ // we use a timeout so that the change made in ZK is reflected in the watched copy inside ZkStateReader
+ TimeOut timeOut = new TimeOut(5, TimeUnit.SECONDS, new TimeSource.NanoTimeSource());
+ while (!timeOut.hasTimedOut()) {
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NRT_REPLICAS), null);
+ if (clusterProperty == null) break;
+ }
+ assertNull(clusterProperty);
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS), null);
+ assertNull(clusterProperty);
+
+ // delete all defaults the old way
+ rsp = new V2Request.Builder("/cluster")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{set-obj-property:{collectionDefaults:null}}")
+ .build()
+ .process(cluster.getSolrClient());
+ // assert that it is really gone in both old and new paths
+ timeOut = new TimeOut(5, TimeUnit.SECONDS, new TimeSource.NanoTimeSource());
+ while (!timeOut.hasTimedOut()) {
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NUM_SHARDS_PROP), null);
+ if (clusterProperty == null) break;
+ }
+ assertNull(clusterProperty);
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP), null);
+ assertNull(clusterProperty);
} finally {
+ // clean up in case there was an exception during the test
V2Response rsp = new V2Request.Builder("/cluster")
.withMethod(SolrRequest.METHOD.POST)
.withPayload("{set-obj-property:{collectionDefaults: null}}")
.build()
.process(cluster.getSolrClient());
+ }
+
+ }
+
+ @Test
+ public void testCreateCollWithDefaultClusterPropertiesNewFormat() throws Exception {
+ String COLL_NAME = "CollWithDefaultClusterProperties";
+ try {
+ V2Response rsp = new V2Request.Builder("/cluster")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{set-obj-property:{defaults : {collection:{numShards : 2 , nrtReplicas : 2}}}}")
+ .build()
+ .process(cluster.getSolrClient());
+
+ for (int i = 0; i < 10; i++) {
+ Map m = cluster.getSolrClient().getZkStateReader().getClusterProperty(COLLECTION_DEF, null);
+ if (m != null) break;
+ Thread.sleep(10);
+ }
+ Object clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NUM_SHARDS_PROP), null);
+ assertEquals("2", String.valueOf(clusterProperty));
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NRT_REPLICAS), null);
+ assertEquals("2", String.valueOf(clusterProperty));
+ CollectionAdminResponse response = CollectionAdminRequest
+ .createCollection(COLL_NAME, "conf", null, null, null, null)
+ .process(cluster.getSolrClient());
+ assertEquals(0, response.getStatus());
+ assertTrue(response.isSuccess());
+
+ DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getClusterState().getCollection(COLL_NAME);
+ Map<String, Slice> slices = coll.getSlicesMap();
+ assertEquals(2, slices.size());
+ for (Slice slice : slices.values()) {
+ assertEquals(2, slice.getReplicas().size());
+ }
+ CollectionAdminRequest.deleteCollection(COLL_NAME).process(cluster.getSolrClient());
+
+ // unset only a single value
+ rsp = new V2Request.Builder("/cluster")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{\n" +
+ " \"set-obj-property\": {\n" +
+ " \"defaults\" : {\n" +
+ " \"collection\": {\n" +
+ " \"nrtReplicas\": null\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ "}")
+ .build()
+ .process(cluster.getSolrClient());
+ // we use a timeout so that the change made in ZK is reflected in the watched copy inside ZkStateReader
+ TimeOut timeOut = new TimeOut(5, TimeUnit.SECONDS, new TimeSource.NanoTimeSource());
+ while (!timeOut.hasTimedOut()) {
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NRT_REPLICAS), null);
+ if (clusterProperty == null) break;
+ }
+ assertNull(clusterProperty);
+
+ rsp = new V2Request.Builder("/cluster")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{set-obj-property:{defaults: {collection:null}}}")
+ .build()
+ .process(cluster.getSolrClient());
+ // assert that it is really gone in both old and new paths
+ timeOut = new TimeOut(5, TimeUnit.SECONDS, new TimeSource.NanoTimeSource());
+ while (!timeOut.hasTimedOut()) {
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(DEFAULTS, COLLECTION, NUM_SHARDS_PROP), null);
+ if (clusterProperty == null) break;
+ }
+ assertNull(clusterProperty);
+ clusterProperty = cluster.getSolrClient().getZkStateReader().getClusterProperty(ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP), null);
+ assertNull(clusterProperty);
+ } finally {
+ V2Response rsp = new V2Request.Builder("/cluster")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{set-obj-property:{defaults: null}}")
+ .build()
+ .process(cluster.getSolrClient());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/core/src/test/org/apache/solr/util/TestUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.java
index 27db2d6..beb0bf6 100644
--- a/solr/core/src/test/org/apache/solr/util/TestUtils.java
+++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java
@@ -318,9 +318,7 @@ public class TestUtils extends SolrTestCaseJ4 {
sink = new HashMap<>();
sink.put("legacyCloud", "false");
assertTrue(Utils.mergeJson(sink, (Map<String, Object>) Utils.fromJSONString("collectionDefaults:{numShards:3 , nrtReplicas:2}")));
- assertEquals(3l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP)));
- assertEquals(2l, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS)));
-
-
+ assertEquals(3L, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NUM_SHARDS_PROP)));
+ assertEquals(2L, Utils.getObjectByPath(sink, true, ImmutableList.of(COLLECTION_DEF, NRT_REPLICAS)));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index d601069..a5b4e56 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -1201,7 +1201,7 @@ http://localhost:8983/solr/admin/collections?action=CLUSTERPROP&name=urlScheme&v
=== Deeply Nested Cluster Properties ===
-==== `collectionDefaults` ====
+==== `defaults` ====
It is possible to set cluster-wide default values for certain attributes of a collection.
@@ -1209,14 +1209,17 @@ It is possible to set cluster-wide default values for certain attributes of a co
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
-{ "set-obj-property" : {
- "collectionDefaults" : {
- "numShards" : 2,
- "nrtReplicas" : 1,
- "tlogReplicas" : 1,
- "pullReplicas" : 1,
-
- }
+{
+ "set-obj-property": {
+ "defaults" : {
+ "collection": {
+ "numShards": 2,
+ "nrtReplicas": 1,
+ "tlogReplicas": 1,
+ "pullReplicas": 1
+ }
+ }
+ }
}' http://localhost:8983/api/cluster
----
@@ -1224,22 +1227,29 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
-{ "set-obj-property" : {
- "collectionDefaults" : {
- "nrtReplicas" : null,
- }
+{
+ "set-obj-property": {
+ "defaults" : {
+ "collection": {
+ "nrtReplicas": null
+ }
+ }
+ }
}' http://localhost:8983/api/cluster
----
-*Example 2: Unset all values in `collectionDefaults`*
+*Example 2: Unset all values in `defaults`*
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
{ "set-obj-property" : {
- "collectionDefaults" : null
+ "defaults" : null
}' http://localhost:8983/api/cluster
----
+NOTE: Until Solr 7.5, cluster properties supported a "collectionDefaults" key which is no longer supported. Using the API
+structure for Solr 7.4 or Solr 7.5 will continue to work but the format of the properties will automatically be converted
+to the new nested structure. The old "collectionDefaults" key is deprecated and will be removed in Solr 9.
[[collectionprop]]
== COLLECTIONPROP: Collection Properties
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
index 2452540..21d1298 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -32,6 +33,8 @@ import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_DEF;
+
/**
* Interact with solr cluster properties
*
@@ -93,7 +96,8 @@ public class ClusterProperties {
@SuppressWarnings("unchecked")
public Map<String, Object> getClusterProperties() throws IOException {
try {
- return (Map<String, Object>) Utils.fromJSON(client.getData(ZkStateReader.CLUSTER_PROPS, null, new Stat(), true));
+ Map<String, Object> properties = (Map<String, Object>) Utils.fromJSON(client.getData(ZkStateReader.CLUSTER_PROPS, null, new Stat(), true));
+ return convertCollectionDefaultsToNestedFormat(properties);
} catch (KeeperException.NoNodeException e) {
return Collections.emptyMap();
} catch (KeeperException | InterruptedException e) {
@@ -103,14 +107,45 @@ public class ClusterProperties {
public void setClusterProperties(Map<String, Object> properties) throws IOException, KeeperException, InterruptedException {
client.atomicUpdate(ZkStateReader.CLUSTER_PROPS, zkData -> {
- if (zkData == null) return Utils.toJSON(properties);
+ if (zkData == null) return Utils.toJSON(convertCollectionDefaultsToNestedFormat(properties));
Map<String, Object> zkJson = (Map<String, Object>) Utils.fromJSON(zkData);
- boolean modified = Utils.mergeJson(zkJson, properties);
+ zkJson = convertCollectionDefaultsToNestedFormat(zkJson);
+ boolean modified = Utils.mergeJson(zkJson, convertCollectionDefaultsToNestedFormat(properties));
return modified ? Utils.toJSON(zkJson) : null;
});
}
/**
+ * See SOLR-12827 for background. We auto convert any "collectionDefaults" keys to "defaults/collection" format.
+ * This method will modify the given map and return the same object. Remove this method in Solr 9.
+ *
+ * @param properties the properties to be converted
+ * @return the converted map
+ */
+ static Map<String, Object> convertCollectionDefaultsToNestedFormat(Map<String, Object> properties) {
+ if (properties.containsKey(COLLECTION_DEF)) {
+ Map<String, Object> values = (Map<String, Object>) properties.remove(COLLECTION_DEF);
+ if (values != null) {
+ properties.putIfAbsent(CollectionAdminParams.DEFAULTS, new LinkedHashMap<>());
+ Map<String, Object> defaults = (Map<String, Object>) properties.get(CollectionAdminParams.DEFAULTS);
+ defaults.compute(CollectionAdminParams.COLLECTION, (k, v) -> {
+ if (v == null) return values;
+ else {
+ ((Map) v).putAll(values);
+ return v;
+ }
+ });
+ } else {
+ // explicitly set to null, so set null in the nested format as well
+ properties.putIfAbsent(CollectionAdminParams.DEFAULTS, new LinkedHashMap<>());
+ Map<String, Object> defaults = (Map<String, Object>) properties.get(CollectionAdminParams.DEFAULTS);
+ defaults.put(CollectionAdminParams.COLLECTION, null);
+ }
+ }
+ return properties;
+ }
+
+ /**
* This method sets a cluster property.
*
* @param propertyName The property name to be set.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 6abfba8..6011f8a 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -127,6 +127,11 @@ public class ZkStateReader implements Closeable {
public final static String CONFIGNAME_PROP="configName";
public static final String LEGACY_CLOUD = "legacyCloud";
+
+ /**
+ * @deprecated use {@link org.apache.solr.common.params.CollectionAdminParams#DEFAULTS} instead.
+ */
+ @Deprecated
public static final String COLLECTION_DEF = "collectionDefaults";
public static final String URL_SCHEME = "urlScheme";
@@ -1001,7 +1006,7 @@ public class ZkStateReader implements Closeable {
while (true) {
try {
byte[] data = zkClient.getData(ZkStateReader.CLUSTER_PROPS, clusterPropertiesWatcher, new Stat(), true);
- this.clusterProperties = (Map<String, Object>) Utils.fromJSON(data);
+ this.clusterProperties = ClusterProperties.convertCollectionDefaultsToNestedFormat((Map<String, Object>) Utils.fromJSON(data));
log.debug("Loaded cluster properties: {}", this.clusterProperties);
return;
} catch (KeeperException.NoNodeException e) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
index a0ef11f..cb70fb8 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
@@ -90,4 +90,9 @@ public interface CollectionAdminParams {
* and points to the collection on which the `withCollection` was specified.
*/
String COLOCATED_WITH = "COLOCATED_WITH";
+
+ /**
+ * Used by cluster properties API to provide defaults for collection, cluster etc.
+ */
+ String DEFAULTS = "defaults";
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152fd966/solr/solrj/src/resources/apispec/cluster.Commands.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.json b/solr/solrj/src/resources/apispec/cluster.Commands.json
index 7adc671..d1f5738 100644
--- a/solr/solrj/src/resources/apispec/cluster.Commands.json
+++ b/solr/solrj/src/resources/apispec/cluster.Commands.json
@@ -90,24 +90,50 @@
"location": {
"type": "string"
},
+ "defaults" : {
+ "type" : "object",
+ "properties": {
+ "collection": {
+ "type": "object",
+ "properties": {
+ "numShards": {
+ "type": "integer",
+ "description": "Default number of shards for a collection"
+ },
+ "tlogReplicas": {
+ "type": "integer",
+ "description": "Default number of TLOG replicas"
+ },
+ "pullReplicas": {
+ "type": "integer",
+ "description": "Default number of PULL replicas"
+ },
+ "nrtReplicas": {
+ "type": "integer",
+ "description": "Default number of NRT replicas"
+ }
+ }
+ }
+ }
+ },
"collectionDefaults": {
"type": "object",
"properties": {
"numShards": {
"type": "integer",
- "description": "Default no:of shards for a collection"
+ "description": "Default number of shards for a collection"
},
"tlogReplicas": {
"type": "integer",
- "description": "Default no:of TLOG replicas"
+ "description": "Default number of TLOG replicas"
},
"pullReplicas": {
"type": "integer",
- "description": "Default no:of PULL replicas"
+ "description": "Default number of PULL replicas"
},
"nrtReplicas": {
"type": "integer",
- "description": "Default no:of NRT replicas"
+ "description": "Default number of NRT replicas"
}
}
}
[23/30] lucene-solr:jira/http2: use MapWriter._get()
Posted by da...@apache.org.
use MapWriter._get()
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/14e6eb2c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/14e6eb2c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/14e6eb2c
Branch: refs/heads/jira/http2
Commit: 14e6eb2c2488a5983c4449da32757d1f3bf17ce0
Parents: 5fb384c
Author: Noble Paul <no...@apache.org>
Authored: Fri Oct 5 08:32:00 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Fri Oct 5 08:32:00 2018 +1000
----------------------------------------------------------------------
.../org/apache/solr/cloud/TestCryptoKeys.java | 13 +-
.../apache/solr/core/TestDynamicLoading.java | 22 +--
.../apache/solr/core/TestSolrConfigHandler.java | 175 +++++++++----------
.../apache/solr/handler/TestBlobHandler.java | 45 ++---
.../apache/solr/handler/TestConfigReload.java | 19 +-
.../handler/TestSolrConfigHandlerCloud.java | 3 +-
.../TestSolrConfigHandlerConcurrent.java | 44 +++--
.../apache/solr/common/LinkedHashMapWriter.java | 60 +++++++
.../java/org/apache/solr/common/MapWriter.java | 13 +-
.../org/apache/solr/common/MapWriterMap.java | 14 ++
.../java/org/apache/solr/common/util/Utils.java | 16 +-
11 files changed, 250 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java b/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
index b2044b1..146ad82 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Map;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.common.LinkedHashMapWriter;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.MemClassLoader;
@@ -125,10 +126,10 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
Arrays.asList("overlay", "runtimeLib", blobName, "version"),
1l, 10);
- Map map = TestSolrConfigHandler.getRespMap("/runtime", client);
- String s = (String) Utils.getObjectByPath(map, false, Arrays.asList("error", "msg"));
- assertNotNull(TestBlobHandler.getAsString(map), s);
- assertTrue(TestBlobHandler.getAsString(map), s.contains("should be signed with one of the keys in ZK /keys/exe"));
+ LinkedHashMapWriter map = TestSolrConfigHandler.getRespMap("/runtime", client);
+ String s = map._getStr( "error/msg",null);
+ assertNotNull(map.toString(), s);
+ assertTrue(map.toString(), s.contains("should be signed with one of the keys in ZK /keys/exe"));
String wrongSig = "QKqHtd37QN02iMW9UEgvAO9g9qOOuG5vEBNkbUsN7noc2hhXKic/ABFIOYJA9PKw61mNX2EmNFXOcO3WClYdSw==";
@@ -146,8 +147,8 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
map = TestSolrConfigHandler.getRespMap("/runtime", client);
s = (String) Utils.getObjectByPath(map, false, Arrays.asList("error", "msg"));
- assertNotNull(TestBlobHandler.getAsString(map), s);//No key matched signature for jar
- assertTrue(TestBlobHandler.getAsString(map), s.contains("No key matched signature for jar"));
+ assertNotNull(map.toString(), s);//No key matched signature for jar
+ assertTrue(map.toString(), s.contains("No key matched signature for jar"));
String rightSig = "YkTQgOtvcM/H/5EQdABGl3wjjrPhonAGlouIx59vppBy2cZEofX3qX1yZu5sPNRmJisNXEuhHN2149dxeUmk2Q==";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
index 5e6f2a0..90d9afd 100644
--- a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
+++ b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
@@ -16,14 +16,6 @@
*/
package org.apache.solr.core;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
-import org.apache.solr.handler.TestBlobHandler;
-import org.apache.solr.util.RestTestHarness;
-import org.apache.solr.util.SimplePostTool;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -34,6 +26,14 @@ import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.handler.TestBlobHandler;
+import org.apache.solr.util.RestTestHarness;
+import org.apache.solr.util.SimplePostTool;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
import static java.util.Arrays.asList;
import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues;
@@ -86,8 +86,8 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
Map map = TestSolrConfigHandler.getRespMap("/test1", client);
- assertNotNull(TestBlobHandler.getAsString(map), map = (Map) map.get("error"));
- assertTrue(TestBlobHandler.getAsString(map), map.get("msg").toString().contains(".system collection not available"));
+ assertNotNull(map.toString(), map = (Map) map.get("error"));
+ assertTrue(map.toString(), map.get("msg").toString().contains(".system collection not available"));
TestBlobHandler.createSystemCollection(getHttpSolrClient(baseURL, randomClient.getHttpClient()));
@@ -97,7 +97,7 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
assertNotNull(map = (Map) map.get("error"));
- assertTrue("full output " + TestBlobHandler.getAsString(map), map.get("msg").toString().contains("no such blob or version available: colltest/1" ));
+ assertTrue("full output " + map, map.get("msg").toString().contains("no such blob or version available: colltest/1" ));
payload = " {\n" +
" 'set' : {'watched': {" +
" 'x':'X val',\n" +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
index 88286e0..a8de25e 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
@@ -20,8 +20,6 @@ import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -34,11 +32,12 @@ import com.google.common.collect.ImmutableList;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.LinkedHashMapWriter;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.handler.DumpRequestHandler;
-import org.apache.solr.handler.TestBlobHandler;
import org.apache.solr.handler.TestSolrConfigHandlerConcurrent;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -55,8 +54,8 @@ import org.restlet.ext.servlet.ServerServlet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.Arrays.asList;
import static org.apache.solr.common.util.Utils.getObjectByPath;
-import static org.apache.solr.handler.TestBlobHandler.getAsString;
public class TestSolrConfigHandler extends RestTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -106,49 +105,46 @@ public class TestSolrConfigHandler extends RestTestBase {
public void testProperty() throws Exception {
RestTestHarness harness = restTestHarness;
- Map confMap = getRespMap("/config", harness);
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/luke")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/system")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/mbeans")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/plugins")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/threads")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/properties")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/logging")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/file")));
- assertNotNull(getObjectByPath(confMap, false, Arrays.asList("config", "requestHandler", "/admin/ping")));
+ MapWriter confMap = getRespMap("/config", harness);
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/luke"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/system"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/mbeans"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/plugins"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/threads"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/properties"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/logging"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/file"), null));
+ assertNotNull(confMap._get(asList("config", "requestHandler", "/admin/ping"), null));
String payload = "{\n" +
" 'set-property' : { 'updateHandler.autoCommit.maxDocs':100, 'updateHandler.autoCommit.maxTime':10 , 'requestDispatcher.requestParsers.addHttpRequestToContext':true} \n" +
" }";
runConfigCommand(harness, "/config", payload);
- Map m = (Map) getRespMap("/config/overlay", harness).get("overlay");
- Map props = (Map) m.get("props");
- assertNotNull(props);
- assertEquals("100", String.valueOf(getObjectByPath(props, true, ImmutableList.of("updateHandler", "autoCommit", "maxDocs"))));
- assertEquals("10", String.valueOf(getObjectByPath(props, true, ImmutableList.of("updateHandler", "autoCommit", "maxTime"))));
+ MapWriter m = getRespMap("/config/overlay", harness);
+ MapWriter props =null;
+ assertEquals("100", m._getStr("overlay/props/updateHandler/autoCommit/maxDocs", null));
+ assertEquals("10", m._getStr("overlay/props/updateHandler/autoCommit/maxTime",null));
m = getRespMap("/config/updateHandler", harness);
- assertNotNull(getObjectByPath(m, true, ImmutableList.of("config","updateHandler", "commitWithin", "softCommit")));
- assertNotNull(getObjectByPath(m, true, ImmutableList.of("config","updateHandler", "autoCommit", "maxDocs")));
- assertNotNull(getObjectByPath(m, true, ImmutableList.of("config","updateHandler", "autoCommit", "maxTime")));
+ assertNotNull(m._get("config/updateHandler/commitWithin/softCommit",null));
+ assertNotNull(m._get("config/updateHandler/autoCommit/maxDocs",null));
+ assertNotNull(m._get("config/updateHandler/autoCommit/maxTime",null));
- m = (Map) getRespMap("/config", harness).get("config");
+ m = getRespMap("/config", harness);
assertNotNull(m);
- assertEquals("100", String.valueOf(getObjectByPath(m, true, ImmutableList.of("updateHandler", "autoCommit", "maxDocs"))));
- assertEquals("10", String.valueOf(getObjectByPath(m, true, ImmutableList.of("updateHandler", "autoCommit", "maxTime"))));
- assertEquals("true", String.valueOf(getObjectByPath(m, true, ImmutableList.of("requestDispatcher", "requestParsers", "addHttpRequestToContext"))));
+ assertEquals("100", m._getStr("config/updateHandler/autoCommit/maxDocs",null));
+ assertEquals("10", m._getStr("config/updateHandler/autoCommit/maxTime",null));
+ assertEquals("true", m._getStr("config/requestDispatcher/requestParsers/addHttpRequestToContext",null));
payload = "{\n" +
" 'unset-property' : 'updateHandler.autoCommit.maxDocs'} \n" +
" }";
runConfigCommand(harness, "/config", payload);
- m = (Map) getRespMap("/config/overlay", harness).get("overlay");
- props = (Map) m.get("props");
- assertNotNull(props);
- assertNull(getObjectByPath(props, true, ImmutableList.of("updateHandler", "autoCommit", "maxDocs")));
- assertEquals("10", String.valueOf(getObjectByPath(props, true, ImmutableList.of("updateHandler", "autoCommit", "maxTime"))));
+ m = getRespMap("/config/overlay", harness);
+ assertNull(m._get("overlay/props/updateHandler/autoCommit/maxDocs",null));
+ assertEquals("10", m._getStr("overlay/props/updateHandler/autoCommit/maxTime",null));
}
public void testUserProp() throws Exception {
@@ -159,17 +155,14 @@ public class TestSolrConfigHandler extends RestTestBase {
" }";
runConfigCommand(harness, "/config", payload);
- Map m = (Map) getRespMap("/config/overlay", harness).get("overlay");
- Map props = (Map) m.get("userProps");
- assertNotNull(props);
- assertEquals(props.get("my.custom.variable.a"), "MODIFIEDA");
- assertEquals(props.get("my.custom.variable.b"), "MODIFIEDB");
+ MapWriter m = getRespMap("/config/overlay", harness);//.get("overlay");
+ assertEquals(m._get("overlay/userProps/my.custom.variable.a",null), "MODIFIEDA");
+ assertEquals(m._get("overlay/userProps/my.custom.variable.b",null), "MODIFIEDB");
- m = (Map) getRespMap("/dump?json.nl=map&initArgs=true", harness).get("initArgs");
+ m = getRespMap("/dump?json.nl=map&initArgs=true", harness);//.get("initArgs");
- m = (Map) m.get(PluginInfo.DEFAULTS);
- assertEquals("MODIFIEDA", m.get("a"));
- assertEquals("MODIFIEDB", m.get("b"));
+ assertEquals("MODIFIEDA", m._get("initArgs/defaults/a",null));
+ assertEquals("MODIFIEDB", m._get("initArgs/defaults/b",null));
}
@@ -210,7 +203,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config/overlay",
cloudSolrClient,
- Arrays.asList("overlay", "requestHandler", "/x", "startup"),
+ asList("overlay", "requestHandler", "/x", "startup"),
"lazy",
10);
@@ -224,7 +217,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config/overlay",
cloudSolrClient,
- Arrays.asList("overlay", "requestHandler", "/x", "a"),
+ asList("overlay", "requestHandler", "/x", "a"),
"b",
10);
@@ -240,7 +233,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config/overlay",
cloudSolrClient,
- Arrays.asList("overlay", "requestHandler", "/dump", "defaults", "c" ),
+ asList("overlay", "requestHandler", "/dump", "defaults", "c"),
"C",
10);
@@ -248,7 +241,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/x?getdefaults=true&json.nl=map",
cloudSolrClient,
- Arrays.asList("getdefaults", "def_a"),
+ asList("getdefaults", "def_a"),
"def A val",
10);
@@ -256,8 +249,8 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/x?param=multival&json.nl=map",
cloudSolrClient,
- Arrays.asList("params", "multival"),
- Arrays.asList("a", "b", "c"),
+ asList("params", "multival"),
+ asList("a", "b", "c"),
10);
payload = "{\n" +
@@ -270,7 +263,7 @@ public class TestSolrConfigHandler extends RestTestBase {
while (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
String uri = "/config/overlay";
Map m = testServerBaseUrl == null ? getRespMap(uri, writeHarness) : TestSolrConfigHandlerConcurrent.getAsMap(testServerBaseUrl + uri, cloudSolrClient);
- if (null == Utils.getObjectByPath(m, true, Arrays.asList("overlay", "requestHandler", "/x", "a"))) {
+ if (null == Utils.getObjectByPath(m, true, asList("overlay", "requestHandler", "/x", "a"))) {
success = true;
break;
}
@@ -287,7 +280,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "queryConverter", "qc", "class"),
+ asList("config", "queryConverter", "qc", "class"),
"org.apache.solr.spelling.SpellingQueryConverter",
10);
payload = "{\n" +
@@ -298,7 +291,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "queryConverter", "qc", "class"),
+ asList("config", "queryConverter", "qc", "class"),
"org.apache.solr.spelling.SuggestQueryConverter",
10);
@@ -310,7 +303,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "queryConverter", "qc"),
+ asList("config", "queryConverter", "qc"),
null,
10);
@@ -322,7 +315,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "searchComponent", "tc", "class"),
+ asList("config", "searchComponent", "tc", "class"),
"org.apache.solr.handler.component.TermsComponent",
10);
payload = "{\n" +
@@ -333,7 +326,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "searchComponent", "tc", "class"),
+ asList("config", "searchComponent", "tc", "class"),
"org.apache.solr.handler.component.TermVectorComponent",
10);
@@ -345,7 +338,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "searchComponent", "tc"),
+ asList("config", "searchComponent", "tc"),
null,
10);
//<valueSourceParser name="countUsage" class="org.apache.solr.core.CountUsageValueSourceParser"/>
@@ -357,7 +350,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "valueSourceParser", "cu", "class"),
+ asList("config", "valueSourceParser", "cu", "class"),
"org.apache.solr.core.CountUsageValueSourceParser",
10);
// <valueSourceParser name="nvl" class="org.apache.solr.search.function.NvlValueSourceParser">
@@ -371,7 +364,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "valueSourceParser", "cu", "class"),
+ asList("config", "valueSourceParser", "cu", "class"),
"org.apache.solr.search.function.NvlValueSourceParser",
10);
@@ -383,7 +376,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "valueSourceParser", "cu"),
+ asList("config", "valueSourceParser", "cu"),
null,
10);
// <transformer name="mytrans2" class="org.apache.solr.response.transform.ValueAugmenterFactory" >
@@ -397,7 +390,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "transformer", "mytrans", "class"),
+ asList("config", "transformer", "mytrans", "class"),
"org.apache.solr.response.transform.ValueAugmenterFactory",
10);
@@ -409,7 +402,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "transformer", "mytrans", "value"),
+ asList("config", "transformer", "mytrans", "value"),
"6",
10);
@@ -422,12 +415,12 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "transformer", "mytrans"),
+ asList("config", "transformer", "mytrans"),
null,
10);
- List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("config", "initParams"));
- assertNotNull("no object /config/initParams : "+ TestBlobHandler.getAsString(map) , l);
+ List l = (List) Utils.getObjectByPath(map, false, asList("config", "initParams"));
+ assertNotNull("no object /config/initParams : "+ map , l);
assertEquals( 2, l.size());
assertEquals( "val", ((Map)l.get(1)).get("key") );
@@ -449,7 +442,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "searchComponent","myspellcheck", "spellchecker", "class"),
+ asList("config", "searchComponent", "myspellcheck", "spellchecker", "class"),
"solr.DirectSolrSpellChecker",
10);
@@ -467,7 +460,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config",
cloudSolrClient,
- Arrays.asList("config", "requestHandler","/dump100", "class"),
+ asList("config", "requestHandler", "/dump100", "class"),
"org.apache.solr.handler.DumpRequestHandler",
10);
@@ -490,7 +483,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config/overlay",
cloudSolrClient,
- Arrays.asList("overlay", "requestHandler", "/dump101", "startup"),
+ asList("overlay", "requestHandler", "/dump101", "startup"),
"lazy",
10);
@@ -503,7 +496,7 @@ public class TestSolrConfigHandler extends RestTestBase {
testServerBaseUrl,
"/config/overlay",
cloudSolrClient,
- Arrays.asList("overlay", "cache", "lfuCacheDecayFalse", "class"),
+ asList("overlay", "cache", "lfuCacheDecayFalse", "class"),
"solr.search.LFUCache",
10);
assertEquals("solr.search.LRUCache",getObjectByPath(map, true, ImmutableList.of("overlay", "cache", "perSegFilter", "class")));
@@ -555,7 +548,7 @@ public class TestSolrConfigHandler extends RestTestBase {
}
}
- public static Map testForResponseElement(RestTestHarness harness,
+ public static LinkedHashMapWriter testForResponseElement(RestTestHarness harness,
String testServerBaseUrl,
String uri,
CloudSolrClient cloudSolrClient, List<String> jsonPath,
@@ -564,7 +557,7 @@ public class TestSolrConfigHandler extends RestTestBase {
boolean success = false;
long startTime = System.nanoTime();
- Map m = null;
+ LinkedHashMapWriter m = null;
while (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
try {
@@ -592,7 +585,7 @@ public class TestSolrConfigHandler extends RestTestBase {
Thread.sleep(100);
}
- assertTrue(StrUtils.formatString("Could not get expected value ''{0}'' for path ''{1}'' full output: {2}, from server: {3}", expected, StrUtils.join(jsonPath, '/'), getAsString(m), testServerBaseUrl), success);
+ assertTrue(StrUtils.formatString("Could not get expected value ''{0}'' for path ''{1}'' full output: {2}, from server: {3}", expected, StrUtils.join(jsonPath, '/'), m.toString(), testServerBaseUrl), success);
return m;
}
@@ -614,7 +607,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "x", "a"),
+ asList("response", "params", "x", "a"),
"A val",
10);
@@ -623,7 +616,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "x", "b"),
+ asList("response", "params", "x", "b"),
"B val",
10);
@@ -638,7 +631,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/overlay",
null,
- Arrays.asList("overlay", "requestHandler", "/d", "name"),
+ asList("overlay", "requestHandler", "/d", "name"),
"/d",
10);
@@ -646,14 +639,14 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/d?useParams=x",
null,
- Arrays.asList("params", "a"),
+ asList("params", "a"),
"A val",
5);
TestSolrConfigHandler.testForResponseElement(harness,
null,
"/d?useParams=x&a=fomrequest",
null,
- Arrays.asList("params", "a"),
+ asList("params", "a"),
"fomrequest",
5);
@@ -667,7 +660,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/overlay",
null,
- Arrays.asList("overlay", "requestHandler", "/dump1", "name"),
+ asList("overlay", "requestHandler", "/dump1", "name"),
"/dump1",
10);
@@ -676,7 +669,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/dump1",
null,
- Arrays.asList("params", "a"),
+ asList("params", "a"),
"A val",
5);
@@ -697,7 +690,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "y", "c"),
+ asList("response", "params", "y", "c"),
"CY val",
10);
@@ -705,7 +698,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/dump1?useParams=y",
null,
- Arrays.asList("params", "c"),
+ asList("params", "c"),
"CY val",
5);
@@ -715,7 +708,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/dump1?useParams=y",
null,
- Arrays.asList("params", "b"),
+ asList("params", "b"),
"BY val",
5);
@@ -724,7 +717,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/dump1?useParams=y",
null,
- Arrays.asList("params", "a"),
+ asList("params", "a"),
"A val",
5);
@@ -733,8 +726,8 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/dump1?useParams=y",
null,
- Arrays.asList("params", "d"),
- Arrays.asList("val 1", "val 2"),
+ asList("params", "d"),
+ asList("val 1", "val 2"),
5);
payload = " {\n" +
@@ -754,7 +747,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "y", "c"),
+ asList("response", "params", "y", "c"),
"CY val modified",
10);
@@ -763,7 +756,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "y", "e"),
+ asList("response", "params", "y", "e"),
"EY val",
10);
@@ -782,7 +775,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "y", "p"),
+ asList("response", "params", "y", "p"),
"P val",
10);
@@ -791,7 +784,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "y", "c"),
+ asList("response", "params", "y", "c"),
null,
10);
payload = " {'delete' : 'y'}";
@@ -801,7 +794,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/params",
null,
- Arrays.asList("response", "params", "y", "p"),
+ asList("response", "params", "y", "p"),
null,
10);
@@ -829,7 +822,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/config/overlay",
null,
- Arrays.asList("overlay", "requestHandler", "aRequestHandler", "class"),
+ asList("overlay", "requestHandler", "aRequestHandler", "class"),
"org.apache.solr.handler.DumpRequestHandler",
10);
RESTfulServerProvider oldProvider = restTestHarness.getServerProvider();
@@ -840,7 +833,7 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
"/something/part1_Value/fixed/part2_Value?urlTemplateValues=part1&urlTemplateValues=part2",
null,
- Arrays.asList("urlTemplateValues"),
+ asList("urlTemplateValues"),
new ValidatingJsonMap.PredicateWithErrMsg() {
@Override
public String test(Object o) {
@@ -863,13 +856,13 @@ public class TestSolrConfigHandler extends RestTestBase {
}
- public static Map getRespMap(String path, RestTestHarness restHarness) throws Exception {
+ public static LinkedHashMapWriter getRespMap(String path, RestTestHarness restHarness) throws Exception {
String response = restHarness.query(path);
try {
- return (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ return (LinkedHashMapWriter) Utils.MAPWRITEROBJBUILDER.apply(Utils.getJSONParser(new StringReader(response))).getVal();
} catch (JSONParser.ParseException e) {
log.error(response);
- return Collections.emptyMap();
+ return new LinkedHashMapWriter();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java b/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
index 737e16e..62fd8c0 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestBlobHandler.java
@@ -21,7 +21,6 @@ import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -39,11 +38,11 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.common.util.Utils;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.SimplePostTool;
import org.junit.Test;
@@ -52,7 +51,7 @@ import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.util.Utils.getObjectByPath;
+import static java.util.Arrays.asList;
public class TestBlobHandler extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -71,21 +70,21 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
String url = baseUrl + "/.system/config/requestHandler";
- Map map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
+ MapWriter map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
assertNotNull(map);
- assertEquals("solr.BlobHandler", getObjectByPath(map, true, Arrays.asList(
+ assertEquals("solr.BlobHandler", map._get(asList(
"config",
"requestHandler",
"/blob",
- "class")));
+ "class"),null));
map = TestSolrConfigHandlerConcurrent.getAsMap(baseUrl + "/.system/schema/fields/blob", cloudClient);
assertNotNull(map);
- assertEquals("blob", getObjectByPath(map, true, Arrays.asList(
+ assertEquals("blob", map._get(asList(
"field",
- "name")));
- assertEquals("bytes", getObjectByPath(map, true, Arrays.asList(
+ "name"),null));
+ assertEquals("bytes", map._get( asList(
"field",
- "type")));
+ "type"),null));
checkBlobPost(baseUrl, cloudClient);
}
@@ -93,7 +92,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
static void checkBlobPost(String baseUrl, CloudSolrClient cloudClient) throws Exception {
String url;
- Map map;
+ MapWriter map;
byte[] bytarr = new byte[1024];
for (int i = 0; i < bytarr.length; i++) bytarr[i] = (byte) (i % 127);
byte[] bytarr2 = new byte[2048];
@@ -104,11 +103,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
url = baseUrl + "/.system/blob/test/1";
map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
- List l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs"));
- assertNotNull("" + map, l);
- assertTrue("" + map, l.size() > 0);
- map = (Map) l.get(0);
- assertEquals("" + bytarr.length, String.valueOf(map.get("size")));
+ assertEquals("" + bytarr.length, map._getStr("response/docs[0]/size",null));
compareInputAndOutput(baseUrl + "/.system/blob/test?wt=filestream", bytarr2, cloudClient);
compareInputAndOutput(baseUrl + "/.system/blob/test/1?wt=filestream", bytarr, cloudClient);
@@ -126,30 +121,24 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
postData(cloudClient, baseUrl, blobName, bytes);
String url;
- Map map = null;
+ MapWriter map = null;
List l;
final RTimer timer = new RTimer();
int i = 0;
for (; i < 150; i++) {//15 secs
url = baseUrl + "/.system/blob/" + blobName;
map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
- String numFound = String.valueOf(Utils.getObjectByPath(map, false, Arrays.asList("response", "numFound")));
+ String numFound = map._getStr(asList("response", "numFound"),null);
if (!("" + count).equals(numFound)) {
Thread.sleep(100);
continue;
}
- l = (List) Utils.getObjectByPath(map, false, Arrays.asList("response", "docs"));
- assertNotNull(l);
- map = (Map) l.get(0);
- assertEquals("" + bytes.limit(), String.valueOf(map.get("size")));
+
+ assertEquals("" + bytes.limit(), map._getStr("response/docs[0]/size",null));
return;
}
fail(StrUtils.formatString("Could not successfully add blob after {0} attempts. Expecting {1} items. time elapsed {2} output for url is {3}",
- i, count, timer.getTime(), getAsString(map)));
- }
-
- public static String getAsString(Map map) {
- return new String(Utils.toJSON(map), StandardCharsets.UTF_8);
+ i, count, timer.getTime(), map.toString()));
}
static void compareInputAndOutput(String url, byte[] bytarr, CloudSolrClient cloudClient) throws IOException {
@@ -182,7 +171,7 @@ public class TestBlobHandler extends AbstractFullDistribZkTestBase {
try {
response = EntityUtils.toString(entity, StandardCharsets.UTF_8);
Map m = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
- assertFalse("Error in posting blob " + getAsString(m), m.containsKey("error"));
+ assertFalse("Error in posting blob " + m.toString(), m.containsKey("error"));
} catch (JSONParser.ParseException e) {
log.error("$ERROR$", response, e);
fail();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
index bac990c..f95d14e 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
@@ -16,22 +16,20 @@
*/
package org.apache.solr.handler;
-import static java.util.Arrays.asList;
-import static org.apache.solr.common.util.Utils.getObjectByPath;
-
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.LinkedHashMapWriter;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -39,16 +37,17 @@ import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrConfig;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.junit.Test;
-import org.noggit.JSONParser;
-import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.Arrays.asList;
+
public class TestConfigReload extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -100,8 +99,8 @@ public class TestConfigReload extends AbstractFullDistribZkTestBase {
while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds){
Thread.sleep(50);
for (String url : urls) {
- Map respMap = getAsMap(url+uri);
- if(String.valueOf(newVersion).equals(String.valueOf( getObjectByPath(respMap, true, asList(name, "znodeVersion"))))){
+ MapWriter respMap = getAsMap(url + uri);
+ if (String.valueOf(newVersion).equals(respMap._getStr(asList(name, "znodeVersion"), null))) {
succeeded.add(url);
}
}
@@ -111,13 +110,13 @@ public class TestConfigReload extends AbstractFullDistribZkTestBase {
assertEquals(StrUtils.formatString("tried these servers {0} succeeded only in {1} ", urls, succeeded) , urls.size(), succeeded.size());
}
- private Map getAsMap(String uri) throws Exception {
+ private LinkedHashMapWriter getAsMap(String uri) throws Exception {
HttpGet get = new HttpGet(uri) ;
HttpEntity entity = null;
try {
entity = cloudClient.getLbClient().getHttpClient().execute(get).getEntity();
String response = EntityUtils.toString(entity, StandardCharsets.UTF_8);
- return (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ return (LinkedHashMapWriter) Utils.MAPWRITEROBJBUILDER.apply(Utils.getJSONParser(new StringReader(response))).getVal();
} finally {
EntityUtils.consumeQuietly(entity);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
index 944fa65..76957b8 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
@@ -39,7 +39,6 @@ import org.apache.solr.util.RestTestHarness;
import org.junit.Test;
import static java.util.Arrays.asList;
-import static org.apache.solr.handler.TestBlobHandler.getAsString;
public class TestSolrConfigHandlerCloud extends AbstractFullDistribZkTestBase {
@@ -266,7 +265,7 @@ public class TestSolrConfigHandlerCloud extends AbstractFullDistribZkTestBase {
public static void compareValues(Map result, Object expected, List<String> jsonPath) {
Object val = Utils.getObjectByPath(result, false, jsonPath);
- assertTrue(StrUtils.formatString("Could not get expected value {0} for path {1} full output {2}", expected, jsonPath, getAsString(result)),
+ assertTrue(StrUtils.formatString("Could not get expected value {0} for path {1} full output {2}", expected, jsonPath, result.toString()),
expected instanceof Predicate ?
((Predicate)expected ).test(val) :
Objects.equals(expected, val)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
index 8af461b..6aa4e1f 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
@@ -16,10 +16,6 @@
*/
package org.apache.solr.handler;
-import static java.util.Arrays.asList;
-import static org.apache.solr.common.util.Utils.getObjectByPath;
-import static org.noggit.ObjectBuilder.getVal;
-
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
@@ -36,6 +32,8 @@ import org.apache.http.util.EntityUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.common.LinkedHashMapWriter;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -45,10 +43,12 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.util.RestTestHarness;
import org.junit.Test;
import org.noggit.JSONParser;
-import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.Arrays.asList;
+import static org.noggit.ObjectBuilder.getVal;
+
public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBase {
@@ -67,18 +67,15 @@ public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBa
final Map.Entry e = (Map.Entry) o;
if (e.getValue() instanceof Map) {
Map value = (Map) e.getValue();
- Thread t = new Thread() {
- @Override
- public void run() {
- try {
- List<String> errs = new ArrayList<>();
- collectErrors.add(errs);
- invokeBulkCall((String)e.getKey() , errs, value);
- } catch (Exception e) {
- e.printStackTrace();
- }
+ Thread t = new Thread(() -> {
+ try {
+ List<String> errs = new ArrayList<>();
+ collectErrors.add(errs);
+ invokeBulkCall((String)e.getKey() , errs, value);
+ } catch (Exception e1) {
+ e1.printStackTrace();
}
- };
+ });
threads.add(t);
t.start();
}
@@ -155,22 +152,21 @@ public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBa
while ( TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
Thread.sleep(100);
errmessages.clear();
- Map respMap = getAsMap(url+"/config/overlay", cloudClient);
- Map m = (Map) respMap.get("overlay");
- if(m!= null) m = (Map) m.get("props");
+ MapWriter respMap = getAsMap(url + "/config/overlay", cloudClient);
+ MapWriter m = (MapWriter) respMap._get("overlay/props", null);
if(m == null) {
errmessages.add(StrUtils.formatString("overlay does not exist for cache: {0} , iteration: {1} response {2} ", cacheName, i, respMap.toString()));
continue;
}
- Object o = getObjectByPath(m, true, asList("query", cacheName, "size"));
+ Object o = m._get(asList("query", cacheName, "size"), null);
if(!val1.equals(o.toString())) errmessages.add(StrUtils.formatString("'size' property not set, expected = {0}, actual {1}", val1, o));
- o = getObjectByPath(m, true, asList("query", cacheName, "initialSize"));
+ o = m._get(asList("query", cacheName, "initialSize"), null);
if(!val2.equals(o.toString())) errmessages.add(StrUtils.formatString("'initialSize' property not set, expected = {0}, actual {1}", val2, o));
- o = getObjectByPath(m, true, asList("query", cacheName, "autowarmCount"));
+ o = m._get(asList("query", cacheName, "autowarmCount"), null);
if(!val3.equals(o.toString())) errmessages.add(StrUtils.formatString("'autowarmCount' property not set, expected = {0}, actual {1}", val3, o));
if(errmessages.isEmpty()) break;
}
@@ -182,14 +178,14 @@ public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBa
}
- public static Map getAsMap(String uri, CloudSolrClient cloudClient) throws Exception {
+ public static LinkedHashMapWriter getAsMap(String uri, CloudSolrClient cloudClient) throws Exception {
HttpGet get = new HttpGet(uri) ;
HttpEntity entity = null;
try {
entity = cloudClient.getLbClient().getHttpClient().execute(get).getEntity();
String response = EntityUtils.toString(entity, StandardCharsets.UTF_8);
try {
- return (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
+ return (LinkedHashMapWriter) Utils.MAPWRITEROBJBUILDER.apply(new JSONParser(new StringReader(response))).getVal();
} catch (JSONParser.ParseException e) {
log.error(response,e);
throw e;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java
new file mode 100644
index 0000000..fe43400
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class LinkedHashMapWriter<V> extends LinkedHashMap<String, V> implements MapWriter {
+
+ public LinkedHashMapWriter(int initialCapacity) {
+ super(initialCapacity);
+ }
+
+ public LinkedHashMapWriter() {
+ super();
+ }
+
+ public LinkedHashMapWriter(Map<? extends String, ? extends V> m) {
+ super(m);
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ forEach((k, v) -> ew.putNoEx(k, v));
+ }
+
+ @Override
+ public Object _get(String path, Object def) {
+ if (path.indexOf('/') == -1) return getOrDefault(path, (V) def);
+ return MapWriter.super._get(path, def);
+ }
+
+ @Override
+ public Object _get(List<String> path, Object def) {
+ if (path.size() == 1) return getOrDefault(path.get(0), (V) def);
+ return MapWriter.super._get(path, def);
+ }
+
+ @Override
+ public String toString() {
+ return jsonStr();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index 3256b6a..9edd630 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -44,7 +44,7 @@ public interface MapWriter extends MapSerializable {
try {
writeMap(new EntryWriter() {
@Override
- public EntryWriter put(String k, Object v) throws IOException {
+ public EntryWriter put(String k, Object v) {
if (v instanceof MapWriter) v = ((MapWriter) v).toMap(new LinkedHashMap<>());
if (v instanceof IteratorWriter) v = ((IteratorWriter) v).toList(new ArrayList<>());
if (v instanceof Iterable) {
@@ -92,6 +92,11 @@ public interface MapWriter extends MapSerializable {
return v == null ? def : v;
}
+ default String _getStr(String path, String def) {
+ Object v = Utils.getObjectByPath(this, false, path);
+ return v == null ? def : String.valueOf(v);
+ }
+
default void _forEachEntry(String path, BiConsumer fun) {
Utils.forEachMapEntry(this, path, fun);
}
@@ -115,6 +120,12 @@ public interface MapWriter extends MapSerializable {
Object v = Utils.getObjectByPath(this, false, path);
return v == null ? def : v;
}
+
+ default String _getStr(List<String> path, String def) {
+ Object v = Utils.getObjectByPath(this, false, path);
+ return v == null ? def : String.valueOf(v);
+ }
+
/**
* An interface to push one entry at a time to the output.
* The order of the keys is not defined, but we assume they are distinct -- don't call {@code put} more than once
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
index f6580b2..e0dd0ac 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
@@ -18,6 +18,7 @@
package org.apache.solr.common;
import java.io.IOException;
+import java.util.List;
import java.util.Map;
public class MapWriterMap implements MapWriter {
@@ -33,6 +34,19 @@ public class MapWriterMap implements MapWriter {
}
@Override
+ public Object _get(String path, Object def) {
+ if (path.indexOf('/') == -1) return delegate.getOrDefault(path, def);
+ return MapWriter.super._get(path, def);
+ }
+
+ @Override
+ public Object _get(List<String> path, Object def) {
+ if (path.size() == 1) return delegate.getOrDefault(path.get(0), def);
+ return MapWriter.super._get(path, def);
+ }
+
+
+ @Override
public Map toMap(Map<String, Object> map) {
return delegate;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/14e6eb2c/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index ac3f120..095ef57 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -55,6 +55,7 @@ import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.LinkedHashMapWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.MapWriterMap;
import org.apache.solr.common.SolrException;
@@ -241,7 +242,7 @@ public class Utils {
JSONParser.ALLOW_MISSING_COLON_COMMA_BEFORE_OBJECT |
JSONParser.OPTIONAL_OUTER_BRACES);
try {
- return ObjectBuilder.getVal(parser);
+ return STANDARDOBJBUILDER.apply(parser).getVal(parser);
} catch (IOException e) {
throw new RuntimeException(e); // should never happen w/o using real IO
}
@@ -279,6 +280,18 @@ public class Utils {
throw new RuntimeException(e);
}
};
+ public static final Function<JSONParser, ObjectBuilder> MAPWRITEROBJBUILDER = jsonParser -> {
+ try {
+ return new ObjectBuilder(jsonParser){
+ @Override
+ public Object newObject() {
+ return new LinkedHashMapWriter();
+ }
+ };
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ };
public static Object fromJSON(InputStream is, Function<JSONParser, ObjectBuilder> objBuilderProvider) {
try {
@@ -675,4 +688,5 @@ public class Utils {
}
return def;
}
+
}
[07/30] lucene-solr:jira/http2: SOLR-12648: Autoscaling framework
based replica placement is not used unless a policy is specified or non-empty
cluster policy exists
Posted by da...@apache.org.
SOLR-12648: Autoscaling framework based replica placement is not used unless a policy is specified or non-empty cluster policy exists
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/65105aa8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/65105aa8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/65105aa8
Branch: refs/heads/jira/http2
Commit: 65105aa81b1ac61b5e163ca249419184433233df
Parents: e687748
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Oct 3 15:52:51 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Oct 3 15:52:51 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../java/org/apache/solr/cloud/CloudUtil.java | 49 +++-
.../solr/cloud/api/collections/Assign.java | 3 +-
.../api/collections/CreateCollectionCmd.java | 4 +-
.../client/solrj/cloud/autoscaling/Policy.java | 20 +-
.../solrj/cloud/autoscaling/Preference.java | 8 +
.../solrj/cloud/autoscaling/TestPolicy.java | 237 +++++++++++++++++++
7 files changed, 308 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65105aa8/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ca1dacf..576140c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -128,6 +128,9 @@ Bug Fixes
* SOLR-12776: Setting of TMP in solr.cmd causes invisibility of Solr to JDK tools (Petr Bodnar via Erick Erickson)
+* SOLR-12648: Autoscaling framework based replica placement is not used unless a policy is specified or
+ non-empty cluster policy exists. (shalin)
+
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65105aa8/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
index 13734f6..26ba1b8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CloudUtil.java
@@ -23,10 +23,13 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
import org.apache.commons.io.FileUtils;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
@@ -45,8 +48,8 @@ import org.slf4j.LoggerFactory;
public class CloudUtil {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-
+
+
/**
* See if coreNodeName has been taken over by another baseUrl and unload core
* + throw exception if it has been.
@@ -57,26 +60,26 @@ public class CloudUtil {
ZkController zkController = cc.getZkController();
String thisCnn = zkController.getCoreNodeName(desc);
String thisBaseUrl = zkController.getBaseUrl();
-
+
log.debug("checkSharedFSFailoverReplaced running for coreNodeName={} baseUrl={}", thisCnn, thisBaseUrl);
// if we see our core node name on a different base url, unload
- final DocCollection docCollection = zkController.getClusterState().getCollectionOrNull(desc.getCloudDescriptor().getCollectionName());
+ final DocCollection docCollection = zkController.getClusterState().getCollectionOrNull(desc.getCloudDescriptor().getCollectionName());
if (docCollection != null && docCollection.getSlicesMap() != null) {
Map<String,Slice> slicesMap = docCollection.getSlicesMap();
for (Slice slice : slicesMap.values()) {
for (Replica replica : slice.getReplicas()) {
-
+
String cnn = replica.getName();
String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
log.debug("compare against coreNodeName={} baseUrl={}", cnn, baseUrl);
-
+
if (thisCnn != null && thisCnn.equals(cnn)
&& !thisBaseUrl.equals(baseUrl)) {
if (cc.getLoadedCoreNames().contains(desc.getName())) {
cc.unload(desc.getName());
}
-
+
try {
FileUtils.deleteDirectory(desc.getInstanceDir().toFile());
} catch (IOException e) {
@@ -144,9 +147,37 @@ public class CloudUtil {
}
+ /**
+ * <b>Note:</b> where possible, the {@link #usePolicyFramework(DocCollection, SolrCloudManager)} method should
+ * be used instead of this method
+ *
+ * @return true if autoscaling policy framework should be used for replica placement
+ */
+ public static boolean usePolicyFramework(SolrCloudManager cloudManager) throws IOException, InterruptedException {
+ Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null");
+ return usePolicyFramework(Optional.empty(), cloudManager);
+ }
+
+ /**
+ * @return true if auto scaling policy framework should be used for replica placement
+ * for this collection, otherwise false
+ */
public static boolean usePolicyFramework(DocCollection collection, SolrCloudManager cloudManager)
throws IOException, InterruptedException {
+ Objects.requireNonNull(collection, "The DocCollection instance cannot be null");
+ Objects.requireNonNull(cloudManager, "The SolrCloudManager instance cannot be null");
+ return usePolicyFramework(Optional.of(collection), cloudManager);
+ }
+
+ private static boolean usePolicyFramework(Optional<DocCollection> collection, SolrCloudManager cloudManager) throws IOException, InterruptedException {
AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
- return !autoScalingConfig.getPolicy().getClusterPolicy().isEmpty() || collection.getPolicyName() != null;
+ // if no autoscaling configuration exists then obviously we cannot use the policy framework
+ if (autoScalingConfig.getPolicy().isEmpty()) return false;
+ // do custom preferences exist
+ if (!autoScalingConfig.getPolicy().getClusterPreferences().equals(Policy.DEFAULT_PREFERENCES)) return true;
+ // does a cluster policy exist
+ if (!autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) return true;
+ // finally we check if the current collection has a policy
+ return !collection.isPresent() || collection.get().getPolicyName() != null;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65105aa8/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
index 542ca1b..a24ad1a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
@@ -41,6 +41,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException;
import org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper;
import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
+import org.apache.solr.cloud.CloudUtil;
import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
@@ -583,7 +584,7 @@ public class Assign {
AutoScalingConfig autoScalingConfig = solrCloudManager.getDistribStateManager().getAutoScalingConfig();
StrategyType strategyType = null;
- if ((ruleMaps == null || ruleMaps.isEmpty()) && policyName == null && autoScalingConfig.getPolicy().getClusterPolicy().isEmpty()) {
+ if ((ruleMaps == null || ruleMaps.isEmpty()) && !CloudUtil.usePolicyFramework(collection, solrCloudManager)) {
strategyType = StrategyType.LEGACY;
} else if (ruleMaps != null && !ruleMaps.isEmpty()) {
strategyType = StrategyType.RULES;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65105aa8/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index 542345d..212437e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -36,7 +36,6 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException;
import org.apache.solr.client.solrj.cloud.autoscaling.NotEmptyException;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
@@ -134,8 +133,7 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd
String router = message.getStr("router.name", DocRouter.DEFAULT_NAME);
String policy = message.getStr(Policy.POLICY);
- AutoScalingConfig autoScalingConfig = ocmh.cloudManager.getDistribStateManager().getAutoScalingConfig();
- boolean usePolicyFramework = !autoScalingConfig.getPolicy().getClusterPolicy().isEmpty() || policy != null;
+ boolean usePolicyFramework = CloudUtil.usePolicyFramework(ocmh.cloudManager) || policy != null;
// fail fast if parameters are wrong or incomplete
List<String> shardNames = populateShardNames(message, router);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65105aa8/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index b39951a..78043df 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -94,7 +94,15 @@ public class Policy implements MapWriter {
final List<Pair<String, Type>> params;
final List<String> perReplicaAttributes;
final int zkVersion;
+ /**
+ * True if cluster policy, preferences and custom policies are all non-existent
+ */
final boolean empty;
+ /**
+ * True if cluster preferences was originally empty, false otherwise. It is used to figure out if
+ * the current preferences were implicitly added or not.
+ */
+ final boolean emptyPreferences;
public Policy() {
this(Collections.emptyMap());
@@ -115,7 +123,8 @@ public class Policy implements MapWriter {
Preference preference = initialClusterPreferences.get(i);
preference.next = initialClusterPreferences.get(i + 1);
}
- if (initialClusterPreferences.isEmpty()) {
+ emptyPreferences = initialClusterPreferences.isEmpty();
+ if (emptyPreferences) {
initialClusterPreferences.addAll(DEFAULT_PREFERENCES);
}
this.clusterPreferences = Collections.unmodifiableList(initialClusterPreferences);
@@ -162,7 +171,8 @@ public class Policy implements MapWriter {
this.zkVersion = version;
this.policies = policies != null ? Collections.unmodifiableMap(policies) : Collections.emptyMap();
this.clusterPolicy = clusterPolicy != null ? Collections.unmodifiableList(clusterPolicy) : Collections.emptyList();
- this.clusterPreferences = clusterPreferences != null ? Collections.unmodifiableList(clusterPreferences) : DEFAULT_PREFERENCES;
+ this.emptyPreferences = clusterPreferences == null;
+ this.clusterPreferences = emptyPreferences ? DEFAULT_PREFERENCES : Collections.unmodifiableList(clusterPreferences);
this.params = Collections.unmodifiableList(
buildParams(this.clusterPreferences, this.clusterPolicy, this.policies).stream()
.map(s -> new Pair<>(s, VariableBase.getTagType(s)))
@@ -211,6 +221,10 @@ public class Policy implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
+ // if we were initially empty then we don't want to persist any implicitly added
+ // policy or preferences
+ if (empty) return;
+
if (!policies.isEmpty()) {
ew.put(POLICIES, (MapWriter) ew1 -> {
for (Map.Entry<String, List<Clause>> e : policies.entrySet()) {
@@ -218,7 +232,7 @@ public class Policy implements MapWriter {
}
});
}
- if (!clusterPreferences.isEmpty()) {
+ if (!emptyPreferences && !clusterPreferences.isEmpty()) {
ew.put(CLUSTER_PREFERENCES, (IteratorWriter) iw -> {
for (Preference p : clusterPreferences) iw.add(p);
});
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65105aa8/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
index 422f987..127012a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -137,4 +138,11 @@ public class Preference implements MapWriter {
public String toString() {
return Utils.toJSONString(this);
}
+
+ /**
+ * @return an unmodifiable copy of the original map from which this object was constructed
+ */
+ public Map getOriginal() {
+ return Collections.unmodifiableMap(original);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65105aa8/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 29e45ed..23184a0 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -21,6 +21,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -33,6 +34,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -47,11 +49,15 @@ import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.api.collections.Assign;
+import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
+import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
import org.apache.solr.common.params.CollectionParams;
@@ -71,9 +77,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.CLUSTER_PREFERENCES;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.CORES;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.FREEDISK;
import static org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type.REPLICA;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -3036,5 +3046,232 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
+ public void testAutoscalingPreferencesUsedWithNoPolicy() throws IOException, InterruptedException {
+ String dataproviderdata = "{" +
+ " 'liveNodes': [" +
+ " 'node1:8983'," +
+ " 'node2:8984'," +
+ " 'node3:8985'" +
+ " ]," +
+ " 'replicaInfo': {" +
+ " 'node1:8983': {" +
+ " 'c1': {" +
+ " 's1': [" +
+ " {'r1': {'type': 'NRT', 'INDEX.sizeInGB':'1100'}}," +
+ " {'r2': {'type': 'NRT'}}" +
+ " ]," +
+ " 's2': [" +
+ " {'r1': {'type': 'NRT', 'INDEX.sizeInGB':'1100'}}," +
+ " {'r2': {'type': 'NRT'}}" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " 'nodeValues': {" +
+ " 'node1:8983': {" +
+ " 'cores': 4," +
+ " 'freedisk': 300," +
+ " 'totaldisk': 4700," +
+ " 'port': 8983" +
+ " }," +
+ " 'node2:8984': {" +
+ " 'cores': 0," +
+ " 'freedisk': 1000," +
+ " 'totaldisk': 1200," +
+ " 'port': 8984" +
+ " }," +
+ " 'node3:8985': {" +
+ " 'cores': 0," +
+ " 'freedisk': 1651," +
+ " 'totaldisk': 1700," +
+ " 'port': 8985" +
+ " }" +
+ " }," +
+ " 'autoscalingJson': {" +
+ " 'cluster-preferences': [" +
+ " { 'maximize': 'freedisk'}," +
+ " { 'minimize': 'cores', 'precision': 3}" +
+ " ]" +
+ " }" +
+ "}";
+
+ String clusterState = "{\n" +
+ " \"c1\" : {\n" +
+ " \"router\":{\"name\":\"compositeId\"},\n" +
+ " \"maxShardsPerNode\":-1,\n" +
+ " \"shards\" : {\n" +
+ " \"s1\" : {\n" +
+ " \"replicas\" : {\n" +
+ " \"r1\" : {\n" +
+ " \"type\" : \"NRT\",\n" +
+ " \"node_name\" : \"node1:8983\",\n" +
+ " \"state\" : \"active\",\n" +
+ " \"leader\" : \"true\"\n" +
+ " },\n" +
+ " \"r2\" : {\n" +
+ " \"type\" : \"NRT\",\n" +
+ " \"node_name\" : \"node1:8983\",\n" +
+ " \"state\" : \"active\"\n" +
+ " }\n" +
+ " }\n" +
+ " },\n" +
+ " \"s2\" : {\n" +
+ " \"replicas\" : {\n" +
+ " \"r1\" : {\n" +
+ " \"type\" : \"NRT\",\n" +
+ " \"node_name\" : \"node1:8983\",\n" +
+ " \"state\" : \"active\",\n" +
+ " \"leader\" : \"true\"\n" +
+ " },\n" +
+ " \"r2\" : {\n" +
+ " \"type\" : \"NRT\",\n" +
+ " \"node_name\" : \"node1:8983\",\n" +
+ " \"state\" : \"active\"\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+
+ Map m = (Map) Utils.fromJSONString(dataproviderdata);
+
+ Map replicaInfo = (Map) m.get("replicaInfo");
+ replicaInfo.forEach((node, val) -> {
+ Map m1 = (Map) val;
+ m1.forEach((coll, val2) -> {
+ Map m2 = (Map) val2;
+ m2.forEach((shard, val3) -> {
+ List l3 = (List) val3;
+ for (int i = 0; i < l3.size(); i++) {
+ Object o = l3.get(i);
+ Map m3 = (Map) o;
+ String name = m3.keySet().iterator().next().toString();
+ m3 = (Map) m3.get(name);
+ Replica.Type type = Replica.Type.get((String) m3.get("type"));
+ l3.set(i, new ReplicaInfo(name, name
+ , coll.toString(), shard.toString(), type, (String) node, m3));
+ }
+ });
+
+ });
+ });
+ AutoScalingConfig asc = m.containsKey("autoscalingJson") ? new AutoScalingConfig((Map<String, Object>) m.get("autoscalingJson")) : null;
+ DelegatingCloudManager cloudManager = new DelegatingCloudManager(null) {
+
+ @Override
+ public DistribStateManager getDistribStateManager() {
+ return new DelegatingDistribStateManager(null) {
+ @Override
+ public AutoScalingConfig getAutoScalingConfig() {
+ return asc;
+ }
+ };
+ }
+
+ @Override
+ public ClusterStateProvider getClusterStateProvider() {
+ return new DelegatingClusterStateProvider(null) {
+ @Override
+ public Set<String> getLiveNodes() {
+ return new HashSet<>((Collection<String>) m.get("liveNodes"));
+ }
+
+ @Override
+ public ClusterState getClusterState() throws IOException {
+ return ClusterState.load(0, clusterState.getBytes(Charset.forName("UTF-8")), getLiveNodes(), ZkStateReader.getCollectionPath("c1"));
+ }
+ };
+ }
+ @Override
+ public NodeStateProvider getNodeStateProvider() {
+ return new DelegatingNodeStateProvider(null) {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ Map<String, Object> result = (Map<String, Object>) Utils.getObjectByPath(m, false, Arrays.asList("nodeValues", node));
+ return result == null ? Collections.emptyMap() : result;
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ Map<String, Map<String, List<ReplicaInfo>>> result = (Map<String, Map<String, List<ReplicaInfo>>>) Utils.getObjectByPath(m, false, Arrays.asList("replicaInfo", node));
+ return result == null ? Collections.emptyMap() : result;
+ }
+ };
+ }
+ };
+
+ ZkNodeProps message = new ZkNodeProps(
+ Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(),
+ COLLECTION_PROP, "c1",
+ SHARD_ID_PROP, "s1",
+ REPLICA_TYPE, Replica.Type.NRT.toString()
+ );
+
+ Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
+ .forCollection("c1")
+ .forShard(Collections.singletonList("s1"))
+ .assignNrtReplicas(1)
+ .build();
+ Assign.AssignStrategyFactory assignStrategyFactory = new Assign.AssignStrategyFactory(cloudManager);
+ ClusterState state = cloudManager.getClusterStateProvider().getClusterState();
+ DocCollection collection = state.getCollection("c1");
+ Assign.AssignStrategy assignStrategy = assignStrategyFactory.create(state, collection);
+ List<ReplicaPosition> replicaPositions = assignStrategy.assign(cloudManager, assignRequest);
+
+ assertEquals(1, replicaPositions.size());
+ ReplicaPosition replicaPosition = replicaPositions.get(0);
+ assertEquals("node3:8985", replicaPosition.node); // only node3:8985 has enough space to handle the new replica
+ assertEquals("s1", replicaPosition.shard); // sanity check
+ }
+
+ /**
+ * Tests that an empty policy should not persist implicitly added keys to MapWriter
+ * <p>
+ * The reason behind doing this is to ensure that implicitly added cluster preferences do not ever
+ * go to ZooKeeper so that we can decide whether to enable autoscaling policy framework or not.
+ *
+ * @see org.apache.solr.cloud.CloudUtil#usePolicyFramework(DocCollection, SolrCloudManager)
+ */
+ public void testPolicyMapWriterWithEmptyPreferences() throws IOException {
+ List<Map> defaultPreferences = Policy.DEFAULT_PREFERENCES
+ .stream().map(preference -> preference.getOriginal()).collect(Collectors.toList());
+
+ // first we create a completely empty policy
+ Policy policy = new Policy();
+ // sanity check that the default cluster preferences were added implicitly
+ assertNotNull(policy.getClusterPreferences());
+ // and they were the same as the default preferences
+ assertEquals(policy.getClusterPreferences().size(), defaultPreferences.size());
+ Set<String> writtenKeys = new HashSet<>();
+ policy.writeMap(new MapWriter.EntryWriter() {
+ @Override
+ public MapWriter.EntryWriter put(String k, Object v) throws IOException {
+ writtenKeys.add(k);
+ return this;
+ }
+ });
+ // but those implicitly added cluster preferences are never written by MapWriter
+ assertEquals(0, writtenKeys.size());
+
+ // reset
+ writtenKeys.clear();
+ // now we create a policy that only has cluster preferences which happen to be the same as the default
+ // preferences
+ policy = new Policy(Utils.makeMap(CLUSTER_PREFERENCES, defaultPreferences));
+ // sanity checks
+ assertNotNull(policy.getClusterPreferences());
+ assertEquals(policy.getClusterPreferences().size(), defaultPreferences.size());
+ policy.writeMap(new MapWriter.EntryWriter() {
+ @Override
+ public MapWriter.EntryWriter put(String k, Object v) throws IOException {
+ writtenKeys.add(k);
+ return this;
+ }
+ });
+ // since the user explicitly added those preferences, they should be written by MapWriter
+ assertEquals(1, writtenKeys.size());
+ assertTrue(writtenKeys.contains(CLUSTER_PREFERENCES));
+ }
}
[14/30] lucene-solr:jira/http2: SOLR-12524: Marking the test as
AwaitsFix
Posted by da...@apache.org.
SOLR-12524: Marking the test as AwaitsFix
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/554ac646
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/554ac646
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/554ac646
Branch: refs/heads/jira/http2
Commit: 554ac64666a16ac88d84a15ac1acdc62cd04712e
Parents: 46f753d
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Thu Oct 4 06:53:10 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Thu Oct 4 06:53:10 2018 +0530
----------------------------------------------------------------------
.../src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/554ac646/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java
index a11b9ac..6be951d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBidirectionalTest.java
@@ -44,6 +44,7 @@ public class CdcrBidirectionalTest extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Test
+ @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12524")
public void testBiDir() throws Exception {
MiniSolrCloudCluster cluster2 = new MiniSolrCloudCluster(1, createTempDir("cdcr-cluster2"), buildJettyConfig("/solr"));
cluster2.waitForAllNodes(30);
[21/30] lucene-solr:jira/http2: LUCENE-8479: Fix precommit
Posted by da...@apache.org.
LUCENE-8479: Fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/36c60251
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/36c60251
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/36c60251
Branch: refs/heads/jira/http2
Commit: 36c60251f2d99e173c7ca99d423b3d18156cad82
Parents: a0487b0
Author: Jan Høydahl <ja...@apache.org>
Authored: Thu Oct 4 18:17:35 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Thu Oct 4 18:17:35 2018 +0200
----------------------------------------------------------------------
lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36c60251/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java b/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
index 4af077b..1604f51 100644
--- a/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
+++ b/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
@@ -18,7 +18,6 @@ package org.apache.lucene.util;
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CannedBinaryTokenStream;
[13/30] lucene-solr:jira/http2: SOLR-12767: Always include the
achieved rf in the response
Posted by da...@apache.org.
SOLR-12767: Always include the achieved rf in the response
This commit deprecates the min_rf parameter. Solr now always includes the achieved replication
factor in the update requests (as if min_rf was always specified). Also, reverts the changes
introduced in SOLR-8034, replicas that don't ack an update will have to recover to prevent
inconsistent shards.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/46f753d7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/46f753d7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/46f753d7
Branch: refs/heads/jira/http2
Commit: 46f753d7c6df52c06d970a13d3b742310276f2ca
Parents: 751bf7d
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Wed Oct 3 09:40:15 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Wed Oct 3 09:40:15 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 14 +++
.../apache/solr/update/SolrCmdDistributor.java | 8 +-
.../processor/DistributedUpdateProcessor.java | 106 +++++++------------
.../apache/solr/cloud/HttpPartitionTest.java | 103 +-----------------
.../solr/cloud/ReplicationFactorTest.java | 61 ++++++++---
.../solr/update/SolrCmdDistributorTest.java | 4 +-
...olrcloud-recoveries-and-write-tolerance.adoc | 6 +-
.../client/solrj/request/UpdateRequest.java | 4 +
.../solr/BaseDistributedSearchTestCase.java | 2 +
.../cloud/AbstractFullDistribZkTestBase.java | 1 -
10 files changed, 117 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 36a63d2..7337a23 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -101,6 +101,11 @@ Velocity 1.7 and Velocity Tools 2.0
Apache ZooKeeper 3.4.11
Jetty 9.4.11.v20180605
+Upgrade Notes
+----------------------
+* SOLR-12767: The min_rf parameter is no longer needed, Solr will always return the achieved replication factor (rf)
+ in the response header.
+
New Features
----------------------
@@ -140,6 +145,15 @@ Bug Fixes
* SOLR-12750: Migrate API should lock the collection instead of shard. (shalin)
+* SOLR-12767: When using min_rf, shard leader skipped bad replicas instead of marking them for recovery, this could
+ create inconsistencies between replicas of the same shard. min_rf parameter is now deprecated, and even if provided
+ replicas that don't ack an update from the leader will be marked for recovery. (Tomás Fernández Löbbe)
+
+Improvements
+----------------------
+* SOLR-12767: Solr now always includes in the response of update requests the achieved replication factor
+ (Tomás Fernández Löbbe)
+
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index cb7f9cb..9536f9d 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -386,10 +386,10 @@ public class SolrCmdDistributor implements Closeable {
// we need to add the data to the rollup tracker.
//
// In the case of a leaderTracker and rollupTracker both being present, then we need to take care when assembling
- // the final response to check both the rollup and leader trackers on the aggrator node.
+ // the final response to check both the rollup and leader trackers on the aggregator node.
public void trackRequestResult(HttpResponse resp, boolean success) {
-
- // Returing Integer.MAX_VALUE here means there was no "rf" on the response, therefore we just need to increment
+
+ // Returning Integer.MAX_VALUE here means there was no "rf" on the response, therefore we just need to increment
// our achieved rf if we are a leader, i.e. have a leaderTracker.
int rfFromResp = getRfFromResponse(resp);
@@ -420,7 +420,7 @@ public class SolrCmdDistributor implements Closeable {
}
}
} catch (Exception e) {
- log.warn("Failed to parse response from " + node + " during replication factor accounting due to: " + e);
+ log.warn("Failed to parse response from {} during replication factor accounting", node, e);
} finally {
if (inputStream != null) {
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index 247f593..bab8607 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -692,6 +692,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
zkController.getBaseUrl(), req.getCore().getName()));
if (req.getParams().get(UpdateRequest.MIN_REPFACT) != null) {
+ // TODO: Kept for rolling upgrades only. Should be removed in Solr 9
params.set(UpdateRequest.MIN_REPFACT, req.getParams().get(UpdateRequest.MIN_REPFACT));
}
@@ -729,43 +730,32 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
// helper method, processAdd was getting a bit large.
- // Sets replicationTracker = null if we aren't the leader or don't care abeout minRf
- // We have three possibilities here:
+ // Sets replicationTracker = null if we aren't the leader
+ // We have two possibilities here:
//
- // 1> there is no min_rf specified: Just return
- // 2> we are a leader: Allocate a LeaderTracker and, if we're getting the original request, a RollupTracker
- // 3> we're a follower: allocat a RollupTracker
+ // 1> we are a leader: Allocate a LeaderTracker and, if we're getting the original request, a RollupTracker
+ // 2> we're a follower: allocat a RollupTracker
//
private void checkReplicationTracker(UpdateCommand cmd) {
- String repFact = req.getParams().get(UpdateRequest.MIN_REPFACT);
-
- if (zkEnabled == false || repFact == null) {
+ if (zkEnabled == false) {
rollupReplicationTracker = null; // never need one of these in stand-alone
leaderReplicationTracker = null;
return;
}
- int requestedReplicationFactor;
-
- try {
- requestedReplicationFactor = Integer.parseInt(repFact);
- } catch (NumberFormatException nfe) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "MinRF must be an integer, was " + repFact);
- }
-
SolrParams rp = cmd.getReq().getParams();
String distribUpdate = rp.get(DISTRIB_UPDATE_PARAM);
// Ok,we're receiving the original request, we need a rollup tracker, but only one so we accumulate over the
// course of a batch.
if ((distribUpdate == null || DistribPhase.NONE.toString().equals(distribUpdate)) &&
rollupReplicationTracker == null) {
- rollupReplicationTracker = new RollupRequestReplicationTracker(repFact);
+ rollupReplicationTracker = new RollupRequestReplicationTracker();
}
// If we're a leader, we need a leader replication tracker, so let's do that. If there are multiple docs in
// a batch we need to use the _same_ leader replication tracker.
if (isLeader && leaderReplicationTracker == null) {
leaderReplicationTracker = new LeaderRequestReplicationTracker(
- req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId(), requestedReplicationFactor);
+ req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
}
}
@@ -877,12 +867,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
}
- // If the client specified minRf and we didn't achieve the minRf, don't send recovery and let client retry
- if (leaderReplicationTracker != null &&
- leaderReplicationTracker.getAchievedRf() < leaderReplicationTracker.getRequestedRf()) {
- continue;
- }
-
if (leaderCoreNodeName != null && cloudDesc.getCoreNodeName().equals(leaderCoreNodeName) // we are still same leader
&& foundErrorNodeInReplicaList // we found an error for one of replicas
&& !stdNode.getNodeProps().getCoreUrl().equals(leaderProps.getCoreUrl())) { // we do not want to put ourself into LIR
@@ -933,15 +917,23 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
zkController.getShardTerms(cloudDesc.getCollectionName(), cloudDesc.getShardId())
.ensureTermsIsHigher(cloudDesc.getCoreNodeName(), replicasShouldBeInLowerTerms);
}
- // in either case, we need to attach the achieved and min rf to the response.
+ handleReplicationFactor();
+ if (0 < errorsForClient.size()) {
+ throw new DistributedUpdatesAsyncException(errorsForClient);
+ }
+ }
+
+ /**
+ * If necessary, include in the response the achieved replication factor
+ */
+ @SuppressWarnings("deprecation")
+ private void handleReplicationFactor() {
if (leaderReplicationTracker != null || rollupReplicationTracker != null) {
int achievedRf = Integer.MAX_VALUE;
- int requestedRf = Integer.MAX_VALUE;
if (leaderReplicationTracker != null) {
achievedRf = leaderReplicationTracker.getAchievedRf();
- requestedRf = leaderReplicationTracker.getRequestedRf();
// Transfer this to the rollup tracker if it exists
if (rollupReplicationTracker != null) {
@@ -952,19 +944,18 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// Rollup tracker has accumulated stats.
if (rollupReplicationTracker != null) {
achievedRf = rollupReplicationTracker.getAchievedRf();
- requestedRf = rollupReplicationTracker.getRequestedRf();
}
- rsp.getResponseHeader().add(UpdateRequest.MIN_REPFACT, requestedRf);
+ if (req.getParams().get(UpdateRequest.MIN_REPFACT) != null) {
+ // Unused, but kept for back compatibility. To be removed in Solr 9
+ rsp.getResponseHeader().add(UpdateRequest.MIN_REPFACT, Integer.parseInt(req.getParams().get(UpdateRequest.MIN_REPFACT)));
+ }
rsp.getResponseHeader().add(UpdateRequest.REPFACT, achievedRf);
rollupReplicationTracker = null;
leaderReplicationTracker = null;
}
- if (0 < errorsForClient.size()) {
- throw new DistributedUpdatesAsyncException(errorsForClient);
- }
}
-
+
// must be synchronized by bucket
private void doLocalAdd(AddUpdateCommand cmd) throws IOException {
super.processAdd(cmd);
@@ -1455,6 +1446,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
zkController.getBaseUrl(), req.getCore().getName()));
if (req.getParams().get(UpdateRequest.MIN_REPFACT) != null) {
+ // TODO: Kept for rolling upgrades only. Remove in Solr 9
params.add(UpdateRequest.MIN_REPFACT, req.getParams().get(UpdateRequest.MIN_REPFACT));
}
cmdDistrib.distribDelete(cmd, nodes, params, false, rollupReplicationTracker, leaderReplicationTracker);
@@ -1520,8 +1512,8 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
? zkController.getClusterState().getCollection(collection) : null;
if (zkEnabled && DistribPhase.NONE == phase) {
- if (req.getParams().get(UpdateRequest.MIN_REPFACT) != null && rollupReplicationTracker == null) {
- rollupReplicationTracker = new RollupRequestReplicationTracker(req.getParams().get(UpdateRequest.MIN_REPFACT));
+ if (rollupReplicationTracker == null) {
+ rollupReplicationTracker = new RollupRequestReplicationTracker();
}
boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard
@@ -1564,10 +1556,8 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
outParams.remove("commit"); // this will be distributed from the local commit
- if (req.getParams().get(UpdateRequest.MIN_REPFACT) != null) {
- // If we've determined that there are no docs on this shard that need to be deleted, then we don't send
- // sub-requests to any other replicas for this shard. In this case, min_rf is meaningless for this shard
- // so flag that in replicationTracker
+ if (params.get(UpdateRequest.MIN_REPFACT) != null) {
+ // TODO: Kept this for rolling upgrades. Remove in Solr 9
outParams.add(UpdateRequest.MIN_REPFACT, req.getParams().get(UpdateRequest.MIN_REPFACT));
}
cmdDistrib.distribDelete(cmd, leaders, outParams, false, rollupReplicationTracker, null);
@@ -1580,10 +1570,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
phase = DistribPhase.TOLEADER;
}
- // check if client has requested minimum replication factor information. will set replicationTracker to null if
- // we aren't the leader or subShardLeader
- checkReplicationTracker(cmd);
-
List<Node> replicas = null;
if (zkEnabled && DistribPhase.TOLEADER == phase) {
@@ -1593,6 +1579,11 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
} else if (DistribPhase.FROMLEADER == phase) {
isLeader = false;
}
+
+
+ // check if client has requested minimum replication factor information. will set replicationTracker to null if
+ // we aren't the leader or subShardLeader
+ checkReplicationTracker(cmd);
if (vinfo == null) {
super.processDelete(cmd);
@@ -2094,7 +2085,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// lifetime of the batch. The leader for each shard keeps track of it's own achieved replicaiton for its shard
// and attaches that to the response to the originating node (i.e. the one with the RollupReplicationTracker).
// Followers in general do not need a tracker of any sort with the sole exception of the RollupReplicationTracker
- // allocated on the original node that recieves the top-level request.
+ // allocated on the original node that receives the top-level request.
//
// DeleteById is tricky. Since the docs are sent one at a time, there has to be some fancy dancing. In the
// deleteById case, here are the rules:
@@ -2117,24 +2108,11 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
public static class RollupRequestReplicationTracker {
private int achievedRf = Integer.MAX_VALUE;
- private final int requestedRf;
-
- public RollupRequestReplicationTracker(String minRepFact) {
- try {
- this.requestedRf = Integer.parseInt(minRepFact);
- } catch (NumberFormatException nfe) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "MinRF must be an integer, was " + minRepFact);
- }
- }
public int getAchievedRf() {
return achievedRf;
}
- public int getRequestedRf() {
- return requestedRf;
- }
-
// We want to report only the minimun _ever_ achieved...
public void testAndSetAchievedRf(int rf) {
this.achievedRf = Math.min(this.achievedRf, rf);
@@ -2142,8 +2120,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
public String toString() {
StringBuilder sb = new StringBuilder("RollupRequestReplicationTracker")
- .append(", requestedRf: ")
- .append(requestedRf)
.append(" achievedRf: ")
.append(achievedRf);
return sb.toString();
@@ -2164,16 +2140,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// Since we only allocate one of these on the leader and, by definition, the leader has been found and is running,
// we have a replication factor of one by default.
private int achievedRf = 1;
- private final int requestedRf;
private final String myShardId;
- int getRequestedRf() {
- return requestedRf;
- }
-
- public LeaderRequestReplicationTracker(String shardId, int requestedRf) {
- this.requestedRf = requestedRf;
+ public LeaderRequestReplicationTracker(String shardId) {
this.myShardId = shardId;
}
@@ -2197,9 +2167,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
sb.append(", achievedRf=")
.append(getAchievedRf())
.append(" for shard ")
- .append(myShardId)
- .append(" requested replication factor: ")
- .append(requestedRf);
+ .append(myShardId);
return sb.toString();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
index f0477ab..15fdb3b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
@@ -16,6 +16,9 @@
*/
package org.apache.solr.cloud;
+import static org.apache.solr.common.cloud.Replica.State.DOWN;
+import static org.apache.solr.common.cloud.Replica.State.RECOVERING;
+
import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@@ -31,7 +34,6 @@ import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.JSONTestUtil;
@@ -41,7 +43,6 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
@@ -66,9 +67,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.cloud.Replica.State.DOWN;
-import static org.apache.solr.common.cloud.Replica.State.RECOVERING;
-
/**
* Simulates HTTP partitions between a leader and replica but the replica does
* not lose its ZooKeeper connection.
@@ -135,12 +133,6 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
testDoRecoveryOnRestart();
- // Tests that if we set a minRf that's not satisfied, no recovery is requested, but if minRf is satisfied,
- // recovery is requested
- testMinRf();
-
- waitForThingsToLevelOut(30000);
-
// test a 1x2 collection
testRf2();
@@ -267,95 +259,6 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
attemptCollectionDelete(cloudClient, testCollectionName);
}
- protected void testMinRf() throws Exception {
- // create a collection that has 1 shard and 3 replicas
- String testCollectionName = "collMinRf_1x3";
- createCollection(testCollectionName, "conf1", 1, 3, 1);
- cloudClient.setDefaultCollection(testCollectionName);
-
- // term of the core still be watched even when the core is reloaded
- CollectionAdminRequest.reloadCollection(testCollectionName).process(cloudClient);
-
- sendDoc(1, 2);
-
- JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(getShardLeader(testCollectionName, "shard1", 1000)));
- List<Replica> notLeaders =
- ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, maxWaitSecsToSeeAllActive);
- assertTrue("Expected 2 non-leader replicas for collection " + testCollectionName
- + " but found " + notLeaders.size() + "; clusterState: "
- + printClusterStateInfo(testCollectionName),
- notLeaders.size() == 2);
-
- assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 1);
-
- // Now introduce a network partition between the leader and 1 replica, so a minRf of 2 is still achieved
- log.info("partitioning replica : " + notLeaders.get(0));
- SocketProxy proxy0 = getProxyForReplica(notLeaders.get(0));
- SocketProxy leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));
-
- proxy0.close();
- // leader still can connect to replica 2, by closing leaderProxy, replica 1 can not do recovery
- leaderProxy.close();
-
- // indexing during a partition
- int achievedRf = sendDoc(2, 2, leaderJetty);
- assertEquals("Unexpected achieved replication factor", 2, achievedRf);
- try (ZkShardTerms zkShardTerms = new ZkShardTerms(testCollectionName, "shard1", cloudClient.getZkStateReader().getZkClient())) {
- assertFalse(zkShardTerms.canBecomeLeader(notLeaders.get(0).getName()));
- }
- Thread.sleep(sleepMsBeforeHealPartition);
- proxy0.reopen();
- leaderProxy.reopen();
-
- notLeaders =
- ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, maxWaitSecsToSeeAllActive);
-
- // Since minRf is achieved, we expect recovery, so we expect seeing 2 documents
- assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 2);
-
- // Now introduce a network partition between the leader and both of its replicas, so a minRf of 2 is NOT achieved
- proxy0 = getProxyForReplica(notLeaders.get(0));
- proxy0.close();
- SocketProxy proxy1 = getProxyForReplica(notLeaders.get(1));
- proxy1.close();
- leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));
- leaderProxy.close();
-
- achievedRf = sendDoc(3, 2, leaderJetty);
- assertEquals("Unexpected achieved replication factor", 1, achievedRf);
-
- Thread.sleep(sleepMsBeforeHealPartition);
-
- // Verify that the partitioned replicas are NOT DOWN since minRf wasn't achieved
- ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, 1);
-
- proxy0.reopen();
- proxy1.reopen();
- leaderProxy.reopen();
-
- notLeaders =
- ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, maxWaitSecsToSeeAllActive);
-
- // Check that doc 3 is on the leader but not on the notLeaders
- Replica leader = cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, "shard1", 10000);
- try (HttpSolrClient leaderSolr = getHttpSolrClient(leader, testCollectionName)) {
- assertDocExists(leaderSolr, testCollectionName, "3");
- }
-
- for (Replica notLeader : notLeaders) {
- try (HttpSolrClient notLeaderSolr = getHttpSolrClient(notLeader, testCollectionName)) {
- assertDocNotExists(notLeaderSolr, testCollectionName, "3");
- }
- }
-
- // Retry sending doc 3
- achievedRf = sendDoc(3, 2);
- assertEquals("Unexpected achieved replication factor", 3, achievedRf);
-
- // Now doc 3 should be on all replicas
- assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 3);
- }
-
protected void testRf2() throws Exception {
// create a collection that has 1 shard but 2 replicas
String testCollectionName = "c8n_1x2";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java
index 2a2bcc3..c4135b5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ReplicationFactorTest.java
@@ -102,7 +102,6 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
int maxShardsPerNode = 1;
String testCollectionName = "repfacttest_c8n_2x2";
String shardId = "shard1";
- int minRf = 2;
createCollectionWithRetry(testCollectionName, "conf1", numShards, replicationFactor, maxShardsPerNode);
@@ -122,7 +121,7 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
// send directly to the leader using HttpSolrServer instead of CloudSolrServer (to test support for non-direct updates)
UpdateRequest up = new UpdateRequest();
- up.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(minRf));
+ maybeAddMinRfExplicitly(2, up);
up.add(batch);
Replica leader = cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, shardId);
@@ -230,13 +229,13 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
// Delete the docs by ID indicated
UpdateRequest req = new UpdateRequest();
req.deleteById(byIdsList);
- req.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(expectedRfByIds));
+ maybeAddMinRfExplicitly(expectedRfByIds, req);
sendNonDirectUpdateRequestReplicaWithRetry(rep, req, expectedRfByIds, coll);
//Delete the docs by query indicated.
req = new UpdateRequest();
req.deleteByQuery("id:(" + StringUtils.join(byQueriesSet, " OR ") + ")");
- req.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(expectedRfDBQ));
+ maybeAddMinRfExplicitly(expectedRfDBQ, req);
sendNonDirectUpdateRequestReplicaWithRetry(rep, req, expectedRfDBQ, coll);
}
@@ -261,6 +260,12 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
// Note that this also tests if we're wonky and return an achieved rf greater than the number of live replicas.
assertTrue("Expected rf="+expectedRf+" for batch but got "+
batchRf + "; clusterState: " + printClusterStateInfo(), batchRf == expectedRf);
+ if (up.getParams() != null && up.getParams().get(UpdateRequest.MIN_REPFACT) != null) {
+ // If "min_rf" was specified in the request, it must be in the response for back compatibility
+ assertNotNull("Expecting min_rf to be in the response, since it was explicitly set in the request", hdr.get(UpdateRequest.MIN_REPFACT));
+ assertEquals("Unexpected min_rf in the response",
+ Integer.parseInt(up.getParams().get(UpdateRequest.MIN_REPFACT)), hdr.get(UpdateRequest.MIN_REPFACT));
+ }
}
}
@@ -282,7 +287,7 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
int rf = sendDoc(1, minRf);
assertRf(3, "all replicas should be active", rf);
- // Uses cloudClient do do it's work
+ // Uses cloudClient to do it's work
doDBIdWithRetry(3, 5, "deletes should have propagated to all 3 replicas", 1);
doDBQWithRetry(3, 5, "deletes should have propagated to all 3 replicas", 1);
@@ -292,7 +297,7 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
rf = sendDoc(2, minRf);
assertRf(2, "one replica should be down", rf);
- // Uses cloudClient do do it's work
+ // Uses cloudClient to do it's work
doDBQWithRetry(2, 5, "deletes should have propagated to 2 replicas", 1);
doDBIdWithRetry(2, 5, "deletes should have propagated to 2 replicas", 1);
@@ -398,8 +403,8 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
addDocs(docIds, expectedRf, retries);
UpdateRequest req = new UpdateRequest();
req.deleteByQuery("id:(" + StringUtils.join(docIds, " OR ") + ")");
- req.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(expectedRf));
- doDelete(req, msg, expectedRf, retries);
+ boolean minRfExplicit = maybeAddMinRfExplicitly(expectedRf, req);
+ doDelete(req, msg, expectedRf, retries, minRfExplicit);
}
protected void doDBIdWithRetry(int expectedRf, int retries, String msg, int docsToAdd) throws Exception {
@@ -407,14 +412,18 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
addDocs(docIds, expectedRf, retries);
UpdateRequest req = new UpdateRequest();
req.deleteById(StringUtils.join(docIds, ","));
- req.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(expectedRf));
- doDelete(req, msg, expectedRf, retries);
+ boolean minRfExplicit = maybeAddMinRfExplicitly(expectedRf, req);
+ doDelete(req, msg, expectedRf, retries, minRfExplicit);
}
- protected void doDelete(UpdateRequest req, String msg, int expectedRf, int retries) throws IOException, SolrServerException, InterruptedException {
+ protected void doDelete(UpdateRequest req, String msg, int expectedRf, int retries, boolean minRfExplicit) throws IOException, SolrServerException, InterruptedException {
int achievedRf = -1;
for (int idx = 0; idx < retries; ++idx) {
- achievedRf = cloudClient.getMinAchievedReplicationFactor(cloudClient.getDefaultCollection(), cloudClient.request(req));
+ NamedList<Object> response = cloudClient.request(req);
+ if (minRfExplicit) {
+ assertMinRfInResponse(expectedRf, response);
+ }
+ achievedRf = cloudClient.getMinAchievedReplicationFactor(cloudClient.getDefaultCollection(), response);
if (achievedRf == expectedRf) return;
Thread.sleep(1000);
}
@@ -423,12 +432,36 @@ public class ReplicationFactorTest extends AbstractFullDistribZkTestBase {
protected int sendDoc(int docId, int minRf) throws Exception {
UpdateRequest up = new UpdateRequest();
- up.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(minRf));
+ boolean minRfExplicit = maybeAddMinRfExplicitly(minRf, up);
SolrInputDocument doc = new SolrInputDocument();
doc.addField(id, String.valueOf(docId));
doc.addField("a_t", "hello" + docId);
up.add(doc);
- return cloudClient.getMinAchievedReplicationFactor(cloudClient.getDefaultCollection(), cloudClient.request(up));
+ return runAndGetAchievedRf(up, minRfExplicit, minRf);
+ }
+
+ private int runAndGetAchievedRf(UpdateRequest up, boolean minRfExplicit, int minRf) throws SolrServerException, IOException {
+ NamedList<Object> response = cloudClient.request(up);
+ if (minRfExplicit) {
+ assertMinRfInResponse(minRf, response);
+ }
+ return cloudClient.getMinAchievedReplicationFactor(cloudClient.getDefaultCollection(), response);
+ }
+
+ private void assertMinRfInResponse(int minRf, NamedList<Object> response) {
+ Object minRfFromResponse = response.findRecursive("responseHeader", UpdateRequest.MIN_REPFACT);
+ assertNotNull("Expected min_rf header in the response", minRfFromResponse);
+ assertEquals("Unexpected min_rf in response", ((Integer)minRfFromResponse).intValue(), minRf);
+ }
+
+ private boolean maybeAddMinRfExplicitly(int minRf, UpdateRequest up) {
+ boolean minRfExplicit = false;
+ if (rarely()) {
+ // test back compat behavior. Remove in Solr 9
+ up.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(minRf));
+ minRfExplicit = true;
+ }
+ return minRfExplicit;
}
protected void assertRf(int expected, String explain, int actual) throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
index 24cf717..9202be8 100644
--- a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
@@ -461,8 +461,8 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
AddUpdateCommand cmd = new AddUpdateCommand(null);
cmd.solrDoc = sdoc("id", id.incrementAndGet());
ModifiableSolrParams params = new ModifiableSolrParams();
- RollupRequestReplicationTracker rollupReqTracker = new RollupRequestReplicationTracker("2");
- LeaderRequestReplicationTracker leaderReqTracker = new LeaderRequestReplicationTracker("shard1", 2);
+ RollupRequestReplicationTracker rollupReqTracker = new RollupRequestReplicationTracker();
+ LeaderRequestReplicationTracker leaderReqTracker = new LeaderRequestReplicationTracker("shard1");
cmdDistrib.distribAdd(cmd, nodes, params, false, rollupReqTracker, leaderReqTracker);
cmdDistrib.finish();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc b/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc
index c121d4e..491861a 100644
--- a/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc
@@ -36,6 +36,8 @@ If an update fails because cores are reloading schemas and some have finished bu
When using a replication factor greater than one, an update request may succeed on the shard leader but fail on one or more of the replicas. For instance, consider a collection with one shard and a replication factor of three. In this case, you have a shard leader and two additional replicas. If an update request succeeds on the leader but fails on both replicas, for whatever reason, the update request is still considered successful from the perspective of the client. The replicas that missed the update will sync with the leader when they recover.
-Behind the scenes, this means that Solr has accepted updates that are only on one of the nodes (the current leader). Solr supports the optional `min_rf` parameter on update requests that cause the server to return the achieved replication factor for an update request in the response. For the example scenario described above, if the client application included `min_rf >= 1`, then Solr would return `rf=1` in the Solr response header because the request only succeeded on the leader. The update request will still be accepted as the `min_rf` parameter only tells Solr that the client application wishes to know what the achieved replication factor was for the update request. In other words, `min_rf` does not mean Solr will enforce a minimum replication factor as Solr does not support rolling back updates that succeed on a subset of replicas.
+Behind the scenes, this means that Solr has accepted updates that are only on one of the nodes (the current leader). To make the client aware of this, Solr includes in the response header the "Achieved Replication Factor" (`rf`). The achieved replication factor is the number of replicas of the shard that actually received the update request (including the leader), in the above example, 1. In the case of multi-shard update requests, the achieved replication factor is the minimum achieved replication factor across all shards.
-On the client side, if the achieved replication factor is less than the acceptable level, then the client application can take additional measures to handle the degraded state. For instance, a client application may want to keep a log of which update requests were sent while the state of the collection was degraded and then resend the updates once the problem has been resolved. In short, `min_rf` is an optional mechanism for a client application to be warned that an update request was accepted while the collection is in a degraded state.
+On the client side, if the achieved replication factor is less than the acceptable level, then the client application can take additional measures to handle the degraded state. For instance, a client application may want to keep a log of which update requests were sent while the state of the collection was degraded and then resend the updates once the problem has been resolved.
+
+NOTE: In previous version of Solr, the `min_rf` parameter had to be specified to ask Solr for the achieved replication factor. Now it is always included in the response.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java
index d4da965..8a142bf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java
@@ -54,6 +54,10 @@ import static org.apache.solr.common.params.ShardParams._ROUTE_;
public class UpdateRequest extends AbstractUpdateRequest {
public static final String REPFACT = "rf";
+ /**
+ * @deprecated Solr now always includes in the response the {@link #REPFACT}, this parameter
+ * doesn't need to be explicitly set
+ */
public static final String MIN_REPFACT = "min_rf";
public static final String VER = "ver";
public static final String OVERWRITE = "ow";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
index e9428ca..78c54db 100644
--- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
@@ -922,6 +922,8 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
protected void compareSolrResponses(SolrResponse a, SolrResponse b) {
// SOLR-3345: Checking QTime value can be skipped as there is no guarantee that the numbers will match.
handle.put("QTime", SKIPVAL);
+ // rf will be different since the control collection doesn't usually have multiple replicas
+ handle.put("rf", SKIPVAL);
String cmp = compare(a.getResponse(), b.getResponse(), flags, handle);
if (cmp != null) {
log.error("Mismatched responses:\n" + a + "\n" + b);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/46f753d7/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index fc645c7..9d0e4bf 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -794,7 +794,6 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
@SuppressWarnings("rawtypes")
protected static int sendDocsWithRetry(CloudSolrClient cloudClient, String collection, List<SolrInputDocument> batch, int minRf, int maxRetries, int waitBeforeRetry) throws Exception {
UpdateRequest up = new UpdateRequest();
- up.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(minRf));
up.add(batch);
NamedList resp = null;
int numRetries = 0;