You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2019/06/10 07:14:14 UTC
[lucene-solr] 01/01: SOLR-13329: change put: on-each to put:
on-each-node
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/SOLR-13229
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 23f019be169faa5659babcc41cd6f083eaa0f2c1
Author: Noble Paul <no...@apache.org>
AuthorDate: Mon Jun 10 17:13:42 2019 +1000
SOLR-13329: change put: on-each to put: on-each-node
---
.../src/solrcloud-autoscaling-api.adoc | 2 +-
.../solrcloud-autoscaling-policy-preferences.adoc | 8 +--
.../client/solrj/cloud/autoscaling/Clause.java | 60 +++++++++++++++++++---
.../solrj/cloud/autoscaling/ReplicaVariable.java | 17 +++++-
.../client/solrj/cloud/autoscaling/TestPolicy.java | 31 ++++++++---
5 files changed, 99 insertions(+), 19 deletions(-)
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc
index e8203ec..ed51b1d 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-api.adoc
@@ -334,7 +334,7 @@ If there is no autoscaling policy configured or if you wish to use a configurati
----
curl -X POST -H 'Content-type:application/json' -d '{
"cluster-policy": [
- {"replica": 0, "put" : "on-each", "nodeset": {"port" : "7574"}}
+ {"replica": 0, "put" : "on-each-node", "nodeset": {"port" : "7574"}}
]
}' http://localhost:8983/solr/admin/autoscaling/suggestions?omitHeader=true
----
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
index e5a0b09..8262bda 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
@@ -115,7 +115,7 @@ Per-collection rules have five parts:
* <<Replica Selector and Rule Evaluation Context>>
* <<Replica Count Constraint>> (`"replica": "..."`)
* <<Rule Strictness>> (optional)
-* `put` (optional) specifies how to place these replicas on the selected nodes. All the selected nodes are considered as one bucket by default. `"put" : "on-each"` treats each selected node as a bucket
+* `put` (optional) specifies how to place these replicas on the selected nodes. All the selected nodes are considered as one bucket by default. `"put" : "on-each-node"` treats each selected node as a bucket
==== Node Selector
@@ -140,11 +140,11 @@ when using the `nodeset` attribute, an optional attribute `put` can be used to s
example: _put one replica on each node with a system property zone=east_
[source,json]
-{ "replica":1, "put" :"on-each", "nodeset":{"sysprop.zone":"east"}}
+{ "replica":1, "put" :"on-each-node", "nodeset":{"sysprop.zone":"east"}}
example: _put a total of 2 replicas on the set of nodes with property zone=east_
[source,json]
-{ "replica":2, "put" :"on-each" "nodeset":{"sysprop.zone":"east"}}
+{ "replica":2, "put" :"on-each-node" "nodeset":{"sysprop.zone":"east"}}
@@ -392,7 +392,7 @@ For <<each-function,each>> shard of each collection, distribute replicas equally
Do not place any replica on any node that has the overseer role. Note that the role is added by the `addRole` collection API. It is *not* automatically the node which is currently the overseer.
[source,json]
-{"replica": 0, "put" :"on-each", "nodeset":{ "nodeRole": "overseer"}}
+{"replica": 0, "put" :"on-each-node", "nodeset":{ "nodeRole": "overseer"}}
==== Place Replicas Based on Free Disk
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index adfbfcb..f89b28a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -18,6 +18,7 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -150,14 +151,12 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (!m.containsKey(NODESET)) return false;
Object o = m.get(NODESET);
if (o instanceof Map) {
- Map map = (Map) o;
- if (map.size() != 1) {
- throwExp(m, "nodeset must only have one and only one key");
- }
- String key = (String) map.keySet().iterator().next();
+ String key = validateObjectInNodeset(m, (Map) o);
parseCondition(key, o, m);
} else if (o instanceof List) {
List l = (List) o;
+ if(l.size()<2) throwExp(m, "nodeset [] must have atleast 2 items");
+ if( checkMapArray(l, m)) return true;
for (Object it : l) {
if (it instanceof String) continue;
else throwExp(m, "nodeset :[]must have only string values");
@@ -169,6 +168,46 @@ public class Clause implements MapWriter, Comparable<Clause> {
return true;
}
+ private String validateObjectInNodeset(Map<String, Object> m, Map map) {
+ if (map.size() != 1) {
+ throwExp(m, "nodeset must only have one and only one key");
+ }
+ String key = (String) map.keySet().iterator().next();
+ Object val = map.get(key);
+ if(val instanceof String && ((String )val).trim().charAt(0) == '#'){
+ throwExp(m, formatString("computed value {0} not allowed in nodeset", val));
+ }
+ return key;
+ }
+
+ private boolean checkMapArray(List l, Map<String, Object> m) {
+ List<Map> maps = null;
+ for (Object o : l) {
+ if (o instanceof Map) {
+ if (maps == null) maps = new ArrayList<>();
+ maps.add((Map) o);
+ }
+ }
+ String key = null;
+ if (maps != null) {
+ if (maps.size() != l.size()) throwExp(m, "all elements of nodeset must be Objects");
+ List<Condition> tags = new ArrayList<>(maps.size());
+ for (Map map : maps) {
+ String s = validateObjectInNodeset(m, map);
+ if(key == null) key = s;
+ if(!Objects.equals(key, s)){
+ throwExp(m, "all element must have same key");
+ }
+ tags.add(parse(s, m));
+ }
+ if(this.put == Put.ON_EACH) throwExp(m, "cannot use put: ''on-each-node'' with an array value in nodeset ");
+ this.tag = new Condition(key, tags,Operand.IN, null,this);
+ return true;
+ }
+ return false;
+
+ }
+
public Condition getThirdTag() {
return globalTag == null ? tag : globalTag;
}
@@ -468,6 +507,15 @@ public class Clause implements MapWriter, Comparable<Clause> {
private Set getUniqueTags(Policy.Session session, ComputedValueEvaluator eval) {
Set tags = new HashSet();
+
+ if(nodeSetPresent){
+ if(tag.val instanceof List && ((List)tag.val).get(0) instanceof Condition){
+ tags.addAll((List)tag.val);
+ } else {
+ tags.add(tag);
+ }
+ return tags;
+ }
if(tag.op == WILDCARD){
for (Row row : session.matrix) {
eval.node = row.node;
@@ -751,7 +799,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
public static final String METRICS_PREFIX = "metrics:";
enum Put {
- ON_ALL("on-all"), ON_EACH("on-each");
+ ON_ALL(""), ON_EACH("on-each-node");
public final String val;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java
index f0bfafa..675382a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaVariable.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import org.apache.solr.common.util.StrUtils;
@@ -90,15 +91,29 @@ class ReplicaVariable extends VariableBase {
@Override
public String postValidate(Condition condition) {
+ Object val = condition.clause.getThirdTag().val;
+ boolean isNodesetObjectList = condition.clause.nodeSetPresent && (val instanceof List) && ((List)val).get(0) instanceof Condition ;
+ if(condition.clause.nodeSetPresent ){
+ if(condition.computedType == ComputedType.EQUAL){
+ if(!isNodesetObjectList) return " 'nodeset' must have an array value when 'replica': '#EQUAL` is used";
+ } else {
+ if(isNodesetObjectList){
+ return "cannot use array value for nodeset if replica : '#EQUAL' is not used";
+ }
+
+ }
+
+ }
+
if (condition.computedType == ComputedType.EQUAL) {
if (condition.getClause().tag != null &&
-// condition.getClause().tag.varType == NODE &&
(condition.getClause().tag.op == Operand.WILDCARD || condition.getClause().tag.op == Operand.IN)) {
return null;
} else {
return "'replica': '#EQUAL` must be used with 'node':'#ANY'";
}
} else if (condition.computedType == ComputedType.ALL) {
+ if(isNodesetObjectList) return "replica: '#ALL' cannot be used with a list of values in nodeset";
if (condition.getClause().tag != null && (condition.getClause().getTag().op == Operand.IN ||
condition.getClause().getTag().op == Operand.WILDCARD)) {
return StrUtils.formatString("array value or wild card cannot be used for tag {0} with replica : '#ALL'",
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index dacfc37..a793995 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -721,11 +721,28 @@ public class TestPolicy extends SolrTestCaseJ4 {
clause = Clause.create("{'replica': '#ALL', 'nodeset': {'freedisk': '>700'}, 'strict': false}");
assertEquals(clause.put, Clause.Put.ON_ALL);
assertEquals(Operand.GREATER_THAN , clause.tag.op);
- clause = Clause.create("{'replica': '#ALL', put: on-each, 'nodeset': {sysprop.zone : east}}");
+ clause = Clause.create("{'replica': '#ALL', put: on-each-node, 'nodeset': {sysprop.zone : east}}");
assertEquals(clause.put, Clause.Put.ON_EACH);
exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{'replica': '#ALL', put: on-Each, 'nodeset': {sysprop.zone : east}}"));
assertTrue(exp.getMessage().contains("invalid value for put : on-Each"));
+ clause = Clause.create("{replica : '#EQUAL', nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}");
+ assertTrue(((List)clause.tag.val).get(0) instanceof Condition);
+ assertTrue( Utils.fromJSONString(Utils.toJSONString(clause)) instanceof Map);
+ exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', nodeset : [{sysprop.zone : east}, {port : '8983'}]}"));
+ assertTrue(exp.getMessage().contains("all element must have same key"));
+ exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', nodeset : [{sysprop.zone : east}, {sysprop.zone : '#EACH'}]}"));
+ assertTrue(exp.getMessage().contains("computed value #EACH not allowed in nodeset"));
+ exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', nodeset : {sysprop.zone : east}}"));
+ assertTrue(exp.getMessage().contains("'nodeset' must have an array value when 'replica': '#EQUAL` is used"));
+ exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#ALL', nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}"));
+ assertTrue(exp.getMessage().contains("cannot use array value for nodeset if replica : '#EQUAL' is not used"));
+ exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '50%', nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}"));
+ assertTrue(exp.getMessage().contains("cannot use array value for nodeset if replica : '#EQUAL' is not used"));
+ exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : 3, nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}"));
+ assertTrue(exp.getMessage().contains("cannot use array value for nodeset if replica : '#EQUAL' is not used"));
+ exp = expectThrows(IllegalArgumentException.class, ()-> Clause.create("{replica : '#EQUAL', put: on-each-node, nodeset : [{sysprop.zone : east}, {sysprop.zone : west}]}"));
+ assertTrue(exp.getMessage().contains("cannot use put: 'on-each-node' with an array value in nodeset "));
}
@@ -1246,7 +1263,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
" { 'replica': 0, nodeset : {'nodeRole': 'overseer'}}" +
" { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
" { 'replica': 0, 'shard': '#EACH', nodeset : { sysprop.fs : '!ssd'}, type : TLOG }" +
- " { 'replica': 0, 'shard': '#EACH', put:'on-each' nodeset : {sysprop.fs : '!slowdisk'} , type : PULL }" +
+ " { 'replica': 0, 'shard': '#EACH', put:'on-each-node' nodeset : {sysprop.fs : '!slowdisk'} , type : PULL }" +
" ]" +
"}");
@@ -1371,8 +1388,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
" { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl'}," +
" { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl2', type : PULL}," +
" { 'replica': '<3', 'shard': '#EACH', 'node': '#ANY', 'collection':'newColl2'}," +
- " { 'replica': 0, 'shard': '#EACH', put: on-each , nodeset:{ sysprop.fs : '!ssd'}, type : TLOG }" +
- " { 'replica': 0, 'shard': '#EACH', put: on-each ,nodeset : {sysprop.fs : '!slowdisk'} , type : PULL }" +
+ " { 'replica': 0, 'shard': '#EACH', put: on-each-node , nodeset:{ sysprop.fs : '!ssd'}, type : TLOG }" +
+ " { 'replica': 0, 'shard': '#EACH', put: on-each-node ,nodeset : {sysprop.fs : '!slowdisk'} , type : PULL }" +
" ]" +
"}");
@@ -2394,8 +2411,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
" cluster-preferences :[{ minimize : cores, precision : 2 }]}";
if(useNodeset){
autoScalingjson = " { cluster-policy:[" +
- " { replica :'0', put:on-each , nodeset:{ freedisk:'<1000'}}," +
- " { replica :0, put : on-each , nodeset : {nodeRole : overseer}}]," +
+ " { replica :'0', put:on-each-node , nodeset:{ freedisk:'<1000'}}," +
+ " { replica :0, put : on-each-node , nodeset : {nodeRole : overseer}}]," +
" cluster-preferences :[{ minimize : cores, precision : 2 }]}";
}
AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
@@ -2427,7 +2444,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
if(useNodeset){
autoScalingjson = " { cluster-policy:[" +
" { replica :'#ALL', nodeset:{ freedisk:'>1000'}}," +
- " { replica :0 , put: on-each , nodeset : {nodeRole : overseer}}]," +
+ " { replica :0 , put: on-each-node , nodeset : {nodeRole : overseer}}]," +
" cluster-preferences :[{ minimize : cores, precision : 2 }]}";
}
cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));