You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/09/20 08:47:05 UTC
[49/50] lucene-solr:jira/solr-11285: Merge branch
'feature/autoscaling_72' into jira/solr-11285
Merge branch 'feature/autoscaling_72' into jira/solr-11285
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/877d9227
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/877d9227
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/877d9227
Branch: refs/heads/jira/solr-11285
Commit: 877d9227294367cfeca064576ea9bbdffbce5bc6
Parents: 0a834f2 f28904b
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Sep 19 18:08:09 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Sep 19 18:08:09 2017 +0200
----------------------------------------------------------------------
build.xml | 18 +-
dev-tools/doap/lucene.rdf | 9 +-
dev-tools/doap/solr.rdf | 7 +
lucene/CHANGES.txt | 20 +
lucene/NOTICE.txt | 5 +-
lucene/analysis/build.xml | 4 +
.../lucene/analysis/bn/BengaliAnalyzer.java | 132 +++
.../analysis/bn/BengaliNormalizationFilter.java | 59 +
.../bn/BengaliNormalizationFilterFactory.java | 55 +
.../lucene/analysis/bn/BengaliNormalizer.java | 155 +++
.../lucene/analysis/bn/BengaliStemFilter.java | 49 +
.../analysis/bn/BengaliStemFilterFactory.java | 48 +
.../lucene/analysis/bn/BengaliStemmer.java | 183 ++++
.../apache/lucene/analysis/bn/package-info.java | 21 +
...ache.lucene.analysis.util.TokenFilterFactory | 2 +
.../org/apache/lucene/analysis/bn/stopwords.txt | 121 +++
.../lucene/analysis/bn/TestBengaliAnalyzer.java | 53 +
.../lucene/analysis/bn/TestBengaliFilters.java | 80 ++
.../analysis/bn/TestBengaliNormalizer.java | 110 ++
.../lucene/analysis/bn/TestBengaliStemmer.java | 79 ++
.../analysis/icu/ICUNormalizer2CharFilter.java | 35 +-
.../icu/TestICUNormalizer2CharFilter.java | 21 +
.../index/TestBackwardsCompatibility.java | 4 +-
.../lucene/index/unsupported.6.6.1-cfs.zip | Bin 0 -> 15836 bytes
.../lucene/index/unsupported.6.6.1-nocfs.zip | Bin 0 -> 15830 bytes
lucene/build.xml | 13 +
lucene/codecs/build.xml | 2 +
lucene/common-build.xml | 53 +-
lucene/core/build.xml | 2 +
.../lucene/index/DefaultIndexingChain.java | 2 -
.../lucene/store/NativeFSLockFactory.java | 14 +-
.../lucene/store/TestNativeFSLockFactory.java | 42 +
.../writercache/LruTaxonomyWriterCache.java | 20 +-
.../writercache/TestLruTaxonomyWriterCache.java | 50 +
lucene/ivy-versions.properties | 2 +-
.../lucene/spatial3d/geom/GeoBBoxFactory.java | 9 +-
.../spatial3d/geom/GeoDegeneratePath.java | 788 ++++++++++++++
.../spatial3d/geom/GeoDegeneratePoint.java | 2 +-
.../apache/lucene/spatial3d/geom/GeoPath.java | 58 +
.../lucene/spatial3d/geom/GeoPathFactory.java | 3 +
.../lucene/spatial3d/geom/GeoStandardPath.java | 170 +++
.../lucene/spatial3d/geom/StandardObjects.java | 1 +
.../lucene/spatial3d/geom/GeoPathTest.java | 75 ++
.../spatial3d/geom/RandomBinaryCodecTest.java | 10 +-
.../geom/RandomGeo3dShapeGenerator.java | 1013 ++++++++++++++++++
.../spatial3d/geom/RandomGeoShapeGenerator.java | 944 ----------------
.../geom/RandomGeoShapeRelationshipTest.java | 65 +-
.../geom/SimpleGeoPolygonRelationshipsTest.java | 24 +
.../suggest/analyzing/AnalyzingSuggester.java | 6 +-
.../analyzing/AnalyzingSuggesterTest.java | 60 ++
solr/CHANGES.txt | 34 +-
solr/build.xml | 16 +-
.../apache/solr/ltr/model/LTRScoringModel.java | 4 +-
.../org/apache/solr/ltr/TestRerankBase.java | 12 +-
solr/core/build.xml | 7 +
.../solr/cloud/autoscaling/AutoScaling.java | 2 +
.../cloud/autoscaling/ComputePlanAction.java | 50 +
.../cloud/autoscaling/NodeAddedTrigger.java | 3 +-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 7 +-
.../cloud/autoscaling/SearchRateTrigger.java | 275 +++++
.../solr/cloud/autoscaling/TriggerBase.java | 13 +-
.../org/apache/solr/handler/StreamHandler.java | 107 +-
.../solr/handler/admin/CollectionsHandler.java | 14 +-
.../solr/handler/admin/MetricsHandler.java | 11 +-
.../solr/handler/component/QueryComponent.java | 460 ++++----
.../org/apache/solr/parser/FastCharStream.java | 7 +-
.../org/apache/solr/parser/QueryParser.java | 2 +-
.../apache/solr/schema/AbstractEnumField.java | 11 +-
.../org/apache/solr/schema/DatePointField.java | 15 +-
.../org/apache/solr/schema/DateRangeField.java | 4 +-
.../apache/solr/schema/DoublePointField.java | 15 +-
.../java/org/apache/solr/schema/FieldType.java | 65 +-
.../org/apache/solr/schema/FloatPointField.java | 15 +-
.../org/apache/solr/schema/IntPointField.java | 15 +-
.../org/apache/solr/schema/LongPointField.java | 15 +-
.../apache/solr/schema/PreAnalyzedField.java | 6 +-
.../java/org/apache/solr/schema/TextField.java | 5 +-
.../java/org/apache/solr/schema/TrieField.java | 44 +-
.../org/apache/solr/search/FunctionQParser.java | 11 +-
.../apache/solr/search/LuceneQParserPlugin.java | 54 -
.../solr/search/OldLuceneQParserPlugin.java | 37 -
.../java/org/apache/solr/search/Sorting.java | 12 +-
.../apache/solr/search/ValueSourceParser.java | 4 +-
.../org/apache/solr/search/facet/MinMaxAgg.java | 145 ++-
.../org/apache/solr/search/facet/SlotAcc.java | 84 +-
.../search/function/FieldNameValueSource.java | 60 ++
.../FieldLengthUpdateProcessorFactory.java | 6 +-
.../FieldMutatingUpdateProcessorFactory.java | 4 +-
.../ParseDateFieldUpdateProcessorFactory.java | 4 +-
.../ParseDoubleFieldUpdateProcessorFactory.java | 4 +-
.../ParseFloatFieldUpdateProcessorFactory.java | 4 +-
.../ParseIntFieldUpdateProcessorFactory.java | 4 +-
.../ParseLongFieldUpdateProcessorFactory.java | 4 +-
.../configuration/providers/package-info.java | 23 +
.../solr/collection1/conf/schema-enums.xml | 10 +
.../solr/collection1/conf/solrconfig.xml | 3 -
.../org/apache/solr/BasicFunctionalityTest.java | 2 +-
.../org/apache/solr/ConvertedLegacyTest.java | 70 +-
.../apache/solr/TestDistributedMissingSort.java | 8 +-
.../apache/solr/cloud/CdcrBootstrapTest.java | 1 +
.../autoscaling/ComputePlanActionTest.java | 53 +
.../autoscaling/SearchRateTriggerTest.java | 167 +++
.../org/apache/solr/core/TestLazyCores.java | 4 +-
.../apache/solr/handler/SearchHandlerTest.java | 4 +-
.../org/apache/solr/handler/TestSQLHandler.java | 1 -
.../solr/handler/admin/MetricsHandlerTest.java | 9 +
.../org/apache/solr/schema/EnumFieldTest.java | 16 +
.../org/apache/solr/schema/TestPointFields.java | 9 +-
.../org/apache/solr/search/CursorMarkTest.java | 10 +-
.../apache/solr/search/QueryEqualityTest.java | 8 -
.../apache/solr/search/QueryParsingTest.java | 1 -
.../apache/solr/search/TestSolrQueryParser.java | 2 +-
.../test/org/apache/solr/search/TestSort.java | 21 +-
.../solr/search/facet/TestJsonFacets.java | 86 +-
solr/example/example-DIH/.gitignore | 1 +
solr/example/example-DIH/hsqldb/.gitignore | 5 +
solr/example/example-DIH/hsqldb/ex.backup | Bin 11354 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.data | Bin 1048576 -> 0 bytes
solr/example/example-DIH/hsqldb/ex.script | 173 ++-
.../example-DIH/solr/db/conf/managed-schema | 2 +-
.../example-DIH/solr/mail/conf/managed-schema | 2 +-
.../example-DIH/solr/solr/conf/managed-schema | 2 +-
solr/example/files/conf/managed-schema | 8 +-
solr/example/films/README.txt | 6 +-
solr/licenses/commons-math3-3.4.1.jar.sha1 | 1 -
solr/licenses/commons-math3-3.6.1.jar.sha1 | 1 +
solr/solr-ref-guide/README.adoc | 32 +-
solr/solr-ref-guide/src/collections-api.adoc | 3 +
solr/solr-ref-guide/src/schema-api.adoc | 4 +-
solr/solr-ref-guide/src/schemaless-mode.adoc | 62 +-
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
...olrcloud-autoscaling-policy-preferences.adoc | 1 +
solr/solr-ref-guide/src/spatial-search.adoc | 5 +-
solr/solr-ref-guide/src/working-with-dates.adoc | 4 +-
solr/solrj/build.xml | 2 +
.../solrj/cloud/autoscaling/NoneSuggester.java | 4 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 6 +
.../solrj/cloud/autoscaling/ReplicaInfo.java | 24 +
.../impl/SolrClientClusterDataProvider.java | 20 +-
.../io/eval/BinomialDistributionEvaluator.java | 48 +
.../io/eval/CanberraDistanceEvaluator.java | 56 +
.../io/eval/ChebyshevDistanceEvaluator.java | 56 +
.../solrj/io/eval/CopyOfRangeEvaluator.java | 4 +-
.../io/eval/CosineSimilarityEvaluator.java | 67 ++
.../io/eval/CumulativeProbabilityEvaluator.java | 21 +-
.../solrj/io/eval/DotProductEvaluator.java | 58 +
.../client/solrj/io/eval/EBEAddEvaluator.java | 63 ++
.../solrj/io/eval/EBEDivideEvaluator.java | 63 ++
.../solrj/io/eval/EBEMultiplyEvaluator.java | 63 ++
.../solrj/io/eval/EBESubtractEvaluator.java | 63 ++
.../io/eval/EarthMoversDistanceEvaluator.java | 56 +
.../eval/EnumeratedDistributionEvaluator.java | 45 +
.../solrj/io/eval/FrequencyTableEvaluator.java | 84 ++
.../solrj/io/eval/HistogramEvaluator.java | 7 +-
.../io/eval/KendallsCorrelationEvaluator.java | 57 +
.../io/eval/ManhattanDistanceEvaluator.java | 56 +
.../solrj/io/eval/MeanDifferenceEvaluator.java | 55 +
.../io/eval/PoissonDistributionEvaluator.java | 44 +
.../client/solrj/io/eval/PrimesEvaluator.java | 58 +
.../solrj/io/eval/ProbabilityEvaluator.java | 52 +
.../client/solrj/io/eval/SampleEvaluator.java | 18 +-
.../io/eval/SpearmansCorrelationEvaluator.java | 57 +
.../solrj/io/eval/SumDifferenceEvaluator.java | 55 +
.../UniformIntegerDistributionEvaluator.java | 48 +
.../client/solrj/io/stream/FacetStream.java | 10 +-
.../solr/common/params/AutoScalingParams.java | 7 +
.../resources/apispec/collections.Commands.json | 5 +
.../collections.collection.Commands.modify.json | 5 +
.../solrj/cloud/autoscaling/TestPolicy.java | 38 +
.../solrj/io/stream/StreamExpressionTest.java | 357 +++++-
solr/webapp/build.xml | 3 +
171 files changed, 7177 insertions(+), 1749 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index d349fa9,420d4f3..c314cd6
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@@ -149,9 -127,11 +149,11 @@@ public class AutoScaling
}
switch (type) {
case NODEADDED:
- return new NodeAddedTrigger(name, props, coreContainer);
+ return new NodeAddedTrigger(name, props, loader, dataProvider);
case NODELOST:
- return new NodeLostTrigger(name, props, coreContainer);
+ return new NodeLostTrigger(name, props, loader, dataProvider);
+ case SEARCHRATE:
- return new SearchRateTrigger(name, props, coreContainer);
++ return new SearchRateTrigger(name, props, loader, dataProvider);
default:
throw new IllegalArgumentException("Unknown event type: " + type + " in trigger: " + name);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index d578254,6451c67..b8b73e7
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@@ -25,10 -28,17 +28,14 @@@ import java.util.Set
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.NoneSuggester;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
+import org.apache.solr.common.params.CollectionParams;
++import org.apache.solr.client.solrj.cloud.autoscaling.NoneSuggester;
+ import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
-import org.apache.solr.common.cloud.ZkStateReader;
+ import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.common.params.CollectionParams;
+ import org.apache.solr.common.util.Pair;
-import org.apache.solr.core.CoreContainer;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index ebe43bc,5ae28c2..438e383
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@@ -29,7 -28,6 +29,8 @@@ import java.util.NoSuchElementException
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
++
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
@@@ -51,11 -51,11 +52,11 @@@ public class NodeAddedTrigger extends T
private Map<String, Long> nodeNameVsTimeAdded = new HashMap<>();
public NodeAddedTrigger(String name, Map<String, Object> properties,
- CoreContainer container) {
- super(TriggerEventType.NODEADDED, name, properties, container.getResourceLoader(), container.getZkController().getZkClient());
- this.container = container;
+ SolrResourceLoader loader,
+ SolrCloudDataProvider dataProvider) {
- super(name, properties, loader, dataProvider);
++ super(TriggerEventType.NODEADDED, name, properties, loader, dataProvider);
this.timeSource = TimeSource.CURRENT_TIME;
- lastLiveNodes = new HashSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
+ lastLiveNodes = new HashSet<>(dataProvider.getClusterDataProvider().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
log.debug("NodeAddedTrigger {} instantiated with properties: {}", name, properties);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 1df6801,063f9d2..2174d41
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@@ -51,11 -53,11 +51,11 @@@ public class NodeLostTrigger extends Tr
private Map<String, Long> nodeNameVsTimeRemoved = new HashMap<>();
public NodeLostTrigger(String name, Map<String, Object> properties,
- CoreContainer container) {
- super(TriggerEventType.NODELOST, name, properties, container.getResourceLoader(), container.getZkController().getZkClient());
- this.container = container;
+ SolrResourceLoader loader,
+ SolrCloudDataProvider dataProvider) {
- super(name, properties, loader, dataProvider);
++ super(TriggerEventType.NODELOST, name, properties, loader, dataProvider);
this.timeSource = TimeSource.CURRENT_TIME;
- lastLiveNodes = new HashSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
+ lastLiveNodes = new HashSet<>(dataProvider.getClusterDataProvider().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
index 0000000,f8a4eeb..28dff03
mode 000000,100644..100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
@@@ -1,0 -1,290 +1,275 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.solr.cloud.autoscaling;
+
-import java.io.IOException;
+ import java.lang.invoke.MethodHandles;
+ import java.util.ArrayList;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.concurrent.ConcurrentHashMap;
+ import java.util.concurrent.TimeUnit;
+ import java.util.stream.Collectors;
+
+ import com.google.common.util.concurrent.AtomicDouble;
+ import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
+ import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
++import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
+ import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
-import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
+ import org.apache.solr.common.SolrException;
+ import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.core.CoreContainer;
++import org.apache.solr.core.SolrResourceLoader;
+ import org.apache.solr.metrics.SolrCoreMetricManager;
+ import org.apache.solr.util.TimeSource;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * Trigger for the {@link org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType#SEARCHRATE} event.
+ */
+ public class SearchRateTrigger extends TriggerBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final TimeSource timeSource;
- private final CoreContainer container;
+ private final String handler;
+ private final String collection;
+ private final String shard;
+ private final String node;
+ private final double rate;
+ private final Map<String, Long> lastCollectionEvent = new ConcurrentHashMap<>();
+ private final Map<String, Long> lastNodeEvent = new ConcurrentHashMap<>();
+ private final Map<String, Long> lastShardEvent = new ConcurrentHashMap<>();
+ private final Map<String, Long> lastReplicaEvent = new ConcurrentHashMap<>();
+ private final Map<String, Object> state = new HashMap<>();
+
- public SearchRateTrigger(String name, Map<String, Object> properties, CoreContainer container) {
- super(TriggerEventType.SEARCHRATE, name, properties, container.getResourceLoader(), container.getZkController().getZkClient());
++ public SearchRateTrigger(String name, Map<String, Object> properties, SolrResourceLoader loader, SolrCloudDataProvider dataProvider) {
++ super(TriggerEventType.SEARCHRATE, name, properties, loader, dataProvider);
+ this.timeSource = TimeSource.CURRENT_TIME;
- this.container = container;
+ this.state.put("lastCollectionEvent", lastCollectionEvent);
+ this.state.put("lastNodeEvent", lastNodeEvent);
+ this.state.put("lastShardEvent", lastShardEvent);
+ this.state.put("lastReplicaEvent", lastReplicaEvent);
+
+ // parse config options
+ collection = (String)properties.getOrDefault(AutoScalingParams.COLLECTION, Policy.ANY);
+ shard = (String)properties.getOrDefault(AutoScalingParams.SHARD, Policy.ANY);
+ if (collection.equals(Policy.ANY) && !shard.equals(Policy.ANY)) {
+ throw new IllegalArgumentException("When 'shard' is other than #ANY collection name must be also other than #ANY");
+ }
+ node = (String)properties.getOrDefault(AutoScalingParams.NODE, Policy.ANY);
+ handler = (String)properties.getOrDefault(AutoScalingParams.HANDLER, "/select");
+
+ if (properties.get("rate") == null) {
+ throw new IllegalArgumentException("No 'rate' specified in configuration");
+ }
+ String rateString = String.valueOf(properties.get("rate"));
+ try {
+ rate = Double.parseDouble(rateString);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Invalid 'rate' configuration value: '" + rateString + "'", e);
+ }
+ }
+
+ @Override
+ protected Map<String, Object> getState() {
+ return state;
+ }
+
+ @Override
+ protected void setState(Map<String, Object> state) {
+ lastCollectionEvent.clear();
+ lastNodeEvent.clear();
+ lastShardEvent.clear();
+ lastReplicaEvent.clear();
+ Map<String, Long> collTimes = (Map<String, Long>)state.get("lastCollectionEvent");
+ if (collTimes != null) {
+ lastCollectionEvent.putAll(collTimes);
+ }
+ Map<String, Long> nodeTimes = (Map<String, Long>)state.get("lastNodeEvent");
+ if (nodeTimes != null) {
+ lastNodeEvent.putAll(nodeTimes);
+ }
+ Map<String, Long> shardTimes = (Map<String, Long>)state.get("lastShardEvent");
+ if (shardTimes != null) {
+ lastShardEvent.putAll(shardTimes);
+ }
+ Map<String, Long> replicaTimes = (Map<String, Long>)state.get("lastReplicaEvent");
+ if (replicaTimes != null) {
+ lastReplicaEvent.putAll(replicaTimes);
+ }
+ }
+
+ @Override
+ public void restoreState(AutoScaling.Trigger old) {
+ assert old.isClosed();
+ if (old instanceof SearchRateTrigger) {
+ SearchRateTrigger that = (SearchRateTrigger)old;
+ assert this.name.equals(that.name);
+ this.lastCollectionEvent.clear();
+ this.lastNodeEvent.clear();
+ this.lastShardEvent.clear();
+ this.lastReplicaEvent.clear();
+ this.lastCollectionEvent.putAll(that.lastCollectionEvent);
+ this.lastNodeEvent.putAll(that.lastNodeEvent);
+ this.lastShardEvent.putAll(that.lastShardEvent);
+ this.lastReplicaEvent.putAll(that.lastReplicaEvent);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
+ "Unable to restore state from an unknown type of trigger");
+ }
+
+ }
+
+ @Override
+ public void run() {
+ AutoScaling.TriggerEventProcessor processor = processorRef.get();
+ if (processor == null) {
+ return;
+ }
+
+ Map<String, Map<String, List<ReplicaInfo>>> collectionRates = new HashMap<>();
+ Map<String, AtomicDouble> nodeRates = new HashMap<>();
+
- try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder()
- .withClusterStateProvider(new ZkClientClusterStateProvider(container.getZkController().getZkStateReader()))
- .build()) {
-
- SolrClientDataProvider dataProvider = new SolrClientDataProvider(cloudSolrClient);
-
- for (String node : dataProvider.getNodes()) {
- Map<String, ReplicaInfo> metricTags = new HashMap<>();
- // coll, shard, replica
- Map<String, Map<String, List<ReplicaInfo>>> infos = dataProvider.getReplicaInfo(node, Collections.emptyList());
- infos.forEach((coll, shards) -> {
- shards.forEach((sh, replicas) -> {
- replicas.forEach(replica -> {
- // we have to translate to the metrics registry name, which uses "_replica_nN" as suffix
- String replicaName = SolrCoreMetricManager.parseReplicaName(coll, replica.getCore());
- if (replicaName == null) { // should never happen???
- replicaName = replica.getName(); // which is actually coreNode name...
- }
- String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null);
- String tag = "metrics:" + registry
- + ":QUERY." + handler + ".requestTimes:1minRate";
- metricTags.put(tag, replica);
- });
++ for (String node : dataProvider.getClusterDataProvider().getLiveNodes()) {
++ Map<String, ReplicaInfo> metricTags = new HashMap<>();
++ // coll, shard, replica
++ Map<String, Map<String, List<ReplicaInfo>>> infos = dataProvider.getClusterDataProvider().getReplicaInfo(node, Collections.emptyList());
++ infos.forEach((coll, shards) -> {
++ shards.forEach((sh, replicas) -> {
++ replicas.forEach(replica -> {
++ // we have to translate to the metrics registry name, which uses "_replica_nN" as suffix
++ String replicaName = SolrCoreMetricManager.parseReplicaName(coll, replica.getCore());
++ if (replicaName == null) { // should never happen???
++ replicaName = replica.getName(); // which is actually coreNode name...
++ }
++ String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null);
++ String tag = "metrics:" + registry
++ + ":QUERY." + handler + ".requestTimes:1minRate";
++ metricTags.put(tag, replica);
+ });
+ });
- Map<String, Object> rates = dataProvider.getNodeValues(node, metricTags.keySet());
- rates.forEach((tag, rate) -> {
- ReplicaInfo info = metricTags.get(tag);
- if (info == null) {
- log.warn("Missing replica info for response tag " + tag);
- } else {
- Map<String, List<ReplicaInfo>> perCollection = collectionRates.computeIfAbsent(info.getCollection(), s -> new HashMap<>());
- List<ReplicaInfo> perShard = perCollection.computeIfAbsent(info.getShard(), s -> new ArrayList<>());
- info.getVariables().put(AutoScalingParams.RATE, rate);
- perShard.add(info);
- AtomicDouble perNode = nodeRates.computeIfAbsent(node, s -> new AtomicDouble());
- perNode.addAndGet((Double)rate);
- }
- });
- }
- } catch (IOException e) {
- log.warn("Exception getting node values", e);
- return;
++ });
++ Map<String, Object> rates = dataProvider.getClusterDataProvider().getNodeValues(node, metricTags.keySet());
++ rates.forEach((tag, rate) -> {
++ ReplicaInfo info = metricTags.get(tag);
++ if (info == null) {
++ log.warn("Missing replica info for response tag " + tag);
++ } else {
++ Map<String, List<ReplicaInfo>> perCollection = collectionRates.computeIfAbsent(info.getCollection(), s -> new HashMap<>());
++ List<ReplicaInfo> perShard = perCollection.computeIfAbsent(info.getShard(), s -> new ArrayList<>());
++ info.getVariables().put(AutoScalingParams.RATE, rate);
++ perShard.add(info);
++ AtomicDouble perNode = nodeRates.computeIfAbsent(node, s -> new AtomicDouble());
++ perNode.addAndGet((Double) rate);
++ }
++ });
+ }
+
+ long now = timeSource.getTime();
+ // check for exceeded rates and filter out those with less than waitFor from previous events
+ Map<String, Double> hotNodes = nodeRates.entrySet().stream()
+ .filter(entry -> node.equals(Policy.ANY) || node.equals(entry.getKey()))
+ .filter(entry -> waitForElapsed(entry.getKey(), now, lastNodeEvent))
+ .filter(entry -> entry.getValue().get() > rate)
+ .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().get()));
+
+ Map<String, Map<String, Double>> hotShards = new HashMap<>();
+ Map<String, String> warmShards = new HashMap<>();
+ List<ReplicaInfo> hotReplicas = new ArrayList<>();
+ collectionRates.forEach((coll, shardRates) -> {
+ final Object[] warmShard = new Object[2];
+ shardRates.forEach((sh, replicaRates) -> {
+ double shardRate = replicaRates.stream()
+ .map(r -> {
+ if (waitForElapsed(r.getCollection() + "." + r.getCore(), now, lastReplicaEvent) &&
+ ((Double)r.getVariable(AutoScalingParams.RATE) > rate)) {
+ hotReplicas.add(r);
+ }
+ return r;
+ })
+ .mapToDouble(r -> (Double)r.getVariable(AutoScalingParams.RATE)).sum();
+ if (warmShard[0] == null) {
+ warmShard[0] = sh;
+ warmShard[1] = shardRate;
+ }
+ if (shardRate > (double)warmShard[1]) {
+ warmShard[0] = sh;
+ warmShard[1] = shardRate;
+ }
+ if (waitForElapsed(coll + "." + sh, now, lastShardEvent) &&
+ (shardRate > rate) &&
+ (collection.equals(Policy.ANY) || collection.equals(coll)) &&
+ (shard.equals(Policy.ANY) || shard.equals(sh))) {
+ hotShards.computeIfAbsent(coll, s -> new HashMap<>()).put(sh, shardRate);
+ }
+ });
+ warmShards.put(coll, (String)warmShard[0]);
+ });
+
+ Map<String, Double> hotCollections = new HashMap<>();
+ collectionRates.forEach((coll, shRates) -> {
+ double total = shRates.entrySet().stream()
+ .mapToDouble(e -> e.getValue().stream()
+ .mapToDouble(r -> (Double)r.getVariable(AutoScalingParams.RATE)).sum()).sum();
+ if (waitForElapsed(coll, now, lastCollectionEvent) &&
+ (total > rate) &&
+ (collection.equals(Policy.ANY) || collection.equals(coll))) {
+ hotCollections.put(coll, total);
+ }
+ });
+
+ if (hotCollections.isEmpty() && hotShards.isEmpty() && hotReplicas.isEmpty() && hotNodes.isEmpty()) {
+ return;
+ }
+
+ // generate event
+
+ if (processor.process(new SearchRateEvent(getName(), now, hotNodes, hotCollections, hotShards, hotReplicas, warmShards))) {
+ // update lastEvent times
+ hotNodes.keySet().forEach(node -> lastNodeEvent.put(node, now));
+ hotCollections.keySet().forEach(coll -> lastCollectionEvent.put(coll, now));
+ hotShards.entrySet().forEach(e -> e.getValue()
+ .forEach((sh, rate) -> lastShardEvent.put(e.getKey() + "." + sh, now)));
+ hotReplicas.forEach(r -> lastReplicaEvent.put(r.getCollection() + "." + r.getCore(), now));
+ }
+ }
+
+ private boolean waitForElapsed(String name, long now, Map<String, Long> lastEventMap) {
+ Long lastTime = lastEventMap.computeIfAbsent(name, s -> now);
+ long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS);
+ log.info("name=" + name + ", lastTime=" + lastTime + ", elapsed=" + elapsed);
+ if (TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS) < getWaitForSecond()) {
+ return false;
+ }
+ return true;
+ }
+
+ public static class SearchRateEvent extends TriggerEvent {
+ public SearchRateEvent(String source, long eventTime, Map<String, Double> hotNodes,
+ Map<String, Double> hotCollections,
+ Map<String, Map<String, Double>> hotShards, List<ReplicaInfo> hotReplicas,
+ Map<String, String> warmShards) {
+ super(TriggerEventType.SEARCHRATE, source, eventTime, null);
+ properties.put(AutoScalingParams.COLLECTION, hotCollections);
+ properties.put(AutoScalingParams.SHARD, hotShards);
+ properties.put(AutoScalingParams.REPLICA, hotReplicas);
+ properties.put(AutoScalingParams.NODE, hotNodes);
+ properties.put(AutoScalingParams.WARM_SHARD, warmShards);
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
index 926df6d,fec6b05..721383a
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
@@@ -22,15 -22,16 +22,15 @@@ import java.util.ArrayList
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
--import java.util.Locale;
import java.util.Map;
import java.util.Objects;
-import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.util.IOUtils;
+import org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.common.cloud.SolrZkClient;
++
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
@@@ -57,9 -59,10 +57,12 @@@ public abstract class TriggerBase imple
protected final boolean enabled;
protected boolean isClosed;
- protected TriggerBase(TriggerEventType eventType, String name, Map<String, Object> properties, SolrResourceLoader loader, SolrZkClient zkClient) {
+
- protected TriggerBase(String name, Map<String, Object> properties, SolrResourceLoader loader, SolrCloudDataProvider dataProvider) {
++ protected TriggerBase(TriggerEventType eventType, String name, Map<String, Object> properties, SolrResourceLoader loader, SolrCloudDataProvider dataProvider) {
+ this.eventType = eventType;
this.name = name;
- this.zkClient = zkClient;
++ this.dataProvider = dataProvider;
++ this.stateManager = dataProvider.getDistribStateManager();
if (properties != null) {
this.properties.putAll(properties);
}
@@@ -140,11 -139,23 +140,12 @@@
}
@Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (obj.getClass().equals(this.getClass())) {
- TriggerBase that = (TriggerBase) obj;
- return this.name.equals(that.name)
- && this.properties.equals(that.properties);
+ public boolean isClosed() {
+ synchronized (this) {
+ return isClosed;
}
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name, properties);
}
+
@Override
public void close() throws IOException {
synchronized (this) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 8309a35,d7fd99b..1d6a9c6
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@@ -50,8 -51,28 +51,13 @@@ public class PolicyHelper
List<String> nodesList) {
List<ReplicaPosition> positions = new ArrayList<>();
final ClusterDataProvider delegate = cdp;
- cdp = new ClusterDataProvider() {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return delegate.getNodeValues(node, tags);
- }
-
- @Override
- public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return delegate.getReplicaInfo(node, keys);
- }
-
- @Override
- public Collection<String> getNodes() {
- return delegate.getNodes();
- }
-
+ cdp = new DelegatingClusterDataProvider(delegate) {
@Override
+ public ClusterState getClusterState() {
+ return delegate.getClusterState();
+ }
+
+ @Override
public String getPolicyNameByCollection(String coll) {
return policyMapping.get() != null && policyMapping.get().containsKey(coll) ?
optionalPolicyMapping.get(coll) :
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
index 77033c3,0000000..e08eda5
mode 100644,000000..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
@@@ -1,262 -1,0 +1,260 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
+import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.response.SimpleSolrResponse;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.cloud.rule.RemoteCallback;
+import org.apache.solr.common.cloud.rule.SnitchContext;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX;
+
+/**
+ *
+ */
+public class SolrClientClusterDataProvider implements ClusterDataProvider, MapWriter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CloudSolrClient solrClient;
+ private final ZkStateReader zkStateReader;
+ private final Map<String, Map<String, Map<String, List<ReplicaInfo>>>> data = new HashMap<>();
+ private Map<String, Object> snitchSession = new HashMap<>();
+ private Map<String, Map> nodeVsTags = new HashMap<>();
+
+ public SolrClientClusterDataProvider(CloudSolrClient solrClient) {
+ this.solrClient = solrClient;
+ this.zkStateReader = solrClient.getZkStateReader();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Map<String, ClusterState.CollectionRef> all = clusterState.getCollectionStates();
+ all.forEach((collName, ref) -> {
+ DocCollection coll = ref.get();
+ if (coll == null) return;
+ coll.forEachReplica((shard, replica) -> {
+ Map<String, Map<String, List<ReplicaInfo>>> nodeData = data.computeIfAbsent(replica.getNodeName(), k -> new HashMap<>());
+ Map<String, List<ReplicaInfo>> collData = nodeData.computeIfAbsent(collName, k -> new HashMap<>());
+ List<ReplicaInfo> replicas = collData.computeIfAbsent(shard, k -> new ArrayList<>());
- replicas.add(new ReplicaInfo(replica.getName(), collName, shard, replica.getType(), new HashMap<>()));
++ replicas.add(new ReplicaInfo(replica.getName(), collName, shard, replica.getType(), replica.getProperties()));
+ });
+ });
+ }
+
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put("liveNodes", zkStateReader.getClusterState().getLiveNodes());
- ew.put("replicaInfo", Utils.getDeepCopy(data, 5));
- ew.put("nodeValues", nodeVsTags);
++ public Collection<String> getLiveNodes() {
++ return zkStateReader.getClusterState().getLiveNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ ClusterState.CollectionRef state = solrClient.getClusterStateProvider().getState(coll);
+ return state == null || state.get() == null ? null : (String) state.get().getProperties().get("policy");
+ }
+
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ AutoScalingSnitch snitch = new AutoScalingSnitch();
+ ClientSnitchCtx ctx = new ClientSnitchCtx(null, node, snitchSession, solrClient);
+ snitch.getTags(node, new HashSet<>(tags), ctx);
+ nodeVsTags.put(node, ctx.getTags());
+ return ctx.getTags();
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return data.computeIfAbsent(node, s -> Collections.emptyMap());//todo fill other details
+ }
+
+ @Override
- public Collection<String> getLiveNodes() {
- return solrClient.getZkStateReader().getClusterState().getLiveNodes();
- }
-
- @Override
+ public Map<String, Object> getClusterProperties() {
+ return zkStateReader.getClusterProperties();
+ }
+
+ @Override
+ public ClusterState getClusterState() {
+ return zkStateReader.getClusterState();
+ }
+
++ public void writeMap(EntryWriter ew) throws IOException {
++ ew.put("clusterState", zkStateReader.getClusterState());
++ ew.put("replicaInfo", Utils.getDeepCopy(data, 5));
++ ew.put("nodeValues", nodeVsTags);
++ }
++
+ @Override
+ public AutoScalingConfig getAutoScalingConfig(Watcher watcher) throws IOException {
+ try {
+ return zkStateReader.getAutoScalingConfig(watcher);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ static class ClientSnitchCtx
+ extends SnitchContext {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ ZkClientClusterStateProvider zkClientClusterStateProvider;
+ CloudSolrClient solrClient;
+
+ public ClientSnitchCtx(SnitchInfo perSnitch,
+ String node, Map<String, Object> session,
+ CloudSolrClient solrClient) {
+ super(perSnitch, node, session);
+ this.solrClient = solrClient;
+ this.zkClientClusterStateProvider = (ZkClientClusterStateProvider) solrClient.getClusterStateProvider();
+ }
+
+
+ @Override
+ public Map getZkJson(String path) throws KeeperException, InterruptedException {
+ return Utils.getJson(zkClientClusterStateProvider.getZkStateReader().getZkClient(), path, true);
+ }
+
+ public void invokeRemote(String node, ModifiableSolrParams params, String klas, RemoteCallback callback) {
+
+ }
+
+ public SimpleSolrResponse invoke(String solrNode, String path, SolrParams params)
+ throws IOException, SolrServerException {
+ String url = zkClientClusterStateProvider.getZkStateReader().getBaseUrlForNodeName(solrNode);
+
+ GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.GET, path, params);
+ try (HttpSolrClient client = new HttpSolrClient.Builder()
+ .withHttpClient(solrClient.getHttpClient())
+ .withBaseSolrUrl(url)
+ .withResponseParser(new BinaryResponseParser())
+ .build()) {
+ NamedList<Object> rsp = client.request(request);
+ request.response.nl = rsp;
+ return request.response;
+ }
+ }
+
+ }
+
+ //uses metrics API to get node information
+ static class AutoScalingSnitch extends ImplicitSnitch {
+ @Override
+ protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
+ ClientSnitchCtx snitchContext = (ClientSnitchCtx) ctx;
+ Map<String, String> metricsKeyVsTag = new HashMap<>();
+ for (String tag : requestedTags) {
+ if (tag.startsWith(SYSPROP)) {
+ metricsKeyVsTag.put("solr.jvm:system.properties:" + tag.substring(SYSPROP.length()), tag);
+ } else if (tag.startsWith(METRICS_PREFIX)) {
+ metricsKeyVsTag.put(tag.substring(METRICS_PREFIX.length()), tag);
+ }
+ }
+ if (!metricsKeyVsTag.isEmpty()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("key", metricsKeyVsTag.keySet().toArray(new String[metricsKeyVsTag.size()]));
+ try {
+ SimpleSolrResponse rsp = snitchContext.invoke(solrNode, CommonParams.METRICS_PATH, params);
+ metricsKeyVsTag.forEach((key, tag) -> {
+ Object v = Utils.getObjectByPath(rsp.nl, true, Arrays.asList("metrics", key));
+ if (v != null) snitchContext.getTags().put(tag, v);
+ });
+ } catch (Exception e) {
+ log.warn("could not get tags from node " + solrNode, e);
+ }
+ }
+
+ Set<String> groups = new HashSet<>();
+ List<String> prefixes = new ArrayList<>();
+ if (requestedTags.contains(DISK)) {
+ groups.add("solr.node");
+ prefixes.add("CONTAINER.fs.usableSpace");
+ }
+ if (requestedTags.contains(CORES)) {
+ groups.add("solr.core");
+ prefixes.add("CORE.coreName");
+ }
+ if (requestedTags.contains(SYSLOADAVG)) {
+ groups.add("solr.jvm");
+ prefixes.add("os.systemLoadAverage");
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ groups.add("solr.jvm");
+ prefixes.add("memory.heap.usage");
+ }
+ if (groups.isEmpty() || prefixes.isEmpty()) return;
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("group", StrUtils.join(groups, ','));
+ params.add("prefix", StrUtils.join(prefixes, ','));
+
+ try {
+ SimpleSolrResponse rsp = snitchContext.invoke(solrNode, CommonParams.METRICS_PATH, params);
+ Map m = rsp.nl.asMap(4);
+ if (requestedTags.contains(DISK)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.usableSpace");
+ if (n != null) ctx.getTags().put(DISK, n.doubleValue() / 1024.0d / 1024.0d / 1024.0d);
+ }
+ if (requestedTags.contains(CORES)) {
+ int count = 0;
+ Map cores = (Map) m.get("metrics");
+ for (Object o : cores.keySet()) {
+ if (o.toString().startsWith("solr.core.")) count++;
+ }
+ ctx.getTags().put(CORES, count);
+ }
+ if (requestedTags.contains(SYSLOADAVG)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.jvm/os.systemLoadAverage");
+ if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue() * 100.0d);
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.jvm/memory.heap.usage");
+ if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/877d9227/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index c3a227b,8273d2c..2b0cd4a
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@@ -1172,9 -1198,15 +1198,15 @@@ public class TestPolicy extends SolrTes
}
@Override
- public Collection<String> getNodes() {
+ public Collection<String> getLiveNodes() {
return Arrays.asList( "127.0.0.1:50097_solr", "127.0.0.1:50096_solr");
}
+
+ @Override
+ public ClusterState getClusterState() {
+ throw new UnsupportedOperationException("getClusterState");
+ }
+
};
List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
"newColl", new AutoScalingConfig((Map<String, Object>)Utils.fromJSONString(autoScaleJson)),
@@@ -1229,9 -1261,15 +1261,15 @@@
}
@Override
- public Collection<String> getNodes() {
+ public Collection<String> getLiveNodes() {
return Arrays.asList("node1", "node2", "node3", "node4");
}
+
+ @Override
+ public ClusterState getClusterState() {
+ throw new UnsupportedOperationException("getClusterState");
+ }
+
};
List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
"newColl", new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScaleJson)),