You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/07/13 15:38:43 UTC
[46/47] lucene-solr:jira/solr-11000: Merge branch
'feature/autoscaling' into jira/solr-11000
Merge branch 'feature/autoscaling' into jira/solr-11000
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5aaaa491
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5aaaa491
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5aaaa491
Branch: refs/heads/jira/solr-11000
Commit: 5aaaa491ca50ea46823716abd771248907a31d5d
Parents: 088fce8 d2849cd
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Jul 13 14:02:54 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Jul 13 14:02:54 2017 +0200
----------------------------------------------------------------------
dev-tools/scripts/addVersion.py | 2 +-
solr/CHANGES.txt | 30 +-
solr/bin/solr | 24 +-
solr/bin/solr.cmd | 8 +
.../java/org/apache/solr/ltr/LTRRescorer.java | 13 +-
.../org/apache/solr/cloud/AddReplicaCmd.java | 23 +-
.../src/java/org/apache/solr/cloud/Assign.java | 25 +-
.../java/org/apache/solr/cloud/CloudUtil.java | 4 +-
.../apache/solr/cloud/CreateCollectionCmd.java | 43 +-
.../org/apache/solr/cloud/CreateShardCmd.java | 81 +-
.../apache/solr/cloud/DeleteCollectionCmd.java | 33 +
.../org/apache/solr/cloud/MoveReplicaCmd.java | 35 +-
.../java/org/apache/solr/cloud/Overseer.java | 2 +-
.../org/apache/solr/cloud/ZkController.java | 5 +-
.../org/apache/solr/handler/StreamHandler.java | 2 +-
.../solr/handler/admin/CollectionsHandler.java | 2 +
.../component/QueryElevationComponent.java | 82 +-
.../solr/search/stats/ExactStatsCache.java | 5 +-
.../apache/solr/search/stats/LRUStatsCache.java | 2 +-
.../solr/search/stats/LocalStatsCache.java | 18 +-
.../src/java/org/apache/solr/util/SolrCLI.java | 202 ++-
...chema-add-schema-fields-update-processor.xml | 12 +-
.../conf/schema-blockjoinfacetcomponent.xml | 6 +-
.../collection1/conf/schema-custom-field.xml | 4 +-
.../solr/configsets/_default/conf/elevate.xml | 42 -
.../configsets/_default/conf/solrconfig.xml | 1 -
.../solr/cloud/CollectionsAPISolrJTest.java | 3 +-
.../apache/solr/cloud/MoveReplicaHDFSTest.java | 1 -
.../solr/cloud/MoveReplicaHDFSUlogDirTest.java | 142 ++
.../cloud/TestPullReplicaErrorHandling.java | 3 +-
.../cloud/TestRandomRequestDistribution.java | 64 +-
.../autoscaling/AutoScalingHandlerTest.java | 1 -
.../autoscaling/ComputePlanActionTest.java | 5 +-
.../autoscaling/HttpTriggerListenerTest.java | 2 +-
.../solr/cloud/autoscaling/TestPolicyCloud.java | 50 +-
.../autoscaling/TriggerIntegrationTest.java | 4 +-
.../org/apache/solr/schema/TestPointFields.java | 1409 ++++++++++++------
.../solr/schema/TestUseDocValuesAsStored.java | 24 +-
.../solr/configsets/_default/conf/elevate.xml | 42 -
.../configsets/_default/conf/solrconfig.xml | 1 -
solr/solr-ref-guide/src/about-this-guide.adoc | 59 +-
solr/solr-ref-guide/src/about-tokenizers.adoc | 1 -
...adding-custom-plugins-in-solrcloud-mode.adoc | 9 -
solr/solr-ref-guide/src/analyzers.adoc | 2 -
...uthentication-and-authorization-plugins.adoc | 18 +-
.../src/basic-authentication-plugin.adoc | 11 +-
solr/solr-ref-guide/src/blob-store-api.adoc | 3 -
solr/solr-ref-guide/src/blockjoin-faceting.adoc | 4 +-
.../solr-ref-guide/src/charfilterfactories.adoc | 4 -
.../src/collapse-and-expand-results.adoc | 4 +-
solr/solr-ref-guide/src/collections-api.adoc | 136 +-
.../src/collections-core-admin.adoc | 2 +-
.../src/command-line-utilities.adoc | 26 +-
.../src/common-query-parameters.adoc | 4 +-
solr/solr-ref-guide/src/config-api.adoc | 40 +-
solr/solr-ref-guide/src/configsets-api.adoc | 79 +-
.../solr-ref-guide/src/configuring-logging.adoc | 5 -
.../src/configuring-solrconfig-xml.adoc | 8 +-
solr/solr-ref-guide/src/content-streams.adoc | 7 +-
solr/solr-ref-guide/src/coreadmin-api.adoc | 29 +-
.../src/cross-data-center-replication-cdcr.adoc | 140 +-
...adir-and-directoryfactory-in-solrconfig.adoc | 3 -
solr/solr-ref-guide/src/dataimport-screen.adoc | 1 -
solr/solr-ref-guide/src/de-duplication.adoc | 5 -
.../src/defining-core-properties.adoc | 6 +-
solr/solr-ref-guide/src/defining-fields.adoc | 5 +-
.../detecting-languages-during-indexing.adoc | 4 -
.../src/distributed-requests.adoc | 9 +-
.../distributed-search-with-index-sharding.adoc | 7 +-
solr/solr-ref-guide/src/documents-screen.adoc | 16 +-
solr/solr-ref-guide/src/docvalues.adoc | 2 -
solr/solr-ref-guide/src/enabling-ssl.adoc | 22 +-
solr/solr-ref-guide/src/errata.adoc | 2 -
.../src/exporting-result-sets.adoc | 6 -
solr/solr-ref-guide/src/faceting.adoc | 4 +-
.../field-type-definitions-and-properties.adoc | 8 +-
.../src/field-types-included-with-solr.adoc | 6 +-
.../solr-ref-guide/src/filter-descriptions.adoc | 70 +-
solr/solr-ref-guide/src/function-queries.adoc | 14 +-
.../src/getting-started-with-solrcloud.adoc | 5 -
solr/solr-ref-guide/src/graph-traversal.adoc | 32 +-
.../src/hadoop-authentication-plugin.adoc | 7 +-
solr/solr-ref-guide/src/highlighting.adoc | 13 +-
.../solr-ref-guide/src/how-solrcloud-works.adoc | 7 +-
.../src/implicit-requesthandlers.adoc | 13 +-
solr/solr-ref-guide/src/index-replication.adoc | 15 +-
.../src/indexconfig-in-solrconfig.adoc | 20 +-
.../src/indexing-and-basic-data-operations.adoc | 1 -
.../src/initparams-in-solrconfig.adoc | 3 +-
.../src/introduction-to-solr-indexing.adoc | 2 -
solr/solr-ref-guide/src/jvm-settings.adoc | 3 -
.../src/kerberos-authentication-plugin.adoc | 19 +-
solr/solr-ref-guide/src/language-analysis.adoc | 125 +-
solr/solr-ref-guide/src/learning-to-rank.adoc | 74 +-
.../src/local-parameters-in-queries.adoc | 3 -
solr/solr-ref-guide/src/logging.adoc | 1 -
.../major-changes-from-solr-5-to-solr-6.adoc | 4 +-
.../src/making-and-restoring-backups.adoc | 8 +-
solr/solr-ref-guide/src/managed-resources.adoc | 17 +-
.../src/mbean-request-handler.adoc | 3 +-
solr/solr-ref-guide/src/merging-indexes.adoc | 6 +-
solr/solr-ref-guide/src/morelikethis.adoc | 9 +-
.../src/near-real-time-searching.adoc | 10 +-
solr/solr-ref-guide/src/other-parsers.adoc | 99 +-
.../src/other-schema-elements.adoc | 2 -
.../src/overview-of-searching-in-solr.adoc | 2 +-
.../src/pagination-of-results.adoc | 4 +-
.../src/performance-statistics-reference.adoc | 4 +-
solr/solr-ref-guide/src/phonetic-matching.adoc | 20 +-
solr/solr-ref-guide/src/post-tool.adoc | 11 +-
.../src/query-settings-in-solrconfig.adoc | 16 +-
.../read-and-write-side-fault-tolerance.adoc | 6 -
.../src/request-parameters-api.adoc | 15 +-
.../src/requestdispatcher-in-solrconfig.adoc | 6 +-
...lers-and-searchcomponents-in-solrconfig.adoc | 12 +-
solr/solr-ref-guide/src/response-writers.adoc | 69 +-
solr/solr-ref-guide/src/result-clustering.adoc | 36 +-
solr/solr-ref-guide/src/result-grouping.adoc | 15 +-
.../src/rule-based-authorization-plugin.adoc | 16 +-
.../src/rule-based-replica-placement.adoc | 28 +-
.../src/running-solr-on-hdfs.adoc | 20 +-
solr/solr-ref-guide/src/running-solr.adoc | 4 +-
solr/solr-ref-guide/src/schema-api.adoc | 4 +-
...schema-factory-definition-in-solrconfig.adoc | 6 +-
solr/solr-ref-guide/src/schemaless-mode.adoc | 17 +-
solr/solr-ref-guide/src/segments-info.adoc | 2 +-
...tting-up-an-external-zookeeper-ensemble.adoc | 21 +-
.../shards-and-indexing-data-in-solrcloud.adoc | 7 +-
.../src/solr-control-script-reference.adoc | 13 +-
solr/solr-ref-guide/src/solr-glossary.adoc | 4 +-
.../src/solr-jdbc-apache-zeppelin.adoc | 3 -
.../src/solr-jdbc-dbvisualizer.adoc | 15 +-
solr/solr-ref-guide/src/spatial-search.adoc | 23 +-
solr/solr-ref-guide/src/spell-checking.adoc | 2 +-
solr/solr-ref-guide/src/stream-decorators.adoc | 16 +-
.../src/streaming-expressions.adoc | 3 -
solr/solr-ref-guide/src/suggester.adoc | 333 +++--
.../src/taking-solr-to-production.adoc | 23 +-
.../src/the-query-elevation-component.adoc | 11 +-
.../solr-ref-guide/src/the-stats-component.adoc | 16 +-
.../src/the-term-vector-component.adoc | 8 +-
.../solr-ref-guide/src/the-terms-component.adoc | 13 +-
.../src/the-well-configured-solr-instance.adoc | 2 -
solr/solr-ref-guide/src/tokenizers.adoc | 14 -
.../transforming-and-indexing-custom-json.adoc | 52 +-
.../src/transforming-result-documents.adoc | 25 +-
solr/solr-ref-guide/src/uima-integration.adoc | 2 -
...anding-analyzers-tokenizers-and-filters.adoc | 4 -
.../src/update-request-processors.adoc | 25 +-
.../src/updatehandlers-in-solrconfig.adoc | 8 +-
.../src/updating-parts-of-documents.adoc | 27 +-
.../src/upgrading-a-solr-cluster.adoc | 8 -
solr/solr-ref-guide/src/upgrading-solr.adoc | 8 +-
.../src/uploading-data-with-index-handlers.adoc | 37 +-
...g-data-with-solr-cell-using-apache-tika.adoc | 48 +-
solr/solr-ref-guide/src/using-javascript.adoc | 2 +-
.../solr-ref-guide/src/using-jmx-with-solr.adoc | 10 +-
solr/solr-ref-guide/src/using-python.adoc | 4 +-
.../src/using-solr-from-ruby.adoc | 2 +-
solr/solr-ref-guide/src/using-solrj.adoc | 8 -
...zookeeper-to-manage-configuration-files.adoc | 25 +-
solr/solr-ref-guide/src/v2-api.adoc | 5 -
.../src/velocity-response-writer.adoc | 3 -
solr/solr-ref-guide/src/velocity-search-ui.adoc | 4 +-
...king-with-currencies-and-exchange-rates.adoc | 23 +-
solr/solr-ref-guide/src/working-with-dates.adoc | 10 -
.../src/working-with-enum-fields.adoc | 10 +-
...rking-with-external-files-and-processes.adoc | 27 +-
.../src/zookeeper-access-control.adoc | 8 -
.../solrj/io/eval/ResidualsEvaluator.java | 82 +
.../solrj/request/CollectionAdminRequest.java | 13 +
.../solrj/response/CollectionAdminResponse.java | 5 +
.../apache/solr/common/cloud/DocCollection.java | 7 +-
.../org/apache/solr/common/cloud/Replica.java | 7 +-
.../solr/common/cloud/ReplicaPosition.java | 2 +-
.../solrj/io/stream/StreamExpressionTest.java | 24 +
solr/webapp/web/js/angular/controllers/cloud.js | 1 +
177 files changed, 2614 insertions(+), 2491 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5aaaa491/solr/core/src/java/org/apache/solr/cloud/Assign.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/Assign.java
index 2882b30,266dab4..c2fa138
--- a/solr/core/src/java/org/apache/solr/cloud/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Assign.java
@@@ -62,8 -63,8 +63,6 @@@ import static org.apache.solr.cloud.Ove
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.CREATE_NODE_SET_SHUFFLE_DEFAULT;
import static org.apache.solr.common.cloud.DocCollection.SNITCH;
import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
- import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
--
public class Assign {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@@ -321,15 -323,18 +321,18 @@@
int pullReplicas,
String policyName, ZkStateReader zkStateReader,
List<String> nodesList) throws KeeperException, InterruptedException {
+ log.debug("shardnames {} NRT {} TLOG {} PULL {} , policy {}, nodeList {}", shardNames, nrtReplicas, tlogReplicas, pullReplicas, policyName, nodesList);
+ SolrClientDataProvider clientDataProvider = null;
+ List<ReplicaPosition> replicaPositions = null;
++ AutoScalingConfig autoScalingConfig = zkStateReader.getAutoScalingConfig();
try (CloudSolrClient csc = new CloudSolrClient.Builder()
.withClusterStateProvider(new ZkClientClusterStateProvider(zkStateReader))
.build()) {
- SolrClientDataProvider clientDataProvider = new SolrClientDataProvider(csc);
- AutoScalingConfig autoScalingConfig = zkStateReader.getAutoScalingConfig();
+ clientDataProvider = new SolrClientDataProvider(csc);
- Map<String, Object> autoScalingJson = Utils.getJson(zkStateReader.getZkClient(), SOLR_AUTOSCALING_CONF_PATH, true);
Map<String, String> kvMap = Collections.singletonMap(collName, policyName);
- return PolicyHelper.getReplicaLocations(
+ replicaPositions = PolicyHelper.getReplicaLocations(
collName,
- autoScalingJson,
+ autoScalingConfig,
clientDataProvider,
kvMap,
shardNames,
@@@ -337,8 -342,16 +340,16 @@@
tlogReplicas,
pullReplicas,
nodesList);
+ return replicaPositions;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error closing CloudSolrClient",e);
+ } finally {
+ if (log.isTraceEnabled()) {
+ if (clientDataProvider != null) log.trace("CLUSTER_DATA_PROVIDER: " + Utils.toJSONString(clientDataProvider));
+ if (replicaPositions != null)
+ log.trace("REPLICA_POSITIONS: " + Utils.toJSONString(Utils.getDeepCopy(replicaPositions, 7, true)));
- log.trace("AUTOSCALING_JSON: " + Utils.toJSONString(Utils.getJson(zkStateReader.getZkClient(), SOLR_AUTOSCALING_CONF_PATH, true)));
++ log.trace("AUTOSCALING_CONF: " + Utils.toJSONString(autoScalingConfig));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5aaaa491/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5aaaa491/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5aaaa491/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5aaaa491/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------