You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/06/27 05:19:38 UTC
[27/27] lucene-solr:feature/autoscaling: Merge branch 'master' into
feature/autoscaling
Merge branch 'master' into feature/autoscaling
# Conflicts:
# solr/CHANGES.txt
# solr/core/src/java/org/apache/solr/cloud/ZkController.java
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e863d0f5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e863d0f5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e863d0f5
Branch: refs/heads/feature/autoscaling
Commit: e863d0f548ea3fe76e5fdc369ef990ed37377dbb
Parents: 4c47a32 5d44243
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Jun 27 10:48:05 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Jun 27 10:48:05 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 24 +-
solr/bin/solr | 36 +-
solr/bin/solr.cmd | 24 +-
.../accumulator/FacetingAccumulator.java | 24 +-
.../solr/analytics/expression/Expression.java | 15 +-
.../statistics/MinMaxStatsCollector.java | 23 +-
.../statistics/NumericStatsCollector.java | 8 +-
.../StatsCollectorSupplierFactory.java | 14 +-
.../analytics/requestFiles/facetSorting.txt | 4 -
.../analytics/requestXMLFiles/facetSorting.xml | 14 -
.../solr/collection1/conf/schema-analytics.xml | 10 +-
.../collection1/conf/solrconfig-analytics.xml | 40 -
.../solr/collection1/conf/solrconfig-basic.xml | 40 +
.../analytics/AbstractAnalyticsStatsTest.java | 1 +
.../org/apache/solr/analytics/NoFacetTest.java | 3 +-
.../analytics/expression/ExpressionTest.java | 2 +-
.../facet/AbstractAnalyticsFacetTest.java | 16 +-
.../solr/analytics/facet/FacetSortingTest.java | 56 -
.../analytics/facet/FieldFacetExtrasTest.java | 2 +-
.../solr/analytics/facet/FieldFacetTest.java | 21 +-
.../solr/analytics/facet/QueryFacetTest.java | 2 +-
.../solr/analytics/facet/RangeFacetTest.java | 2 +-
.../util/valuesource/FunctionTest.java | 2 +-
.../clustering/solr/collection1/conf/schema.xml | 18 +-
.../conf/dataimport-schema-no-unique-key.xml | 18 +-
.../solr/collection1/conf/dataimport-schema.xml | 10 +-
.../conf/dataimport-solr_id-schema.xml | 18 +-
.../handler/extraction/SolrContentHandler.java | 4 +-
.../extraction/solr/collection1/conf/schema.xml | 18 +-
.../test-files/solr/collection1/conf/schema.xml | 10 +-
.../uima/solr/collection1/conf/schema.xml | 20 +-
.../test-files/uima/uima-tokenizers-schema.xml | 20 +-
.../apache/solr/cloud/CreateCollectionCmd.java | 65 +-
.../org/apache/solr/cloud/ZkController.java | 67 +-
.../handler/admin/ConfigSetsHandlerApi.java | 2 +
.../solr/handler/component/ExpandComponent.java | 2 +
.../apache/solr/metrics/SolrMetricReporter.java | 15 +
.../metrics/reporters/SolrGangliaReporter.java | 5 -
.../metrics/reporters/SolrGraphiteReporter.java | 5 -
.../solr/metrics/reporters/SolrJmxReporter.java | 5 +-
.../metrics/reporters/SolrSlf4jReporter.java | 5 -
.../reporters/solr/SolrClusterReporter.java | 12 +-
.../reporters/solr/SolrShardReporter.java | 12 +-
.../apache/solr/servlet/SolrDispatchFilter.java | 6 +-
.../src/java/org/apache/solr/util/SolrCLI.java | 32 +-
.../solr/configsets/_default/conf/schema.xml | 37 +
.../configsets/_default/conf/solrconfig.xml | 32 +
.../solr/cloud/BaseCdcrDistributedZkTest.java | 4 +-
.../solr/cloud/BasicDistributedZk2Test.java | 2 +-
.../solr/cloud/BasicDistributedZkTest.java | 12 +-
.../solr/cloud/ClusterStateUpdateTest.java | 2 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 19 +
.../org/apache/solr/cloud/ForceLeaderTest.java | 4 +-
.../apache/solr/cloud/HttpPartitionTest.java | 10 +-
.../cloud/LeaderFailoverAfterPartitionTest.java | 2 +-
.../LeaderInitiatedRecoveryOnCommitTest.java | 4 +-
...aderInitiatedRecoveryOnShardRestartTest.java | 2 +-
.../solr/cloud/ReplicationFactorTest.java | 6 +-
.../solr/cloud/ShardRoutingCustomTest.java | 2 +-
.../org/apache/solr/cloud/ShardSplitTest.java | 2 +-
.../apache/solr/cloud/SolrCloudExampleTest.java | 11 +-
.../apache/solr/cloud/TestConfigSetsAPI.java | 6 +-
.../cloud/TestOnReconnectListenerSupport.java | 2 +-
.../org/apache/solr/cloud/TestPullReplica.java | 8 +-
.../cloud/TestRandomRequestDistribution.java | 6 +-
.../cloud/TestSolrCloudWithKerberosAlt.java | 2 +-
.../org/apache/solr/cloud/TestTlogReplica.java | 8 +-
.../solr/cloud/UnloadDistributedZkTest.java | 4 +-
.../autoscaling/AutoScalingHandlerTest.java | 17 +-
.../solr/cloud/autoscaling/TestPolicyCloud.java | 6 +-
.../solr/cloud/hdfs/HdfsNNFailoverTest.java | 2 +-
.../HdfsWriteToMultipleCollectionsTest.java | 2 +-
.../apache/solr/cloud/hdfs/StressHdfsTest.java | 4 +-
.../handler/ThrowErrorOnInitRequestHandler.java | 4 +
.../TestTrackingShardHandlerFactory.java | 2 +-
.../solr/metrics/SolrMetricManagerTest.java | 5 +
.../metrics/reporters/MockMetricReporter.java | 3 +
.../search/stats/TestDefaultStatsCache.java | 2 +-
.../apache/solr/util/TestSolrCLIRunExample.java | 2 +-
solr/server/README.txt | 11 +-
.../solr/configsets/_default/conf/currency.xml | 67 +
.../solr/configsets/_default/conf/elevate.xml | 42 +
.../_default/conf/lang/contractions_ca.txt | 8 +
.../_default/conf/lang/contractions_fr.txt | 15 +
.../_default/conf/lang/contractions_ga.txt | 5 +
.../_default/conf/lang/contractions_it.txt | 23 +
.../_default/conf/lang/hyphenations_ga.txt | 5 +
.../_default/conf/lang/stemdict_nl.txt | 6 +
.../_default/conf/lang/stoptags_ja.txt | 420 ++++++
.../_default/conf/lang/stopwords_ar.txt | 125 ++
.../_default/conf/lang/stopwords_bg.txt | 193 +++
.../_default/conf/lang/stopwords_ca.txt | 220 +++
.../_default/conf/lang/stopwords_cz.txt | 172 +++
.../_default/conf/lang/stopwords_da.txt | 110 ++
.../_default/conf/lang/stopwords_de.txt | 294 ++++
.../_default/conf/lang/stopwords_el.txt | 78 +
.../_default/conf/lang/stopwords_en.txt | 54 +
.../_default/conf/lang/stopwords_es.txt | 356 +++++
.../_default/conf/lang/stopwords_eu.txt | 99 ++
.../_default/conf/lang/stopwords_fa.txt | 313 ++++
.../_default/conf/lang/stopwords_fi.txt | 97 ++
.../_default/conf/lang/stopwords_fr.txt | 186 +++
.../_default/conf/lang/stopwords_ga.txt | 110 ++
.../_default/conf/lang/stopwords_gl.txt | 161 ++
.../_default/conf/lang/stopwords_hi.txt | 235 +++
.../_default/conf/lang/stopwords_hu.txt | 211 +++
.../_default/conf/lang/stopwords_hy.txt | 46 +
.../_default/conf/lang/stopwords_id.txt | 359 +++++
.../_default/conf/lang/stopwords_it.txt | 303 ++++
.../_default/conf/lang/stopwords_ja.txt | 127 ++
.../_default/conf/lang/stopwords_lv.txt | 172 +++
.../_default/conf/lang/stopwords_nl.txt | 119 ++
.../_default/conf/lang/stopwords_no.txt | 194 +++
.../_default/conf/lang/stopwords_pt.txt | 253 ++++
.../_default/conf/lang/stopwords_ro.txt | 233 +++
.../_default/conf/lang/stopwords_ru.txt | 243 +++
.../_default/conf/lang/stopwords_sv.txt | 133 ++
.../_default/conf/lang/stopwords_th.txt | 119 ++
.../_default/conf/lang/stopwords_tr.txt | 212 +++
.../_default/conf/lang/userdict_ja.txt | 29 +
.../configsets/_default/conf/managed-schema | 1076 ++++++++++++++
.../solr/configsets/_default/conf/params.json | 20 +
.../solr/configsets/_default/conf/protwords.txt | 21 +
.../configsets/_default/conf/solrconfig.xml | 1394 +++++++++++++++++
.../solr/configsets/_default/conf/stopwords.txt | 14 +
.../solr/configsets/_default/conf/synonyms.txt | 29 +
.../configsets/basic_configs/conf/currency.xml | 67 -
.../configsets/basic_configs/conf/elevate.xml | 42 -
.../basic_configs/conf/lang/contractions_ca.txt | 8 -
.../basic_configs/conf/lang/contractions_fr.txt | 15 -
.../basic_configs/conf/lang/contractions_ga.txt | 5 -
.../basic_configs/conf/lang/contractions_it.txt | 23 -
.../basic_configs/conf/lang/hyphenations_ga.txt | 5 -
.../basic_configs/conf/lang/stemdict_nl.txt | 6 -
.../basic_configs/conf/lang/stoptags_ja.txt | 420 ------
.../basic_configs/conf/lang/stopwords_ar.txt | 125 --
.../basic_configs/conf/lang/stopwords_bg.txt | 193 ---
.../basic_configs/conf/lang/stopwords_ca.txt | 220 ---
.../basic_configs/conf/lang/stopwords_cz.txt | 172 ---
.../basic_configs/conf/lang/stopwords_da.txt | 110 --
.../basic_configs/conf/lang/stopwords_de.txt | 294 ----
.../basic_configs/conf/lang/stopwords_el.txt | 78 -
.../basic_configs/conf/lang/stopwords_en.txt | 54 -
.../basic_configs/conf/lang/stopwords_es.txt | 356 -----
.../basic_configs/conf/lang/stopwords_eu.txt | 99 --
.../basic_configs/conf/lang/stopwords_fa.txt | 313 ----
.../basic_configs/conf/lang/stopwords_fi.txt | 97 --
.../basic_configs/conf/lang/stopwords_fr.txt | 186 ---
.../basic_configs/conf/lang/stopwords_ga.txt | 110 --
.../basic_configs/conf/lang/stopwords_gl.txt | 161 --
.../basic_configs/conf/lang/stopwords_hi.txt | 235 ---
.../basic_configs/conf/lang/stopwords_hu.txt | 211 ---
.../basic_configs/conf/lang/stopwords_hy.txt | 46 -
.../basic_configs/conf/lang/stopwords_id.txt | 359 -----
.../basic_configs/conf/lang/stopwords_it.txt | 303 ----
.../basic_configs/conf/lang/stopwords_ja.txt | 127 --
.../basic_configs/conf/lang/stopwords_lv.txt | 172 ---
.../basic_configs/conf/lang/stopwords_nl.txt | 119 --
.../basic_configs/conf/lang/stopwords_no.txt | 194 ---
.../basic_configs/conf/lang/stopwords_pt.txt | 253 ----
.../basic_configs/conf/lang/stopwords_ro.txt | 233 ---
.../basic_configs/conf/lang/stopwords_ru.txt | 243 ---
.../basic_configs/conf/lang/stopwords_sv.txt | 133 --
.../basic_configs/conf/lang/stopwords_th.txt | 119 --
.../basic_configs/conf/lang/stopwords_tr.txt | 212 ---
.../basic_configs/conf/lang/userdict_ja.txt | 29 -
.../basic_configs/conf/managed-schema | 1052 -------------
.../configsets/basic_configs/conf/params.json | 20 -
.../configsets/basic_configs/conf/protwords.txt | 21 -
.../basic_configs/conf/solrconfig.xml | 1401 ------------------
.../configsets/basic_configs/conf/stopwords.txt | 14 -
.../configsets/basic_configs/conf/synonyms.txt | 29 -
.../conf/currency.xml | 67 -
.../data_driven_schema_configs/conf/elevate.xml | 42 -
.../conf/lang/contractions_ca.txt | 8 -
.../conf/lang/contractions_fr.txt | 15 -
.../conf/lang/contractions_ga.txt | 5 -
.../conf/lang/contractions_it.txt | 23 -
.../conf/lang/hyphenations_ga.txt | 5 -
.../conf/lang/stemdict_nl.txt | 6 -
.../conf/lang/stoptags_ja.txt | 420 ------
.../conf/lang/stopwords_ar.txt | 125 --
.../conf/lang/stopwords_bg.txt | 193 ---
.../conf/lang/stopwords_ca.txt | 220 ---
.../conf/lang/stopwords_cz.txt | 172 ---
.../conf/lang/stopwords_da.txt | 110 --
.../conf/lang/stopwords_de.txt | 294 ----
.../conf/lang/stopwords_el.txt | 78 -
.../conf/lang/stopwords_en.txt | 54 -
.../conf/lang/stopwords_es.txt | 356 -----
.../conf/lang/stopwords_eu.txt | 99 --
.../conf/lang/stopwords_fa.txt | 313 ----
.../conf/lang/stopwords_fi.txt | 97 --
.../conf/lang/stopwords_fr.txt | 186 ---
.../conf/lang/stopwords_ga.txt | 110 --
.../conf/lang/stopwords_gl.txt | 161 --
.../conf/lang/stopwords_hi.txt | 235 ---
.../conf/lang/stopwords_hu.txt | 211 ---
.../conf/lang/stopwords_hy.txt | 46 -
.../conf/lang/stopwords_id.txt | 359 -----
.../conf/lang/stopwords_it.txt | 303 ----
.../conf/lang/stopwords_ja.txt | 127 --
.../conf/lang/stopwords_lv.txt | 172 ---
.../conf/lang/stopwords_nl.txt | 119 --
.../conf/lang/stopwords_no.txt | 194 ---
.../conf/lang/stopwords_pt.txt | 253 ----
.../conf/lang/stopwords_ro.txt | 233 ---
.../conf/lang/stopwords_ru.txt | 243 ---
.../conf/lang/stopwords_sv.txt | 133 --
.../conf/lang/stopwords_th.txt | 119 --
.../conf/lang/stopwords_tr.txt | 212 ---
.../conf/lang/userdict_ja.txt | 29 -
.../conf/managed-schema | 1076 --------------
.../data_driven_schema_configs/conf/params.json | 20 -
.../conf/protwords.txt | 21 -
.../conf/solrconfig.xml | 1398 -----------------
.../conf/stopwords.txt | 14 -
.../conf/synonyms.txt | 29 -
solr/solr-ref-guide/src/collections-api.adoc | 1044 +++++++------
solr/solr-ref-guide/src/coreadmin-api.adoc | 276 ++--
solr/solr-ref-guide/src/format-of-solr-xml.adoc | 174 ++-
solr/solr-ref-guide/src/function-queries.adoc | 410 +++--
solr/solr-ref-guide/src/highlighting.adoc | 299 ++--
solr/solr-ref-guide/src/other-parsers.adoc | 169 ++-
solr/solr-ref-guide/src/query-re-ranking.adoc | 15 +-
.../src/rule-based-authorization-plugin.adoc | 23 +-
solr/solr-ref-guide/src/schema-api.adoc | 361 ++---
.../src/solr-control-script-reference.adoc | 520 ++++---
.../solr/cloud/autoscaling/package-info.java | 3 +-
.../solr/common/cloud/ZkConfigManager.java | 2 +-
.../collection1/conf/schema-replication1.xml | 2 +-
.../solrj/solr/collection1/conf/schema-sql.xml | 20 +-
.../solrj/solr/collection1/conf/schema.xml | 20 +-
.../solrj/solr/configsets/ml/conf/schema.xml | 20 +-
.../solr/configsets/shared/conf/schema.xml | 2 +-
.../solr/configsets/streaming/conf/schema.xml | 20 +-
.../solrj/solr/multicore/core0/conf/schema.xml | 2 +-
.../solrj/solr/multicore/core1/conf/schema.xml | 2 +-
.../client/solrj/SolrSchemalessExampleTest.java | 2 +-
.../client/solrj/io/stream/JDBCStreamTest.java | 2 +
.../solr/client/solrj/request/SchemaTest.java | 23 +-
.../cloud/AbstractFullDistribZkTestBase.java | 22 +-
.../org/apache/solr/util/ExternalPaths.java | 4 +-
243 files changed, 11060 insertions(+), 19466 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e863d0f5/solr/CHANGES.txt
----------------------------------------------------------------------
diff --cc solr/CHANGES.txt
index a700f71,8d5c321..d614519
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@@ -175,9 -170,11 +180,14 @@@ New Feature
* SOLR-10406: v2 API error messages list the URL request path as /solr/____v2/... when the original path was /v2/... (Cao Manh Dat, noble)
+ * SOLR-10574: New _default config set replacing basic_configs and data_driven_schema_configs.
- (Ishan Chattopadhyaya, noble, shalin, hossman, David Smiley, Jan Hoydahl, Alexandre Rafalovich)
++ (Ishan Chattopadhyaya, noble, shalin, hossman, David Smiley, Jan Hoydahl, Alexandre Rafalovich)
+
+ * SOLR-10272: Use _default config set if no collection.configName is specified with CREATE (Ishan Chattopadhyaya)
+
+* SOLR-10496: New ComputePlanAction for autoscaling which uses the policy framework to compute cluster
+ operations upon a trigger fire. (Noble Paul, shalin)
+
Bug Fixes
----------------------
* SOLR-9262: Connection and read timeouts are being ignored by UpdateShardHandler after SOLR-4509.
@@@ -212,13 -209,8 +222,15 @@@
thus disabling the global check, and replaces it with specific checks where desired via
QueryUtils.build(). (yonik)
+ * SOLR-10948: Fix extraction component to treat DatePointField the same as TrieDateField (hossman)
+
+* SOLR-10602: Triggers should be able to restore state from old instances when taking over. (shalin)
+
+* SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling
+ config watcher is fired. (shalin)
+
+* SOLR-10738: TriggerAction is initialised even if the trigger is never scheduled. (shalin)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e863d0f5/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
index 3d89fd2,0c87658..ee5cc47
--- a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
@@@ -51,10 -48,9 +52,11 @@@ import org.apache.solr.common.util.Cont
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
+ import org.apache.solr.handler.admin.ConfigSetsHandlerApi;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e863d0f5/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/ZkController.java
index c3878e1,391f29c..9c552be
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@@ -55,8 -57,27 +59,19 @@@ import org.apache.solr.cloud.overseer.O
import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.BeforeReconnect;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.ClusterStateUtil;
-import org.apache.solr.common.cloud.DefaultConnectionStrategy;
-import org.apache.solr.common.cloud.DefaultZkACLProvider;
-import org.apache.solr.common.cloud.DefaultZkCredentialsProvider;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.OnReconnect;
-import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.*;
import org.apache.solr.common.cloud.Replica.Type;
+ import org.apache.solr.common.cloud.Slice;
+ import org.apache.solr.common.cloud.SolrZkClient;
+ import org.apache.solr.common.cloud.ZkACLProvider;
+ import org.apache.solr.common.cloud.ZkCmdExecutor;
+ import org.apache.solr.common.cloud.ZkConfigManager;
+ import org.apache.solr.common.cloud.ZkCoreNodeProps;
+ import org.apache.solr.common.cloud.ZkCredentialsProvider;
+ import org.apache.solr.common.cloud.ZkMaintenanceUtils;
+ import org.apache.solr.common.cloud.ZkNodeProps;
+ import org.apache.solr.common.cloud.ZkStateReader;
+ import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e863d0f5/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e863d0f5/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------