You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2019/10/29 16:35:23 UTC
[lucene-solr] 01/02: SOLR-13101: merge 8.3 branch
This is an automated email from the ASF dual-hosted git repository.
yonik pushed a commit to branch jira/SOLR-13101
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 619dde553a0510a5380691bbe6037d0dcdee46c3
Merge: 57245e9 89996f5
Author: yonik <yo...@apache.org>
AuthorDate: Tue Oct 29 10:46:31 2019 -0400
SOLR-13101: merge 8.3 branch
dev-tools/maven/pom.xml.template | 2 +-
dev-tools/scripts/jenkins.build.ref.guide.sh | 9 +-
lucene/CHANGES.txt | 8 +
.../analysis/ja/dict/TokenInfoDictionary$fst.dat | Bin 1698570 -> 1698570 bytes
.../analysis/ko/dict/TokenInfoDictionary$fst.dat | Bin 5641400 -> 5640903 bytes
.../org/apache/lucene/search/BooleanQuery.java | 14 +-
.../java/org/apache/lucene/util/fst/Builder.java | 3 -
.../src/java/org/apache/lucene/util/fst/FST.java | 110 +----
.../org/apache/lucene/search/TestQueryVisitor.java | 18 +
.../lucene/search/grouping/TopGroupsTest.java | 235 ++++++++++
lucene/ivy-versions.properties | 19 +-
.../search/join/GlobalOrdinalsWithScoreQuery.java | 7 +
solr/CHANGES.txt | 25 +-
.../solr/handler/dataimport/DataImportHandler.java | 8 +-
.../org/apache/solr/cloud/ReplicateFromLeader.java | 2 +-
.../java/org/apache/solr/cloud/ZkController.java | 4 +-
.../solr/cloud/api/collections/CreateShardCmd.java | 10 +-
.../solr/cloud/api/collections/MoveReplicaCmd.java | 8 +-
.../OverseerCollectionMessageHandler.java | 8 +-
.../solr/cloud/api/collections/SplitShardCmd.java | 7 +-
.../java/org/apache/solr/core/BlobRepository.java | 2 +-
.../java/org/apache/solr/core/CoreContainer.java | 87 ++--
.../org/apache/solr/core/HdfsDirectoryFactory.java | 15 +-
.../src/java/org/apache/solr/core/PluginBag.java | 18 +-
.../src/java/org/apache/solr/core/SolrCore.java | 70 +--
.../java/org/apache/solr/core/SolrInfoBean.java | 6 +
.../apache/solr/filestore/DistribPackageStore.java | 495 ---------------------
.../org/apache/solr/filestore/PackageStore.java | 122 -----
.../org/apache/solr/filestore/PackageStoreAPI.java | 273 ------------
.../org/apache/solr/filestore/package-info.java | 21 -
.../apache/solr/handler/ReplicationHandler.java | 47 +-
.../apache/solr/handler/RequestHandlerBase.java | 49 +-
.../solr/handler/admin/CoreAdminHandler.java | 9 +-
.../solr/handler/admin/CoreAdminOperation.java | 2 +
.../solr/handler/component/SuggestComponent.java | 23 +-
.../apache/solr/metrics/SolrCoreMetricManager.java | 47 +-
.../org/apache/solr/metrics/SolrMetricManager.java | 28 +-
.../apache/solr/metrics/SolrMetricProducer.java | 82 +++-
.../apache/solr/metrics/SolrMetricsContext.java | 114 +++++
.../java/org/apache/solr/search/CaffeineCache.java | 18 +-
.../java/org/apache/solr/search/FastLRUCache.java | 55 ++-
.../src/java/org/apache/solr/search/LFUCache.java | 27 +-
.../src/java/org/apache/solr/search/LRUCache.java | 37 +-
.../src/java/org/apache/solr/search/SolrCache.java | 4 +-
.../org/apache/solr/search/SolrCacheHolder.java | 12 +-
.../org/apache/solr/search/SolrFieldCacheBean.java | 16 +-
.../org/apache/solr/search/SolrIndexSearcher.java | 65 +--
.../java/org/apache/solr/security/AuditEvent.java | 22 +-
.../apache/solr/security/AuditLoggerPlugin.java | 45 +-
.../apache/solr/security/AuthenticationPlugin.java | 61 ++-
.../solr/security/PermissionNameProvider.java | 2 -
.../java/org/apache/solr/servlet/HttpSolrCall.java | 52 ++-
.../apache/solr/servlet/SolrDispatchFilter.java | 3 +-
.../apache/solr/servlet/SolrRequestParsers.java | 4 +-
.../org/apache/solr/store/blockcache/Metrics.java | 20 +-
.../solr/store/hdfs/HdfsLocalityReporter.java | 21 +-
.../apache/solr/update/DirectUpdateHandler2.java | 67 +--
.../org/apache/solr/update/SolrIndexWriter.java | 38 +-
.../java/org/apache/solr/update/UpdateHandler.java | 6 -
.../org/apache/solr/update/UpdateShardHandler.java | 25 +-
.../processor/DistributedUpdateProcessor.java | 12 +-
.../processor/DistributedZkUpdateProcessor.java | 58 ++-
.../src/java/org/apache/solr/util/CryptoKeys.java | 47 +-
.../stats/InstrumentedHttpListenerFactory.java | 17 +-
...rumentedPoolingHttpClientConnectionManager.java | 30 +-
.../core/src/test-files/cryptokeys/priv_key512.pem | 9 -
solr/core/src/test-files/cryptokeys/pub_key512.der | Bin 94 -> 0 bytes
.../test-files/runtimecode/runtimelibs_v3.jar.bin | Bin 7337 -> 0 bytes
solr/core/src/test-files/runtimecode/sig.txt | 105 -----
.../org/apache/solr/cloud/MoveReplicaTest.java | 49 +-
.../solr/cloud/TestQueryingOnDownCollection.java | 151 +++++++
.../autoscaling/sim/TestSnapshotCloudManager.java | 10 +-
.../solr/filestore/TestDistribPackageStore.java | 250 -----------
.../solr/handler/admin/MetricsHandlerTest.java | 146 +++++-
.../org/apache/solr/search/TestCaffeineCache.java | 1 +
.../org/apache/solr/search/TestFastLRUCache.java | 15 +-
.../test/org/apache/solr/search/TestLFUCache.java | 2 +-
.../test/org/apache/solr/search/TestLRUCache.java | 9 +-
.../solr/security/AuditLoggerIntegrationTest.java | 5 +-
.../solr/security/BasicAuthIntegrationTest.java | 6 +-
.../solr/security/BasicAuthOnSingleNodeTest.java | 12 +-
.../asciidoctor-ant-1.6.0-alpha.5.jar.sha1 | 1 -
solr/licenses/asciidoctor-ant-1.6.2.jar.sha1 | 1 +
solr/licenses/netty-all-4.0.52.Final.jar.sha1 | 1 -
solr/licenses/netty-all-4.1.29.Final.jar.sha1 | 1 +
solr/licenses/netty-buffer-4.1.29.Final.jar.sha1 | 1 +
solr/licenses/netty-buffer-LICENSE-ASL.txt | 202 +++++++++
solr/licenses/netty-buffer-NOTICE.txt | 223 ++++++++++
solr/licenses/netty-codec-4.1.29.Final.jar.sha1 | 1 +
solr/licenses/netty-codec-LICENSE-ASL.txt | 202 +++++++++
solr/licenses/netty-codec-NOTICE.txt | 223 ++++++++++
solr/licenses/netty-common-4.1.29.Final.jar.sha1 | 1 +
solr/licenses/netty-common-LICENSE-ASL.txt | 202 +++++++++
solr/licenses/netty-common-NOTICE.txt | 223 ++++++++++
solr/licenses/netty-handler-4.1.29.Final.jar.sha1 | 1 +
solr/licenses/netty-handler-LICENSE-ASL.txt | 202 +++++++++
solr/licenses/netty-handler-NOTICE.txt | 223 ++++++++++
solr/licenses/netty-resolver-4.1.29.Final.jar.sha1 | 1 +
solr/licenses/netty-resolver-LICENSE-ASL.txt | 202 +++++++++
solr/licenses/netty-resolver-NOTICE.txt | 223 ++++++++++
.../licenses/netty-transport-4.1.29.Final.jar.sha1 | 1 +
solr/licenses/netty-transport-LICENSE-ASL.txt | 202 +++++++++
solr/licenses/netty-transport-NOTICE.txt | 223 ++++++++++
...ty-transport-native-epoll-4.1.29.Final.jar.sha1 | 1 +
.../netty-transport-native-epoll-LICENSE-ASL.txt | 202 +++++++++
.../netty-transport-native-epoll-NOTICE.txt | 223 ++++++++++
...nsport-native-unix-common-4.1.29.Final.jar.sha1 | 1 +
...ty-transport-native-unix-common-LICENSE-ASL.txt | 202 +++++++++
.../netty-transport-native-unix-common-NOTICE.txt | 223 ++++++++++
solr/solr-ref-guide/README.adoc | 26 +-
solr/solr-ref-guide/src/_config.yml.template | 4 +-
.../adding-custom-plugins-in-solrcloud-mode.adoc | 13 +-
solr/solr-ref-guide/src/aliases.adoc | 6 +-
.../authentication-and-authorization-plugins.adoc | 10 +-
.../src/basic-authentication-plugin.adoc | 2 +-
.../src/cluster-node-management.adoc | 2 +-
.../src/common-query-parameters.adoc | 2 +-
solr/solr-ref-guide/src/enabling-ssl.adoc | 2 +-
solr/solr-ref-guide/src/json-facet-api.adoc | 1 -
.../src/jwt-authentication-plugin.adoc | 2 +-
.../src/kerberos-authentication-plugin.adoc | 2 +-
solr/solr-ref-guide/src/metrics-reporting.adoc | 6 +-
...onitoring-solr-with-prometheus-and-grafana.adoc | 6 +-
.../src/query-settings-in-solrconfig.adoc | 3 +-
.../src/resource-and-plugin-loading.adoc | 2 +-
.../src/rule-based-authorization-plugin.adoc | 2 +-
.../setting-up-an-external-zookeeper-ensemble.adoc | 6 +-
.../src/solr-control-script-reference.adoc | 14 +-
solr/solr-ref-guide/src/solr-tracing.adoc | 2 +-
solr/solr-ref-guide/src/solr-tutorial.adoc | 2 +-
solr/solr-ref-guide/src/solr-upgrade-notes.adoc | 65 +++
.../src/solrcloud-autoscaling-api.adoc | 12 +-
.../solrcloud-autoscaling-policy-preferences.adoc | 3 +-
.../src/solrcloud-autoscaling-triggers.adoc | 10 +-
.../src/taking-solr-to-production.adoc | 18 +-
.../src/updating-parts-of-documents.adoc | 2 +-
...ng-zookeeper-to-manage-configuration-files.adoc | 2 +-
solr/solrj/ivy.xml | 9 +
.../org/apache/solr/client/solrj/SolrResponse.java | 9 +-
.../solr/client/solrj/impl/CloudSolrClient.java | 9 +-
.../solr/client/solrj/request/V2Request.java | 39 +-
.../apache/solr/common/params/CommonParams.java | 3 -
.../org/apache/solr/common/util/ExecutorUtil.java | 1 +
.../java/org/apache/solr/common/util/StrUtils.java | 207 ++++-----
.../java/org/apache/solr/common/util/Utils.java | 144 ++----
.../org/apache/solr/cloud/SolrCloudTestCase.java | 4 +-
146 files changed, 5194 insertions(+), 2411 deletions(-)
diff --cc solr/core/src/java/org/apache/solr/cloud/ZkController.java
index ea6886f,421480d..0bf1ec7
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@@ -69,7 -58,9 +68,8 @@@ import org.apache.solr.client.solrj.imp
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.impl.SolrClientCloudManager;
+ import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.common.AlreadyClosedException;
diff --cc solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java
index 83ff70c,bb1081d..994a3e7
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateShardCmd.java
@@@ -70,33 -69,21 +70,36 @@@ public class CreateShardCmd implements
}
DocCollection collection = clusterState.getCollection(collectionName);
- int numNrtReplicas = message.getInt(NRT_REPLICAS, message.getInt(REPLICATION_FACTOR, collection.getInt(NRT_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1))));
+ boolean isSharedIndex = collection.getSharedIndex();
+
+ // We evaluate all the default values each time, even when they're not needed (performance impact?).
+ int numNrtReplicas = message.getInt(NRT_REPLICAS, isSharedIndex ? 0 : message.getInt(REPLICATION_FACTOR, collection.getInt(NRT_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1))));
int numPullReplicas = message.getInt(PULL_REPLICAS, collection.getInt(PULL_REPLICAS, 0));
int numTlogReplicas = message.getInt(TLOG_REPLICAS, collection.getInt(TLOG_REPLICAS, 0));
+ int numSharedReplicas = message.getInt(SHARED_REPLICAS, isSharedIndex ? message.getInt(REPLICATION_FACTOR, collection.getInt(SHARED_REPLICAS, collection.getInt(REPLICATION_FACTOR, 1))) : 0);
- if (numNrtReplicas + numTlogReplicas <= 0) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " must be greater than 0");
+ if (numNrtReplicas + numTlogReplicas + numSharedReplicas <= 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " + " + SHARED_REPLICAS + " must be greater than 0");
+ }
+
+ if (numSharedReplicas > 0 && numNrtReplicas + numTlogReplicas > 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can't have both non 0 " + SHARED_REPLICAS + " (" + numSharedReplicas
+ + ") and non 0 " + NRT_REPLICAS + "+" + TLOG_REPLICAS + " (" + numNrtReplicas + "+" + numTlogReplicas + ")");
+ }
+
+ if (isSharedIndex) {
+ String sharedShardName = Assign.buildSharedShardName(collectionName, sliceName);
+ // this is a bit inefficient since we do a full copy of the properties
+ message = message.plus(ZkStateReader.SHARED_SHARD_NAME, sharedShardName);
}
- ZkStateReader zkStateReader = ocmh.zkStateReader;
+ //ZkStateReader zkStateReader = ocmh.zkStateReader;
ocmh.overseer.offerStateUpdate(Utils.toJSON(message));
// wait for a while until we see the shard
- ocmh.waitForNewShard(collectionName, sliceName);
+ //ocmh.waitForNewShard(collectionName, sliceName);
+ // wait for a while until we see the shard and update the local view of the cluster state
+ clusterState = ocmh.waitForNewShard(collectionName, sliceName);
+
String async = message.getStr(ASYNC);
ZkNodeProps addReplicasProps = new ZkNodeProps(
COLLECTION_PROP, collectionName,
diff --cc solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 40dd864,32329ff..2c2f1d0
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@@ -318,17 -308,10 +318,14 @@@ public class SplitShardCmd implements O
propMap.put("shard_parent_node", nodeName);
propMap.put("shard_parent_zk_session", leaderZnodeStat.getEphemeralOwner());
+ if (collection.getSharedIndex()) {
+ propMap.put(ZkStateReader.SHARED_SHARD_NAME, Assign.buildSharedShardName(collectionName, subSlice));
+ }
+
ocmh.overseer.offerStateUpdate(Utils.toJSON(new ZkNodeProps(propMap)));
- // wait until we are able to see the new shard in cluster state
- ocmh.waitForNewShard(collectionName, subSlice);
-
- // refresh cluster state
- clusterState = zkStateReader.getClusterState();
+ // wait until we are able to see the new shard in cluster state and refresh the local view of the cluster state
+ clusterState = ocmh.waitForNewShard(collectionName, subSlice);
log.debug("Adding first replica " + subShardName + " as part of slice " + subSlice + " of collection " + collectionName
+ " on " + nodeName);
diff --cc solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 6cdc742,4e40fcb..06f1fc8
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@@ -221,11 -220,7 +222,9 @@@ public class CoreContainer
protected volatile MetricsCollectorHandler metricsCollectorHandler;
protected volatile AutoscalingHistoryHandler autoscalingHistoryHandler;
+
+ protected SharedStoreManager sharedStoreManager;
- private PackageStoreAPI packageStoreAPI;
-
// Bits for the state variable.
public final static long LOAD_COMPLETE = 0x1L;
diff --cc solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index 9e30034,3c3e26f..8c5847d
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@@ -78,8 -77,8 +78,9 @@@ import org.apache.solr.metrics.AltBuffe
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.OperatingSystemMetricSet;
import org.apache.solr.metrics.SolrMetricManager;
+ import org.apache.solr.metrics.SolrMetricProducer;
import org.apache.solr.security.AuditEvent;
+import org.apache.solr.security.AuditEvent.EventType;
import org.apache.solr.security.AuthenticationPlugin;
import org.apache.solr.security.PKIAuthenticationPlugin;
import org.apache.solr.security.PublicKeyHandler;
diff --cc solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index ff3f024,569f877..d3c1087
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@@ -112,12 -106,6 +112,12 @@@ public class DistributedZkUpdateProcess
public DistributedZkUpdateProcessor(SolrQueryRequest req,
SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ this(req,rsp,next, new CoreUpdateTracker(req.getCore().getCoreContainer()));
+ }
-
++
+ @VisibleForTesting
+ protected DistributedZkUpdateProcessor(SolrQueryRequest req,
+ SolrQueryResponse rsp, UpdateRequestProcessor next, CoreUpdateTracker sharedCoreTracker) {
super(req, rsp, next);
CoreContainer cc = req.getCore().getCoreContainer();
cloudDesc = req.getCore().getCoreDescriptor().getCloudDescriptor();
@@@ -231,8 -207,7 +231,8 @@@
params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
params.set(COMMIT_END_POINT, "replicas");
-
+
+
useNodes = getReplicaNodesForLeader(cloudDesc.getShardId(), leaderReplica);
if (useNodes != null) {
@@@ -1068,7 -1032,6 +1068,7 @@@
if (isReadOnly()) {
throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only.");
}
-
++
super.processMergeIndexes(cmd);
}
@@@ -1082,42 -1045,10 +1082,35 @@@
super.processRollback(cmd);
}
- @Override
- public void finish() throws IOException {
- clusterState = zkController.getClusterState();
-
- assertNotFinished();
-
- doFinish();
- }
-
+ private void writeToShareStore() throws SolrException {
- log.info("Attempting to initiate index update write to shared store for collection=" + cloudDesc.getCollectionName() +
++ log.info("Attempting to initiate index update write to shared store for collection=" + cloudDesc.getCollectionName() +
+ " and shard=" + cloudDesc.getShardId() + " using core=" + req.getCore().getName());
-
- sharedCoreTracker.persistShardIndexToSharedStore(zkController.zkStateReader.getClusterState(),
- cloudDesc.getCollectionName(),
- cloudDesc.getShardId(),
++
++ sharedCoreTracker.persistShardIndexToSharedStore(zkController.zkStateReader.getClusterState(),
++ cloudDesc.getCollectionName(),
++ cloudDesc.getShardId(),
+ req.getCore().getName());
+ }
-
++
+ private void readFromSharedStoreIfNecessary() throws SolrException {
+ String coreName = req.getCore().getName();
+ String shardName = cloudDesc.getShardId();
+ String collectionName = cloudDesc.getCollectionName();
+ assertEquals(replicaType, Replica.Type.SHARED);
+ // Peers and subShardLeaders should only forward the update request to leader replica,
+ // hence not need to sync with the blob store at this point.
+ if (!isLeader || isSubShardLeader) {
+ return;
+ }
+ BlobStoreUtils.syncLocalCoreWithSharedStore(collectionName,coreName,shardName,req.getCore().getCoreContainer());
+ }
-
-
++
++
+
// TODO: optionally fail if n replicas are not reached...
- private void doFinish() {
+ protected void doDistribFinish() {
+ clusterState = zkController.getClusterState();
+
boolean shouldUpdateTerms = isLeader && isIndexChanged;
if (shouldUpdateTerms) {
ZkShardTerms zkShardTerms = zkController.getShardTerms(cloudDesc.getCollectionName(), cloudDesc.getShardId());
@@@ -1126,34 -1057,10 +1119,34 @@@
}
zkController.getShardTerms(collection, cloudDesc.getShardId()).ensureHighestTermsAreNotZero();
}
-
++
+ /**
- * Track the updated core for push to Blob store.
- *
- * Only, the leader node pushes the updates to blob store but the leader can be change mid update,
++ * Track the updated core for push to Blob store.
++ *
++ * Only, the leader node pushes the updates to blob store but the leader can be change mid update,
+ * so we don't stop peers from pushing updates to the blob store.
- *
- * We also need to check for isLeader here because a peer can also receive commit message if the request was directly send to the peer.
++ *
++ * We also need to check for isLeader here because a peer can also receive commit message if the request was directly send to the peer.
+ */
+ if ( updateCommand != null &&
- updateCommand.getClass() == CommitUpdateCommand.class &&
++ updateCommand.getClass() == CommitUpdateCommand.class &&
+ isLeader && replicaType.equals(Replica.Type.SHARED)
+ && !((CommitUpdateCommand) updateCommand).softCommit) {
+ /*
- * TODO SPLITSHARD triggers soft commits.
++ * TODO SPLITSHARD triggers soft commits.
+ * We don't persist on softCommit because there is nothing to so we should ignore those kinds of commits.
+ * Configuring behavior based on soft/hard commit seems like we're getting into an abstraction deeper then
+ * what the DUP is concerned about so we may want to consider moving this code somewhere more appropriate
- * in the future (deeper in the stack)
++ * in the future (deeper in the stack)
+ */
+ writeToShareStore();
+ }
-
++
// TODO: if not a forward and replication req is not specified, we could
// send in a background thread
cmdDistrib.finish();
-
++
List<SolrCmdDistributor.Error> errors = cmdDistrib.getErrors();
// TODO - we may need to tell about more than one error...