You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/07/23 02:33:39 UTC
lucene-solr:master: SOLR-12305: When a replica is applying updates,
some kind of updates can skip buffering for faster recovery
Repository: lucene-solr
Updated Branches:
refs/heads/master 6e3f61f6f -> 8c06dec16
SOLR-12305: When a replica is applying updates, some kind of updates can skip buffering for faster recovery
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8c06dec1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8c06dec1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8c06dec1
Branch: refs/heads/master
Commit: 8c06dec16dd711c097f4f8eaa7d9df041e0cb563
Parents: 6e3f61f
Author: Cao Manh Dat <da...@apache.org>
Authored: Mon Jul 23 09:33:28 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Mon Jul 23 09:33:28 2018 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../processor/DistributedUpdateProcessor.java | 19 +++++--
.../DistributedUpdateProcessorTest.java | 54 ++++++++++++++++++++
3 files changed, 72 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8c06dec1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e6d468c..e93e0e7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -173,6 +173,9 @@ Optimizations
which we can export is directly proportional to the number of sort fields specified. This change also allows streaming
expressions to group by on more than 4 fields. (Aroop Ganguly, Amrit Sarkar, Varun Thacker)
+* SOLR-12305: When a replica is applying updates, some kind of updates can skip buffering for faster recovery.
+ (Cao Manh Dat)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8c06dec1/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index 8f87510..8d715a6 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -31,6 +31,7 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.solr.client.solrj.SolrRequest;
@@ -1083,7 +1084,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// The leader forwarded us this update.
cmd.setVersion(versionOnUpdate);
- if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {
+ if (shouldBufferUpdate(cmd, isReplayOrPeersync, ulog.getState())) {
// we're not in an active state, and this update isn't from a replay, so buffer it.
cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
ulog.add(cmd);
@@ -1132,9 +1133,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
}
}
- }
-
- if (!cmd.isInPlaceUpdate()) {
+ } else {
// if we aren't the leader, then we need to check that updates were not re-ordered
if (bucketVersion != 0 && bucketVersion < versionOnUpdate) {
// we're OK... this update has a version higher than anything we've seen
@@ -1178,6 +1177,18 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
return false;
}
+ @VisibleForTesting
+ boolean shouldBufferUpdate(AddUpdateCommand cmd, boolean isReplayOrPeersync, UpdateLog.State state) {
+ if (state == UpdateLog.State.APPLYING_BUFFERED
+ && !isReplayOrPeersync
+ && !cmd.isInPlaceUpdate()) {
+ // this a new update sent from the leader, it contains whole document therefore it won't depend on other updates
+ return false;
+ }
+
+ return state != UpdateLog.State.ACTIVE && isReplayOrPeersync == false;
+ }
+
/**
* This method checks the update/transaction logs and index to find out if the update ("previous update") that the current update
* depends on (in the case that this current update is an in-place update) has already been completed. If not,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8c06dec1/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java
new file mode 100644
index 0000000..8f56d68
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.UpdateLog;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class DistributedUpdateProcessorTest extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solr/collection1/conf/solrconfig.xml","solr/collection1/conf/schema-minimal.xml");
+ }
+
+ @Test
+ public void testShouldBufferUpdate() {
+ SolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), new ModifiableSolrParams());
+ DistributedUpdateProcessor processor = new DistributedUpdateProcessor(
+ req, null, null, null);
+
+ AddUpdateCommand cmd = new AddUpdateCommand(req);
+ // applying buffer updates, isReplayOrPeerSync flag doesn't matter
+ assertFalse(processor.shouldBufferUpdate(cmd, false, UpdateLog.State.APPLYING_BUFFERED));
+ assertFalse(processor.shouldBufferUpdate(cmd, true, UpdateLog.State.APPLYING_BUFFERED));
+
+ assertTrue(processor.shouldBufferUpdate(cmd, false, UpdateLog.State.BUFFERING));
+ // this is not an buffer updates and it depend on other updates
+ cmd.prevVersion = 10;
+ assertTrue(processor.shouldBufferUpdate(cmd, false, UpdateLog.State.APPLYING_BUFFERED));
+ }
+
+
+}