You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2017/01/16 21:38:33 UTC
[1/2] lucene-solr:branch_6x: SOLR-9941: Clear deletes lists before
log replay
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 2d705e997 -> 7ef8cf7d6
SOLR-9941: Clear deletes lists before log replay
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/302ce326
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/302ce326
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/302ce326
Branch: refs/heads/branch_6x
Commit: 302ce326d5a2eee043445918fa3e3885dc003b2f
Parents: 2d705e9
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Wed Jan 11 21:51:04 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Tue Jan 17 03:04:20 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../java/org/apache/solr/update/UpdateLog.java | 6 +
.../org/apache/solr/search/TestRecovery.java | 127 +++++++++++++++++++
3 files changed, 136 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/302ce326/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4a71c28..c5aad1e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -33,6 +33,9 @@ Jetty 9.3.14.v20161028
(No Changes)
+* SOLR-9941: Clear the deletes lists at UpdateLog before replaying from log. This prevents redundantly pre-applying
+ DBQs, during the log replay, to every update in the log as if the DBQs were out of order. (hossman, Ishan Chattopadhyaya)
+
================== 6.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/302ce326/solr/core/src/java/org/apache/solr/update/UpdateLog.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 6201436..7c2dae6 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -896,6 +896,12 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
versionInfo.blockUpdates();
try {
state = State.REPLAYING;
+
+ // The deleteByQueries and oldDeletes lists
+ // would've been populated by items from the logs themselves (which we
+ // will replay now). So lets clear them out here before the replay.
+ deleteByQueries.clear();
+ oldDeletes.clear();
} finally {
versionInfo.unblockUpdates();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/302ce326/solr/core/src/test/org/apache/solr/search/TestRecovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecovery.java b/solr/core/src/test/org/apache/solr/search/TestRecovery.java
index 63b9723..15aed5d 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecovery.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecovery.java
@@ -25,6 +25,7 @@ import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
import org.apache.solr.metrics.SolrMetricManager;
import org.noggit.ObjectBuilder;
+import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.DirectUpdateHandler2;
@@ -198,6 +199,132 @@ public class TestRecovery extends SolrTestCaseJ4 {
}
@Test
+ public void testNewDBQAndDocMatchingOldDBQDuringLogReplay() throws Exception {
+ try {
+
+ DirectUpdateHandler2.commitOnClose = false;
+ final Semaphore logReplay = new Semaphore(0);
+ final Semaphore logReplayFinish = new Semaphore(0);
+
+ UpdateLog.testing_logReplayHook = () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
+
+ UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
+
+ clearIndex();
+ assertU(commit());
+
+ // because we're sending updates during log replay, we can't emulate replica logic -- we need to use
+ // normal updates like a leader / single-node instance would get.
+ //
+ // (In SolrCloud mode, when a replica run recoverFromLog, replica in this time period will have state = DOWN,
+ // so It won't receive any updates.)
+
+ updateJ(jsonAdd(sdoc("id","B0")),params());
+ updateJ(jsonAdd(sdoc("id","B1")),params()); // should be deleted by subsequent DBQ in tlog
+ updateJ(jsonAdd(sdoc("id","B2")),params()); // should be deleted by DBQ that arives during tlog replay
+ updateJ(jsonDelQ("id:B1 OR id:B3 OR id:B6"),params());
+ updateJ(jsonAdd(sdoc("id","B3")),params()); // should *NOT* be deleted by previous DBQ in tlog
+ updateJ(jsonAdd(sdoc("id","B4")),params()); // should be deleted by DBQ that arives during tlog replay
+ updateJ(jsonAdd(sdoc("id","B5")),params());
+
+ // sanity check no updates have been applied yet (just in tlog)
+ assertJQ(req("q","*:*"),"/response/numFound==0");
+
+ h.close();
+ createCore(); // (Attempts to) kick off recovery (which is currently blocked by semaphore)
+
+ // verify that previous close didn't do a commit & that recovery should be blocked by our hook
+ assertJQ(req("q","*:*") ,"/response/numFound==0");
+
+ // begin recovery (first few items)
+ logReplay.release(TestUtil.nextInt(random(),1,6));
+ // ... but before recover is completely unblocked/finished, have a *new* DBQ arrive
+ // that should delete some items we either have just replayed, or are about to replay (or maybe both)...
+ updateJ(jsonDelQ("id:B2 OR id:B4"),params());
+ // ...and re-add a doc that would have matched a DBQ already in the tlog
+ // (which may/may-not have been replayed yet)
+ updateJ(jsonAdd(sdoc("id","B6")),params()); // should *NOT* be deleted by DBQ from tlog
+ assertU(commit());
+
+ // now completely unblock recovery
+ logReplay.release(1000);
+
+ // wait until recovery has finished
+ assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
+
+ // verify only the expected docs are found, even with out of order DBQ and DBQ that arived during recovery
+ assertJQ(req("q", "*:*", "fl", "id", "sort", "id asc")
+ , "/response/docs==[{'id':'B0'}, {'id':'B3'}, {'id':'B5'}, {'id':'B6'}]");
+
+ } finally {
+ DirectUpdateHandler2.commitOnClose = true;
+ UpdateLog.testing_logReplayHook = null;
+ UpdateLog.testing_logReplayFinishHook = null;
+ }
+
+ }
+
+ @Test
+ public void testLogReplayWithReorderedDBQ() throws Exception {
+ try {
+
+ DirectUpdateHandler2.commitOnClose = false;
+ final Semaphore logReplay = new Semaphore(0);
+ final Semaphore logReplayFinish = new Semaphore(0);
+
+ UpdateLog.testing_logReplayHook = () -> {
+ try {
+ assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ };
+
+ UpdateLog.testing_logReplayFinishHook = () -> logReplayFinish.release();
+
+
+ clearIndex();
+ assertU(commit());
+
+ updateJ(jsonAdd(sdoc("id","B1", "_version_","1010")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(jsonDelQ("id:B2"), params(DISTRIB_UPDATE_PARAM,FROM_LEADER, "_version_","-1017")); // This should've arrived after the 1015th update
+ updateJ(jsonAdd(sdoc("id","B2", "_version_","1015")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ updateJ(jsonAdd(sdoc("id","B3", "_version_","1020")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+
+
+ assertJQ(req("q","*:*"),"/response/numFound==0");
+
+ h.close();
+ createCore();
+ // Solr should kick this off now
+ // h.getCore().getUpdateHandler().getUpdateLog().recoverFromLog();
+
+ // verify that previous close didn't do a commit
+ // recovery should be blocked by our hook
+ assertJQ(req("q","*:*") ,"/response/numFound==0");
+
+ // unblock recovery
+ logReplay.release(1000);
+
+ // wait until recovery has finished
+ assertTrue(logReplayFinish.tryAcquire(timeout, TimeUnit.SECONDS));
+
+ assertJQ(req("q","*:*") ,"/response/numFound==2");
+ } finally {
+ DirectUpdateHandler2.commitOnClose = true;
+ UpdateLog.testing_logReplayHook = null;
+ UpdateLog.testing_logReplayFinishHook = null;
+ }
+
+ }
+
+ @Test
public void testBuffering() throws Exception {
DirectUpdateHandler2.commitOnClose = false;
[2/2] lucene-solr:branch_6x: SOLR-9941: Adding the Optimizations
section to the CHANGES.txt
Posted by is...@apache.org.
SOLR-9941: Adding the Optimizations section to the CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7ef8cf7d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7ef8cf7d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7ef8cf7d
Branch: refs/heads/branch_6x
Commit: 7ef8cf7d6aad25888de4cffc4c20239694a67a45
Parents: 302ce32
Author: Ishan Chattopadhyaya <is...@apache.org>
Authored: Tue Jan 17 03:07:55 2017 +0530
Committer: Ishan Chattopadhyaya <is...@apache.org>
Committed: Tue Jan 17 03:07:55 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7ef8cf7d/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c5aad1e..588451d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -29,9 +29,11 @@ Apache UIMA 2.3.1
Apache ZooKeeper 3.4.6
Jetty 9.3.14.v20161028
+Detailed Change List
+----------------------
-(No Changes)
-
+Optimizations
+----------------------
* SOLR-9941: Clear the deletes lists at UpdateLog before replaying from log. This prevents redundantly pre-applying
DBQs, during the log replay, to every update in the log as if the DBQs were out of order. (hossman, Ishan Chattopadhyaya)