You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2018/04/01 13:48:13 UTC
lucene-solr:master: SOLR-11673: Slave doesn't commit empty index if
new index appears on master by deafult.
Repository: lucene-solr
Updated Branches:
refs/heads/master d98138ac0 -> 7a920cb0f
SOLR-11673: Slave doesn't commit empty index if new index appears on master by deafult.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7a920cb0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7a920cb0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7a920cb0
Branch: refs/heads/master
Commit: 7a920cb0f64ee3de058a980e7564685cdc311c06
Parents: d98138a
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Sun Apr 1 16:47:23 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Sun Apr 1 16:47:23 2018 +0300
----------------------------------------------------------------------
solr/CHANGES.txt | 7 ++++
.../org/apache/solr/handler/IndexFetcher.java | 35 ++++++++++++++++----
.../org/apache/solr/util/TestInjection.java | 15 +++++++++
.../solr/handler/TestReplicationHandler.java | 7 ++--
4 files changed, 55 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e7349cf..0dfb90b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -61,6 +61,10 @@ Upgrade Notes
This is a server side change only and clients using SolrJ won't need any changes. Clients can still use any logging
implementation which is compatible with SLF4J.
+* SOLR-11673: Slave doesn't commit empty index when completely new index is detected on master during replication.
+ To return the previous behavior pass false to skipCommitOnMasterVersionZero in slave section of replication
+ handler configuration, or pass it to the fetchindex command.
+
New Features
----------------------
@@ -95,6 +99,9 @@ Bug Fixes
document. Multithreaded test for AtomicUpdateRequestProcessor was also beefed up and fixed.
(Ishan Chattopadhyaya, Noble Paul, Amrit Sarkar, shalin)
+* SOLR-11673: By default slave doesn't commit empty index when completely new index appears on master.
+ See Upgrade Notes to find a way to get back to the previous behavior. (Mikhail Khludnev)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 2476a81..3c6859b 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -16,6 +16,31 @@
*/
package org.apache.solr.handler;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.handler.ReplicationHandler.ALIAS;
+import static org.apache.solr.handler.ReplicationHandler.CHECKSUM;
+import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS;
+import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE;
+import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST;
+import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION;
+import static org.apache.solr.handler.ReplicationHandler.COMMAND;
+import static org.apache.solr.handler.ReplicationHandler.COMPRESSION;
+import static org.apache.solr.handler.ReplicationHandler.CONF_FILES;
+import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT;
+import static org.apache.solr.handler.ReplicationHandler.EXTERNAL;
+import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER;
+import static org.apache.solr.handler.ReplicationHandler.FILE;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+import static org.apache.solr.handler.ReplicationHandler.GENERATION;
+import static org.apache.solr.handler.ReplicationHandler.INTERNAL;
+import static org.apache.solr.handler.ReplicationHandler.MASTER_URL;
+import static org.apache.solr.handler.ReplicationHandler.OFFSET;
+import static org.apache.solr.handler.ReplicationHandler.SIZE;
+import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_MASTER_VERSION_ZERO;
+import static org.apache.solr.handler.ReplicationHandler.TLOG_FILE;
+import static org.apache.solr.handler.ReplicationHandler.TLOG_FILES;
+
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -86,7 +111,7 @@ import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.ReplicationHandler.*;
+import org.apache.solr.handler.ReplicationHandler.FileInfo;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
@@ -99,13 +124,10 @@ import org.apache.solr.util.FileUtils;
import org.apache.solr.util.PropertiesOutputStream;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted;
+import org.apache.solr.util.TestInjection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.params.CommonParams.JAVABIN;
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.handler.ReplicationHandler.*;
-
import com.google.common.base.Strings;
/**
@@ -167,7 +189,7 @@ public class IndexFetcher {
private boolean downloadTlogFiles = false;
- private boolean skipCommitOnMasterVersionZero;
+ private boolean skipCommitOnMasterVersionZero = true;
private static final String INTERRUPT_RESPONSE_MESSAGE = "Interrupted while waiting for modify lock";
@@ -453,6 +475,7 @@ public class IndexFetcher {
} finally {
iw.decref();
}
+ assert TestInjection.injectDelayBeforeSlaveCommitRefresh();
if (skipCommitOnMasterVersionZero) {
openNewSearcherAndUpdateCommitPoint();
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/java/org/apache/solr/util/TestInjection.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java
index 422de73..821b37e 100644
--- a/solr/core/src/java/org/apache/solr/util/TestInjection.java
+++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java
@@ -144,6 +144,8 @@ public class TestInjection {
private static AtomicInteger countPrepRecoveryOpPauseForever = new AtomicInteger(0);
+ public static Integer delayBeforeSlaveCommitRefresh=null;
+
public static void reset() {
nonGracefullClose = null;
failReplicaRequests = null;
@@ -158,6 +160,7 @@ public class TestInjection {
waitForReplicasInSync = "true:60";
failIndexFingerprintRequests = null;
wrongIndexFingerprint = null;
+ delayBeforeSlaveCommitRefresh = null;
for (Timer timer : timers) {
timer.cancel();
@@ -455,4 +458,16 @@ public class TestInjection {
return new Pair<>(Boolean.parseBoolean(val), Integer.parseInt(percent));
}
+ public static boolean injectDelayBeforeSlaveCommitRefresh() {
+ if (delayBeforeSlaveCommitRefresh!=null) {
+ try {
+ log.info("Pausing IndexFetcher for {}ms", delayBeforeSlaveCommitRefresh);
+ Thread.sleep(delayBeforeSlaveCommitRefresh);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return true;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a920cb0/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index ddc5b60..e8caf99 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -70,6 +70,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.StandardDirectoryFactory;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.util.FileUtils;
+import org.apache.solr.util.TestInjection;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -494,8 +495,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-11673")
public void doTestIndexAndConfigReplication() throws Exception {
+
+ TestInjection.delayBeforeSlaveCommitRefresh = random().nextInt(10);
+
clearIndexWithReplication();
nDocs--;
@@ -550,7 +553,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
slaveJetty = createJetty(slave);
slaveClient.close();
slaveClient = createNewSolrClient(slaveJetty.getLocalPort());
-
//add a doc with new field and commit on master to trigger index fetch from slave.
index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);
masterClient.commit();
@@ -567,7 +569,6 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
checkForSingleIndex(masterJetty);
checkForSingleIndex(slaveJetty, true);
-
}
@Test