You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/10/26 00:54:47 UTC
[lucene-solr] 01/02: @1046 More tweaks.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 1b7dd66b4c7b2ed780f09222f03f1f1c902ab9d1
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Sun Oct 25 17:52:32 2020 -0500
@1046 More tweaks.
---
.../src/java/org/apache/solr/cloud/Overseer.java | 4 +
.../handler/component/RealTimeGetComponent.java | 3 +-
.../org/apache/solr/update/IndexFingerprint.java | 5 +-
.../src/java/org/apache/solr/update/UpdateLog.java | 96 ++++++++++------------
.../org/apache/solr/search/TestRealTimeGet.java | 14 ++--
.../org/apache/solr/search/TestStressReorder.java | 46 ++++++-----
6 files changed, 81 insertions(+), 87 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index d2836b2..767b2ce 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -934,6 +934,10 @@ public class Overseer implements SolrCloseable {
log.debug("doClose() - start");
}
+ if (closeAndDone) {
+ shardHandler.cancelAll();
+ }
+
if (ccThread != null) {
((OverseerCollectionConfigSetProcessor) ccThread.getThread()).closing();
ccThread.interrupt();
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index b1b3333..e5ecce5 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -452,7 +452,8 @@ public class RealTimeGetComponent extends SearchComponent
*/
private static SolrDocument reopenRealtimeSearcherAndGet(SolrCore core, Term idTerm, ReturnFields returnFields) throws IOException {
UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
- RefCounted<SolrIndexSearcher> searcherHolder = ulog.openRealtimeSearcher(true);
+ ulog.openRealtimeSearcher();
+ RefCounted<SolrIndexSearcher> searcherHolder = core.getRealtimeSearcher();
try {
SolrIndexSearcher searcher = searcherHolder.get();
diff --git a/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java b/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java
index 67503e4..e40f8a2 100644
--- a/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java
+++ b/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java
@@ -91,8 +91,8 @@ public class IndexFingerprint implements MapSerializable {
/** Opens a new realtime searcher and returns it's (possibly cached) fingerprint */
public static IndexFingerprint getFingerprint(SolrCore core, long maxVersion) throws IOException {
RTimer timer = new RTimer();
-
- RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().openRealtimeSearcher(true);
+ core.getUpdateHandler().getUpdateLog().openRealtimeSearcher();
+ RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().uhandler.core.getRealtimeSearcher();
try {
IndexFingerprint f = newestSearcher.get().getIndexFingerprint(maxVersion);
final double duration = timer.stop();
@@ -107,7 +107,6 @@ public class IndexFingerprint implements MapSerializable {
public static IndexFingerprint getFingerprint(SolrIndexSearcher searcher, LeafReaderContext ctx, Long maxVersion)
throws IOException {
-
SchemaField versionField = VersionInfo.getAndCheckVersionField(searcher.getSchema());
ValueSource vs = versionField.getType().getValueSource(versionField, null);
@SuppressWarnings({"rawtypes"})
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 69bf310..50ff617 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -725,40 +725,27 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
- public RefCounted<SolrIndexSearcher> openRealtimeSearcher() {
- return openRealtimeSearcher(false);
- }
-
/** Opens a new realtime searcher and clears the id caches.
* This may also be called when we updates are being buffered (from PeerSync/IndexFingerprint)
- * @return opened searcher if requested
*/
- public RefCounted<SolrIndexSearcher> openRealtimeSearcher(boolean returnSearcher) {
+ public void openRealtimeSearcher() {
// We must cause a new IndexReader to be opened before anything looks at these caches again
// so that a cache miss will read fresh data.
- RefCounted<SolrIndexSearcher> holder = null;
tlogLock.lock();
try {
+ // We must cause a new IndexReader to be opened before anything looks at these caches again
+ // so that a cache miss will read fresh data.
try {
- holder = uhandler.core.openNewSearcher(true, true);
-
+ RefCounted<SolrIndexSearcher> holder = uhandler.core.openNewSearcher(true, true);
+ holder.decref();
} catch (Exception e) {
- ParWork.propagateInterrupt(e, true);
SolrException.log(log, "Error opening realtime searcher", e);
- return null;
- } finally {
-
- if (map != null) map.clear();
- if (prevMap != null) prevMap.clear();
- if (prevMap2 != null) prevMap2.clear();
-
- if (!returnSearcher && holder != null) holder.decref();
+ return;
}
- if (returnSearcher) {
- return holder;
- }
- return null;
+ if (map != null) map.clear();
+ if (prevMap != null) prevMap.clear();
+ if (prevMap2 != null) prevMap2.clear();
} finally {
tlogLock.unlock();
}
@@ -863,25 +850,24 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
public void preCommit(CommitUpdateCommand cmd) {
+ tlogLock.lock();
+ try {
+ if (debug) {
+ log.debug("TLOG: preCommit");
+ }
- if (debug) {
- log.debug("TLOG: preCommit");
- }
-
- if (getState() != State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
- // if we aren't in the active state, and this isn't a replay
- // from the recovery process, then we shouldn't mess with
- // the current transaction log. This normally shouldn't happen
- // as DistributedUpdateProcessor will prevent this. Commits
- // that don't use the processor are possible though.
- return;
- }
+ if (getState() != State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
+ // if we aren't in the active state, and this isn't a replay
+ // from the recovery process, then we shouldn't mess with
+ // the current transaction log. This normally shouldn't happen
+ // as DistributedUpdateProcessor will prevent this. Commits
+ // that don't use the processor are possible though.
+ return;
+ }
- // since we're changing the log, we must change the map.
- newMap();
+ // since we're changing the log, we must change the map.
+ newMap();
- tlogLock.lock();
- try {
if (prevTlog != null) {
globalStrings = prevTlog.getGlobalStrings();
@@ -1140,30 +1126,32 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
// something found in prevMap2 will always be found in prevMapLog2 (which could be tlog or prevTlog)
// SolrCore.verbose("TLOG: lookup ver: for id ",indexedId.utf8ToString(),"in prevMap2",System.identityHashCode(map),"got",entry,"lookupLog=",lookupLog);
}
+ } finally {
+ tlogLock.unlock();
+ }
- if (entry != null) {
- return entry.version;
- }
-
- // Now check real index
- Long version = versionInfo.getVersionFromIndex(indexedId);
+ if (entry != null) {
+ return entry.version;
+ }
- if (version != null) {
- return version;
- }
+ // Now check real index
+ Long version = versionInfo.getVersionFromIndex(indexedId);
- // We can't get any version info for deletes from the index, so if the doc
- // wasn't found, check a cache of recent deletes.
+ if (version != null) {
+ return version;
+ }
+ // We can't get any version info for deletes from the index, so if the doc
+ // wasn't found, check a cache of recent deletes.
+ tlogLock.lock();
+ try {
entry = oldDeletes.get(indexedId);
-
- if (entry != null) {
- return entry.version;
- }
-
} finally {
tlogLock.unlock();
}
+ if (entry != null) {
+ return entry.version;
+ }
return null;
}
diff --git a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
index 6455045..0a0c240 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
@@ -470,22 +470,22 @@ public class TestRealTimeGet extends TestRTGBase {
// req().getCore().getUpdateHandler().getIndexWriterProvider().getIndexWriter(req().getCore()).setInfoStream(System.out);
- final int commitPercent = TEST_NIGHTLY ? 5 + random().nextInt(20) : 2;
- final int softCommitPercent = TEST_NIGHTLY ? 30+random().nextInt(75) : 10; // what percent of the commits are soft
+ final int commitPercent = 5 + random().nextInt(20);
+ final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft
final int deletePercent = 4+random().nextInt(25);
final int deleteByQueryPercent = 1+random().nextInt(5);
final int optimisticPercent = 1+random().nextInt(50); // percent change that an update uses optimistic locking
final int optimisticCorrectPercent = 25+random().nextInt(70); // percent change that a version specified will be correct
final int filteredGetPercent = random().nextInt( random().nextInt(20)+1 ); // percent of time that a get will be filtered... we normally don't want too high.
- final int ndocs = TEST_NIGHTLY ? 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)) : 7;
- int nWriteThreads = TEST_NIGHTLY ? 5 + random().nextInt(25) : 2;
+ final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
+ int nWriteThreads = 5 + random().nextInt(25);
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time...
// query variables
- final int percentRealtimeQuery = TEST_NIGHTLY ? 60 : 10;
- final AtomicLong operations = new AtomicLong(TEST_NIGHTLY ? 50000 : 5000); // number of query operations to perform in total
- int nReadThreads = TEST_NIGHTLY ? 5 + random().nextInt(25) : 3;
+ final int percentRealtimeQuery = 60;
+ final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total
+ int nReadThreads = 5 + random().nextInt(25);
verbose("commitPercent=", commitPercent);
diff --git a/solr/core/src/test/org/apache/solr/search/TestStressReorder.java b/solr/core/src/test/org/apache/solr/search/TestStressReorder.java
index 2640d82..cfc3dd4 100644
--- a/solr/core/src/test/org/apache/solr/search/TestStressReorder.java
+++ b/solr/core/src/test/org/apache/solr/search/TestStressReorder.java
@@ -30,14 +30,12 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.TestHarness;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-@Ignore // nocommit - parallel commit/update (need to track down and harden this) // if the version matches, the val must log.error("ERROR, id={} found={} model {}", id, response, info);
public class TestStressReorder extends TestRTGBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -166,24 +164,27 @@ public class TestStressReorder extends TestRTGBase {
if (before) {
lastId = id;
}
+ Object sync = syncArr[id];
+ // nocommit do we need this like in TestRealTimeGet to make sure our model matches?
+ // maybe try a test httpclient with strict ordering?
+ synchronized (sync) {
+ DocInfo info = model.get(id);
- DocInfo info = model.get(id);
+ long val = info.val;
+ long nextVal = Math.abs(val) + 1;
- long val = info.val;
- long nextVal = Math.abs(val)+1;
+ // the version we set on the update should determine who wins
+ // These versions are not derived from the actual leader update handler hand hence this
+ // test may need to change depending on how we handle version numbers.
+ long version = testVersion.incrementAndGet();
- // the version we set on the update should determine who wins
- // These versions are not derived from the actual leader update handler hand hence this
- // test may need to change depending on how we handle version numbers.
- long version = testVersion.incrementAndGet();
-
- // yield after getting the next version to increase the odds of updates happening out of order
- if (rand.nextBoolean()) Thread.yield();
+ // yield after getting the next version to increase the odds of updates happening out of order
+ if (rand.nextBoolean()) Thread.yield();
if (oper < commitPercent + deletePercent) {
- verbose("deleting id",id,"val=",nextVal,"version",version);
+ verbose("deleting id", id, "val=", nextVal, "version", version);
- Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_", Long.toString(-version), DISTRIB_UPDATE_PARAM, FROM_LEADER));
// TODO: returning versions for these types of updates is redundant
// but if we do return, they had better be equal
@@ -199,12 +200,12 @@ public class TestStressReorder extends TestRTGBase {
}
}
- verbose("deleting id", id, "val=",nextVal,"version",version,"DONE");
+ verbose("deleting id", id, "val=", nextVal, "version", version, "DONE");
} else if (oper < commitPercent + deletePercent + deleteByQueryPercent) {
- verbose("deleteByQuery id",id,"val=",nextVal,"version",version);
+ verbose("deleteByQuery id", id, "val=", nextVal, "version", version);
- Long returnedVersion = deleteByQueryAndGetVersion("id:"+Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ Long returnedVersion = deleteByQueryAndGetVersion("id:" + Integer.toString(id), params("_version_", Long.toString(-version), DISTRIB_UPDATE_PARAM, FROM_LEADER));
// TODO: returning versions for these types of updates is redundant
// but if we do return, they had better be equal
@@ -220,12 +221,13 @@ public class TestStressReorder extends TestRTGBase {
}
}
- verbose("deleteByQuery id", id, "val=",nextVal,"version",version,"DONE");
+ verbose("deleteByQuery id", id, "val=", nextVal, "version", version, "DONE");
} else {
- verbose("adding id", id, "val=", nextVal,"version",version);
+ verbose("adding id", id, "val=", nextVal, "version", version);
- Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), FIELD, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), FIELD, Long.toString(nextVal), "_version_", Long.toString(version)),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
if (returnedVersion != null) {
assertEquals(version, returnedVersion.longValue());
}
@@ -239,11 +241,11 @@ public class TestStressReorder extends TestRTGBase {
}
if (VERBOSE) {
- verbose("adding id", id, "val=", nextVal,"version",version,"DONE");
+ verbose("adding id", id, "val=", nextVal, "version", version, "DONE");
}
}
- // } // end sync
+ } // end sync
if (!before) {
lastId = id;