You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/10/26 00:54:46 UTC
[lucene-solr] branch reference_impl_dev updated (5517100 -> 7b1855a)
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a change to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.
from 5517100 @1045 Raise allowed connections per host from 1 to 4.
new 1b7dd66 @1046 More tweaks.
new 7b1855a @1047 Fix SchemaTest.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/java/org/apache/solr/cloud/Overseer.java | 4 +
.../handler/component/RealTimeGetComponent.java | 3 +-
.../org/apache/solr/update/IndexFingerprint.java | 5 +-
.../src/java/org/apache/solr/update/UpdateLog.java | 96 ++++++++++------------
.../org/apache/solr/search/TestRealTimeGet.java | 14 ++--
.../org/apache/solr/search/TestStressReorder.java | 46 ++++++-----
.../solr/client/solrj/impl/Http2SolrClient.java | 10 ++-
.../solr/client/solrj/request/SchemaTest.java | 8 +-
8 files changed, 92 insertions(+), 94 deletions(-)
[lucene-solr] 01/02: @1046 More tweaks.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 1b7dd66b4c7b2ed780f09222f03f1f1c902ab9d1
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Sun Oct 25 17:52:32 2020 -0500
@1046 More tweaks.
---
.../src/java/org/apache/solr/cloud/Overseer.java | 4 +
.../handler/component/RealTimeGetComponent.java | 3 +-
.../org/apache/solr/update/IndexFingerprint.java | 5 +-
.../src/java/org/apache/solr/update/UpdateLog.java | 96 ++++++++++------------
.../org/apache/solr/search/TestRealTimeGet.java | 14 ++--
.../org/apache/solr/search/TestStressReorder.java | 46 ++++++-----
6 files changed, 81 insertions(+), 87 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index d2836b2..767b2ce 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -934,6 +934,10 @@ public class Overseer implements SolrCloseable {
log.debug("doClose() - start");
}
+ if (closeAndDone) {
+ shardHandler.cancelAll();
+ }
+
if (ccThread != null) {
((OverseerCollectionConfigSetProcessor) ccThread.getThread()).closing();
ccThread.interrupt();
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index b1b3333..e5ecce5 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -452,7 +452,8 @@ public class RealTimeGetComponent extends SearchComponent
*/
private static SolrDocument reopenRealtimeSearcherAndGet(SolrCore core, Term idTerm, ReturnFields returnFields) throws IOException {
UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
- RefCounted<SolrIndexSearcher> searcherHolder = ulog.openRealtimeSearcher(true);
+ ulog.openRealtimeSearcher();
+ RefCounted<SolrIndexSearcher> searcherHolder = core.getRealtimeSearcher();
try {
SolrIndexSearcher searcher = searcherHolder.get();
diff --git a/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java b/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java
index 67503e4..e40f8a2 100644
--- a/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java
+++ b/solr/core/src/java/org/apache/solr/update/IndexFingerprint.java
@@ -91,8 +91,8 @@ public class IndexFingerprint implements MapSerializable {
/** Opens a new realtime searcher and returns it's (possibly cached) fingerprint */
public static IndexFingerprint getFingerprint(SolrCore core, long maxVersion) throws IOException {
RTimer timer = new RTimer();
-
- RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().openRealtimeSearcher(true);
+ core.getUpdateHandler().getUpdateLog().openRealtimeSearcher();
+ RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().uhandler.core.getRealtimeSearcher();
try {
IndexFingerprint f = newestSearcher.get().getIndexFingerprint(maxVersion);
final double duration = timer.stop();
@@ -107,7 +107,6 @@ public class IndexFingerprint implements MapSerializable {
public static IndexFingerprint getFingerprint(SolrIndexSearcher searcher, LeafReaderContext ctx, Long maxVersion)
throws IOException {
-
SchemaField versionField = VersionInfo.getAndCheckVersionField(searcher.getSchema());
ValueSource vs = versionField.getType().getValueSource(versionField, null);
@SuppressWarnings({"rawtypes"})
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 69bf310..50ff617 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -725,40 +725,27 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
- public RefCounted<SolrIndexSearcher> openRealtimeSearcher() {
- return openRealtimeSearcher(false);
- }
-
/** Opens a new realtime searcher and clears the id caches.
* This may also be called when we updates are being buffered (from PeerSync/IndexFingerprint)
- * @return opened searcher if requested
*/
- public RefCounted<SolrIndexSearcher> openRealtimeSearcher(boolean returnSearcher) {
+ public void openRealtimeSearcher() {
// We must cause a new IndexReader to be opened before anything looks at these caches again
// so that a cache miss will read fresh data.
- RefCounted<SolrIndexSearcher> holder = null;
tlogLock.lock();
try {
+ // We must cause a new IndexReader to be opened before anything looks at these caches again
+ // so that a cache miss will read fresh data.
try {
- holder = uhandler.core.openNewSearcher(true, true);
-
+ RefCounted<SolrIndexSearcher> holder = uhandler.core.openNewSearcher(true, true);
+ holder.decref();
} catch (Exception e) {
- ParWork.propagateInterrupt(e, true);
SolrException.log(log, "Error opening realtime searcher", e);
- return null;
- } finally {
-
- if (map != null) map.clear();
- if (prevMap != null) prevMap.clear();
- if (prevMap2 != null) prevMap2.clear();
-
- if (!returnSearcher && holder != null) holder.decref();
+ return;
}
- if (returnSearcher) {
- return holder;
- }
- return null;
+ if (map != null) map.clear();
+ if (prevMap != null) prevMap.clear();
+ if (prevMap2 != null) prevMap2.clear();
} finally {
tlogLock.unlock();
}
@@ -863,25 +850,24 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
public void preCommit(CommitUpdateCommand cmd) {
+ tlogLock.lock();
+ try {
+ if (debug) {
+ log.debug("TLOG: preCommit");
+ }
- if (debug) {
- log.debug("TLOG: preCommit");
- }
-
- if (getState() != State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
- // if we aren't in the active state, and this isn't a replay
- // from the recovery process, then we shouldn't mess with
- // the current transaction log. This normally shouldn't happen
- // as DistributedUpdateProcessor will prevent this. Commits
- // that don't use the processor are possible though.
- return;
- }
+ if (getState() != State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {
+ // if we aren't in the active state, and this isn't a replay
+ // from the recovery process, then we shouldn't mess with
+ // the current transaction log. This normally shouldn't happen
+ // as DistributedUpdateProcessor will prevent this. Commits
+ // that don't use the processor are possible though.
+ return;
+ }
- // since we're changing the log, we must change the map.
- newMap();
+ // since we're changing the log, we must change the map.
+ newMap();
- tlogLock.lock();
- try {
if (prevTlog != null) {
globalStrings = prevTlog.getGlobalStrings();
@@ -1140,30 +1126,32 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
// something found in prevMap2 will always be found in prevMapLog2 (which could be tlog or prevTlog)
// SolrCore.verbose("TLOG: lookup ver: for id ",indexedId.utf8ToString(),"in prevMap2",System.identityHashCode(map),"got",entry,"lookupLog=",lookupLog);
}
+ } finally {
+ tlogLock.unlock();
+ }
- if (entry != null) {
- return entry.version;
- }
-
- // Now check real index
- Long version = versionInfo.getVersionFromIndex(indexedId);
+ if (entry != null) {
+ return entry.version;
+ }
- if (version != null) {
- return version;
- }
+ // Now check real index
+ Long version = versionInfo.getVersionFromIndex(indexedId);
- // We can't get any version info for deletes from the index, so if the doc
- // wasn't found, check a cache of recent deletes.
+ if (version != null) {
+ return version;
+ }
+ // We can't get any version info for deletes from the index, so if the doc
+ // wasn't found, check a cache of recent deletes.
+ tlogLock.lock();
+ try {
entry = oldDeletes.get(indexedId);
-
- if (entry != null) {
- return entry.version;
- }
-
} finally {
tlogLock.unlock();
}
+ if (entry != null) {
+ return entry.version;
+ }
return null;
}
diff --git a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
index 6455045..0a0c240 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
@@ -470,22 +470,22 @@ public class TestRealTimeGet extends TestRTGBase {
// req().getCore().getUpdateHandler().getIndexWriterProvider().getIndexWriter(req().getCore()).setInfoStream(System.out);
- final int commitPercent = TEST_NIGHTLY ? 5 + random().nextInt(20) : 2;
- final int softCommitPercent = TEST_NIGHTLY ? 30+random().nextInt(75) : 10; // what percent of the commits are soft
+ final int commitPercent = 5 + random().nextInt(20);
+ final int softCommitPercent = 30+random().nextInt(75); // what percent of the commits are soft
final int deletePercent = 4+random().nextInt(25);
final int deleteByQueryPercent = 1+random().nextInt(5);
final int optimisticPercent = 1+random().nextInt(50); // percent change that an update uses optimistic locking
final int optimisticCorrectPercent = 25+random().nextInt(70); // percent change that a version specified will be correct
final int filteredGetPercent = random().nextInt( random().nextInt(20)+1 ); // percent of time that a get will be filtered... we normally don't want too high.
- final int ndocs = TEST_NIGHTLY ? 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200)) : 7;
- int nWriteThreads = TEST_NIGHTLY ? 5 + random().nextInt(25) : 2;
+ final int ndocs = 5 + (random().nextBoolean() ? random().nextInt(25) : random().nextInt(200));
+ int nWriteThreads = 5 + random().nextInt(25);
final int maxConcurrentCommits = nWriteThreads; // number of committers at a time...
// query variables
- final int percentRealtimeQuery = TEST_NIGHTLY ? 60 : 10;
- final AtomicLong operations = new AtomicLong(TEST_NIGHTLY ? 50000 : 5000); // number of query operations to perform in total
- int nReadThreads = TEST_NIGHTLY ? 5 + random().nextInt(25) : 3;
+ final int percentRealtimeQuery = 60;
+ final AtomicLong operations = new AtomicLong(50000); // number of query operations to perform in total
+ int nReadThreads = 5 + random().nextInt(25);
verbose("commitPercent=", commitPercent);
diff --git a/solr/core/src/test/org/apache/solr/search/TestStressReorder.java b/solr/core/src/test/org/apache/solr/search/TestStressReorder.java
index 2640d82..cfc3dd4 100644
--- a/solr/core/src/test/org/apache/solr/search/TestStressReorder.java
+++ b/solr/core/src/test/org/apache/solr/search/TestStressReorder.java
@@ -30,14 +30,12 @@ import org.apache.solr.common.util.Utils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.TestHarness;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
-@Ignore // nocommit - parallel commit/update (need to track down and harden this) // if the version matches, the val must log.error("ERROR, id={} found={} model {}", id, response, info);
public class TestStressReorder extends TestRTGBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -166,24 +164,27 @@ public class TestStressReorder extends TestRTGBase {
if (before) {
lastId = id;
}
+ Object sync = syncArr[id];
+ // nocommit do we need this like in TestRealTimeGet to make sure our model matches?
+ // maybe try a test httpclient with strict ordering?
+ synchronized (sync) {
+ DocInfo info = model.get(id);
- DocInfo info = model.get(id);
+ long val = info.val;
+ long nextVal = Math.abs(val) + 1;
- long val = info.val;
- long nextVal = Math.abs(val)+1;
+ // the version we set on the update should determine who wins
+ // These versions are not derived from the actual leader update handler hand hence this
+ // test may need to change depending on how we handle version numbers.
+ long version = testVersion.incrementAndGet();
- // the version we set on the update should determine who wins
- // These versions are not derived from the actual leader update handler hand hence this
- // test may need to change depending on how we handle version numbers.
- long version = testVersion.incrementAndGet();
-
- // yield after getting the next version to increase the odds of updates happening out of order
- if (rand.nextBoolean()) Thread.yield();
+ // yield after getting the next version to increase the odds of updates happening out of order
+ if (rand.nextBoolean()) Thread.yield();
if (oper < commitPercent + deletePercent) {
- verbose("deleting id",id,"val=",nextVal,"version",version);
+ verbose("deleting id", id, "val=", nextVal, "version", version);
- Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ Long returnedVersion = deleteAndGetVersion(Integer.toString(id), params("_version_", Long.toString(-version), DISTRIB_UPDATE_PARAM, FROM_LEADER));
// TODO: returning versions for these types of updates is redundant
// but if we do return, they had better be equal
@@ -199,12 +200,12 @@ public class TestStressReorder extends TestRTGBase {
}
}
- verbose("deleting id", id, "val=",nextVal,"version",version,"DONE");
+ verbose("deleting id", id, "val=", nextVal, "version", version, "DONE");
} else if (oper < commitPercent + deletePercent + deleteByQueryPercent) {
- verbose("deleteByQuery id",id,"val=",nextVal,"version",version);
+ verbose("deleteByQuery id", id, "val=", nextVal, "version", version);
- Long returnedVersion = deleteByQueryAndGetVersion("id:"+Integer.toString(id), params("_version_",Long.toString(-version), DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ Long returnedVersion = deleteByQueryAndGetVersion("id:" + Integer.toString(id), params("_version_", Long.toString(-version), DISTRIB_UPDATE_PARAM, FROM_LEADER));
// TODO: returning versions for these types of updates is redundant
// but if we do return, they had better be equal
@@ -220,12 +221,13 @@ public class TestStressReorder extends TestRTGBase {
}
}
- verbose("deleteByQuery id", id, "val=",nextVal,"version",version,"DONE");
+ verbose("deleteByQuery id", id, "val=", nextVal, "version", version, "DONE");
} else {
- verbose("adding id", id, "val=", nextVal,"version",version);
+ verbose("adding id", id, "val=", nextVal, "version", version);
- Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), FIELD, Long.toString(nextVal), "_version_",Long.toString(version)), params(DISTRIB_UPDATE_PARAM,FROM_LEADER));
+ Long returnedVersion = addAndGetVersion(sdoc("id", Integer.toString(id), FIELD, Long.toString(nextVal), "_version_", Long.toString(version)),
+ params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
if (returnedVersion != null) {
assertEquals(version, returnedVersion.longValue());
}
@@ -239,11 +241,11 @@ public class TestStressReorder extends TestRTGBase {
}
if (VERBOSE) {
- verbose("adding id", id, "val=", nextVal,"version",version,"DONE");
+ verbose("adding id", id, "val=", nextVal, "version", version, "DONE");
}
}
- // } // end sync
+ } // end sync
if (!before) {
lastId = id;
[lucene-solr] 02/02: @1047 Fix SchemaTest.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 7b1855a26a4be410eb3f893b3a1649bba3c625ae
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Sun Oct 25 19:54:17 2020 -0500
@1047 Fix SchemaTest.
---
.../org/apache/solr/client/solrj/impl/Http2SolrClient.java | 10 +++++++++-
.../test/org/apache/solr/client/solrj/request/SchemaTest.java | 8 ++------
2 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 42c35ff..f32f267 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@ -90,6 +90,7 @@ import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -849,8 +850,15 @@ public class Http2SolrClient extends SolrClient {
} else if (metadataObj instanceof Map) {
metadata = new NamedList((Map) metadataObj);
}
+ List details = (ArrayList) Utils.getObjectByPath(error, false, Collections.singletonList("details"));
+ if (details != null) {
+ reason = reason + " " + details;
+ }
+
}
- } catch (Exception ex) {}
+ } catch (Exception ex) {
+ log.warn("Exception parsing error response", ex);
+ }
if (reason == null) {
StringBuilder msg = new StringBuilder();
msg.append(response.getReason())
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
index 947894c..004f830 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
@@ -64,12 +64,8 @@ public class SchemaTest extends RestTestBase {
}
private static void assertFailedSchemaResponse(ThrowingRunnable runnable, String expectedErrorMessage) {
- BaseHttpSolrClient.RemoteExecutionException e = expectThrows(BaseHttpSolrClient.RemoteExecutionException.class, runnable);
- SimpleOrderedMap errorMap = (SimpleOrderedMap)e.getMetaData().get("error");
- assertEquals("org.apache.solr.api.ApiBag$ExceptionWithErrObject",
- ((NamedList)errorMap.get("metadata")).get("error-class"));
- List details = (List)errorMap.get("details");
- assertTrue(((List)((Map)details.get(0)).get("errorMessages")).get(0).toString().contains(expectedErrorMessage));
+ BaseHttpSolrClient.RemoteSolrException e = expectThrows(BaseHttpSolrClient.RemoteSolrException.class, runnable);
+ assertTrue(e.getMessage(), e.getMessage().contains(expectedErrorMessage));
}
private static void createStoredStringField(String fieldName, SolrClient solrClient) throws Exception {