You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/02/27 01:55:51 UTC
[lucene-solr] 01/02: @1408 Zhu Li, the nightly thing please.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 9ca26221bf04dbe51db9256f7ea76ed0cffb57a0
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Fri Feb 26 19:47:28 2021 -0600
@1408 Zhu Li, the nightly thing please.
Took 9 hours 22 minutes
Took 4 minutes
---
.../src/java/org/apache/solr/cloud/Overseer.java | 28 +-
.../src/java/org/apache/solr/core/SolrCore.java | 11 +-
.../apache/solr/filestore/NRTCachingDirectory.java | 10 +-
.../handler/component/RealTimeGetComponent.java | 3 +-
.../org/apache/solr/handler/loader/CSVLoader.java | 3 +-
.../org/apache/solr/update/AddUpdateCommand.java | 11 +-
.../org/apache/solr/update/SolrCmdDistributor.java | 24 +-
.../org/apache/solr/update/UpdateShardHandler.java | 15 -
.../processor/DistributedUpdateProcessor.java | 49 ++-
.../processor/DistributedZkUpdateProcessor.java | 329 +++++++++++----------
.../processor/NestedUpdateProcessorFactory.java | 4 +
.../org/apache/solr/cloud/CleanupOldIndexTest.java | 6 +-
.../solr/cloud/DistributedVersionInfoTest.java | 59 ++--
.../org/apache/solr/cloud/TestCloudRecovery.java | 2 +
.../search/TestRandomCollapseQParserPlugin.java | 6 +-
.../org/apache/solr/search/join/BJQParserTest.java | 2 +-
.../solrj/request/JavaBinUpdateRequestCodec.java | 3 +-
.../src/java/org/apache/solr/common/ParWork.java | 40 ++-
.../java/org/apache/solr/common/SkyHookDoc.java | 165 +++++++++++
.../java/org/apache/solr/common/SolrDocument.java | 16 +
.../org/apache/solr/common/SolrInputDocument.java | 13 +
.../org/apache/solr/common/cloud/SolrZkClient.java | 4 +
.../apache/solr/common/cloud/ZkStateReader.java | 93 +++---
.../solr/common/util/ObjectReleaseTracker.java | 17 +-
.../org/apache/solr/logging/MDCLoggingContext.java | 4 +
.../src/java/org/apache/solr/SolrTestCase.java | 9 +-
.../solr/cloud/AbstractFullDistribZkTestBase.java | 9 +-
.../src/resources/logconf/log4j2-std-debug.xml | 6 +-
28 files changed, 608 insertions(+), 333 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 0cb1288..88db849 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -513,9 +513,13 @@ public class Overseer implements SolrCloseable {
synchronized (this) {
retry = !zkController.getCoreContainer().isShutDown() && !zkController.isShutdownCalled() && !zkController.isClosed() && !closeAndDone;
}
- if (retry) {
+ if (retry && zkController.getZkClient().isAlive()) {
log.info("rejoining the overseer election after closing");
- zkController.rejoinOverseerElection(false);
+ try {
+ zkController.rejoinOverseerElection(false);
+ } catch (AlreadyClosedException e) {
+ return;
+ }
}
}
@@ -880,10 +884,22 @@ public class Overseer implements SolrCloseable {
overseer.writePendingUpdates();
if (overseer.zkStateWriter != null) {
- try {
- zkController.getZkClient().delete(fullPaths, true);
- } catch (Exception e) {
- log.warn("Failed deleting processed items", e);
+ if (zkController.getZkClient().isAlive()) {
+ try {
+ zkController.getZkClient().delete(fullPaths, true);
+ } catch (KeeperException.SessionExpiredException | KeeperException.ConnectionLossException e) {
+ if (zkController.getZkClient().isAlive()) {
+ try {
+ zkController.getZkClient().delete(fullPaths, true);
+ } catch (KeeperException keeperException) {
+ log.warn("Failed deleting processed items", e);
+ }
+ } else {
+ log.warn("Failed deleting processed items", e);
+ }
+ } catch (Exception e) {
+ log.warn("Failed deleting processed items", e);
+ }
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 0bb41b1..ac1bc5f 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -232,7 +232,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
- private final ReentrantLock ruleExpiryLock;
private final ReentrantLock snapshotDelLock; // A lock instance to guard against concurrent deletions.
private volatile Timer newSearcherTimer;
@@ -734,7 +733,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
log.info("Wait for reload lock");
while (!(lock.tryLock() || lock.tryLock(250, TimeUnit.MILLISECONDS))) {
- if (closing) {
+ if (closing || coreContainer.isShutDown()) {
log.warn("Skipping reload because we are closed");
reloadyWaiting.decrementAndGet();
throw new AlreadyClosedException();
@@ -1148,7 +1147,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
bufferUpdatesIfConstructing(coreDescriptor);
timeBufferUpdatesIfConstructing.done();
- this.ruleExpiryLock = new ReentrantLock();
this.snapshotDelLock = new ReentrantLock();
} catch (Throwable e) {
@@ -1750,7 +1748,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {
if (count < -1) {
refCount.set(-1);
- throw new IllegalStateException("Already closed " + count);
+ log.warn("Already closed " + count);
+ return;
}
// if (log.isDebugEnabled()) {
@@ -3410,10 +3409,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return snapshotMgr;
}
- public ReentrantLock getRuleExpiryLock() {
- return ruleExpiryLock;
- }
-
/////////////////////////////////////////////////////////////////////
// SolrInfoBean stuff: Statistics and Module Info
/////////////////////////////////////////////////////////////////////
diff --git a/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java b/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java
index 78a2d96..f58d0e2 100644
--- a/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java
+++ b/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java
@@ -181,9 +181,15 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
for(String fileName : fileNames) {
unCache(fileName);
}
- //if (Boolean.getBoolean("solr.nrtDirSync")) { // MRM TODO:
+
+ // MRM TODO: we should prob link this setting to tlog sync
+ // with replicas, doesn't make a lot of sense to use, but without replicas
+ // or in certain cases, you would want to enable both - otherwise the cost
+ // is very high and a replica that goes out, by the time it comes back is generally
+ // going to just be recovering from the new leader
+ if (Boolean.getBoolean("solr.nrtDirSync")) {
in.sync(fileNames);
- //}
+ }
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 5f108ee..fe30f00 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -641,7 +641,8 @@ public class RealTimeGetComponent extends SearchComponent
SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
if (onlyTheseNonStoredDVs != null) {
- sid = new SolrInputDocument();
+ sid = SolrInputDocument.THREAD_LOCAL_SolrInputDocument.get();
+ sid.clear();
} else {
Document luceneDocument = docFetcher.doc(docid);
sid = toSolrInputDocument(luceneDocument, schema);
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
index 3af21e1..d04ca82 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
@@ -50,7 +50,8 @@ class SingleThreadedCSVLoader extends CSVLoaderBase {
templateAdd.setReq(req);
templateAdd.overwrite = overwrite;
templateAdd.commitWithin = commitWithin;
- SolrInputDocument doc = new SolrInputDocument();
+ SolrInputDocument doc = SolrInputDocument.THREAD_LOCAL_SolrInputDocument.get();
+ doc.clear();
doAdd(line, vals, doc, templateAdd);
}
}
diff --git a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
index 32e5d62..3bcc3cb 100644
--- a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
+++ b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.solr.common.SkyHookDoc;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
@@ -139,14 +140,18 @@ public class AddUpdateCommand extends UpdateCommand {
int count = field==null ? 0 : field.getValueCount();
if (count == 0) {
if (overwrite) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Document is missing mandatory uniqueKey field: " + sf.getName());
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Document is missing mandatory uniqueKey field: " + sf.getName() + solrDoc);
}
} else if (count > 1) {
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Document contains multiple values for uniqueKey field: " + field);
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Document contains multiple values for uniqueKey field: " + field);
} else {
BytesRefBuilder b = new BytesRefBuilder();
sf.getType().readableToIndexed(field.getFirstValue().toString(), b);
indexedId = b.get();
+
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(field.getFirstValue().toString());
+ }
}
}
}
@@ -195,7 +200,7 @@ public class AddUpdateCommand extends UpdateCommand {
if (overwrite) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Document is missing mandatory uniqueKey field: "
- + sf.getName());
+ + sf.getName() + " solrDoc=" + solrDoc);
}
} else if (count > 1) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 023231c..f9f74a5 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -237,20 +237,22 @@ public class SolrCmdDistributor implements Closeable {
private void submit(final Req req, String tag) {
- if (cancelExeption != null) {
- Throwable exp = cancelExeption;
- cancelExeption = null;
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exp);
- }
+// if (cancelExeption != null) {
+// Throwable exp = cancelExeption;
+// cancelExeption = null;
+// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exp);
+// }
- if (log.isDebugEnabled()) {
- log.debug("sending update to " + req.node.getUrl() + " retry:" + req.retries + " " + req.cmd + " params:" + req.uReq.getParams());
+// if (log.isDebugEnabled()) {
+// log.debug("sending update to " + req.node.getUrl() + " retry:" + req.retries + " " + req.cmd + " params:" + req.uReq.getParams());
+// }
+ if (req.cmd instanceof AddUpdateCommand) {
+ log.info("sending update to " + req.node.getUrl() + " retry:" + req.retries + " docid=" + ((AddUpdateCommand) req.cmd).getPrintableId() + " " + req.cmd + " params:" + req.uReq.getParams());
+ } else {
+ log.info("sending update to " + req.node.getUrl() + " retry:" + req.retries + " docid=" + req.cmd + " params:" + req.uReq.getParams());
}
- if (isClosed != null && isClosed.isClosed()) {
- log.warn("Already closed, skippping update");
- throw new AlreadyClosedException();
- }
+
req.uReq.setBasePath(req.node.getUrl());
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
index f270e0e..6e8ead8 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
@@ -24,11 +24,9 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.solr.client.solrj.impl.BaseCloudSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.HttpListenerFactory;
import org.apache.solr.common.ParWork;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -42,7 +40,6 @@ import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.stats.HttpClientMetricNameStrategy;
import org.apache.solr.util.stats.InstrumentedHttpRequestExecutor;
-import org.apache.solr.util.stats.InstrumentedPoolingHttpClientConnectionManager;
import org.apache.solr.util.stats.MetricUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,8 +60,6 @@ public class UpdateShardHandler implements SolrInfoBean {
private ExecutorService recoveryExecutor;
- private final InstrumentedPoolingHttpClientConnectionManager defaultConnectionManager;
-
private final InstrumentedHttpRequestExecutor httpRequestExecutor;
//private final InstrumentedHttpListenerFactory updateHttpListenerFactory;
@@ -79,11 +74,8 @@ public class UpdateShardHandler implements SolrInfoBean {
public UpdateShardHandler(UpdateShardHandlerConfig cfg) {
assert ObjectReleaseTracker.track(this);
assert (closeTracker = new CloseTracker()) != null;
- defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
ModifiableSolrParams clientParams = new ModifiableSolrParams();
if (cfg != null ) {
- defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
- defaultConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout());
clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout());
// following is done only for logging complete configuration.
@@ -178,7 +170,6 @@ public class UpdateShardHandler implements SolrInfoBean {
solrMetricsContext = parentContext.getChildContext(this);
String expandedScope = SolrMetricManager.mkName(scope, getCategory().name());
//.initializeMetrics(solrMetricsContext, expandedScope);
- defaultConnectionManager.initializeMetrics(solrMetricsContext, expandedScope);
recoveryExecutor = MetricUtils.instrumentedExecutorService(recoveryExecutor, this, solrMetricsContext.getMetricRegistry(),
SolrMetricManager.mkName("recoveryExecutor", expandedScope, "threadPool"));
}
@@ -211,11 +202,6 @@ public class UpdateShardHandler implements SolrInfoBean {
return searchOnlyClient;
}
-
- public PoolingHttpClientConnectionManager getDefaultConnectionManager() {
- return defaultConnectionManager;
- }
-
/**
*
* @return executor for recovery operations
@@ -235,7 +221,6 @@ public class UpdateShardHandler implements SolrInfoBean {
closer.collect(recoveryOnlyClient);
closer.collect(searchOnlyClient);
closer.collect(updateOnlyClient);
- closer.collect(defaultConnectionManager);
closer.collect("SolrInfoBean", () -> {
SolrInfoBean.super.close();
return this;
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index b30a584..69fb77f 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -39,6 +39,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
import org.apache.solr.common.ParWork;
+import org.apache.solr.common.SkyHookDoc;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
@@ -229,7 +230,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// the request right here but for now I think it is better to just return the status
// to the client that the minRf wasn't reached and let them handle it
- log.info("docid={} isLeader={} forwardToLeader={} nodes={}, params={}", cmd.getPrintableId(), isLeader, forwardToLeader, getNodes(), req.getParams());
+ if (log.isDebugEnabled()) log.debug("docid={} isLeader={} forwardToLeader={} nodes={}, params={}", cmd.getPrintableId(), isLeader, forwardToLeader, getNodes(), req.getParams());
if (!forwardToLeader) {
Future distFuture = versionAdd(cmd);
@@ -250,9 +251,15 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
} else {
// drop it
log.info("drop docid={}", cmd.getPrintableId());
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "final notice of dropping update, returning from processAdd cmd=" + cmd);
+ }
return;
}
} else {
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "distribute update to replicas if necessary cmd=" + cmd);
+ }
doDistribAdd(cmd);
}
@@ -303,6 +310,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
BytesRef idBytes = cmd.getIndexedId();
if (idBytes == null) {
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "local add due to null idBytes cmd=" + cmd);
+ }
super.processAdd(cmd);
return CompletableFuture.completedFuture(null);
}
@@ -312,6 +322,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Atomic document updates are not supported unless <updateLog/> is configured");
} else {
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "local add due to null vinfo cmd=" + cmd);
+ }
super.processAdd(cmd);
return CompletableFuture.completedFuture(null);
}
@@ -357,6 +370,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
if (dependentVersionFound == -1) {
// it means the document has been deleted by now at the leader. drop this update
log.info("docid={} it means the document has been deleted by now at the leader. drop this update", cmd.getPrintableId());
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, dependent update has been deleted by the leader cmd=" + cmd);
+ }
return null;
}
}
@@ -412,6 +428,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
ulog.add(cmd);
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, leader logic applied, but we are buffering - added to ulog only");
+ }
return null;
}
@@ -423,6 +442,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// specified it must exist (versionOnUpdate==1) and it does.
} else {
if(cmd.getReq().getParams().getBool(CommonParams.FAIL_ON_VERSION_CONFLICTS, true) == false) {
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, failed on version conflict but set not to fail");
+ }
return null;
}
@@ -445,6 +467,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
ulog.add(cmd);
log.info("docid={} dropped because not active and buffering and not a replay update", cmd.getPrintableId());
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, non logic applied, but we are buffering - added to ulog only");
+ }
return null;
}
@@ -465,6 +490,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
}
versionDelete((DeleteUpdateCommand) fetchedFromLeader);
log.info("docid={} dropped due to missing dependent update", cmd.getPrintableId());
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, inplace update with too low a version");
+ }
return null;
} else {
assert fetchedFromLeader instanceof AddUpdateCommand;
@@ -485,6 +513,12 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// this means we got a newer full doc update and in that case it makes no sense to apply the older
// inplace update. Drop this update
log.info("Update was applied on version: {}, but last version I have is: {}. Dropping current update, docid={}", prev, lastVersion, cmd.getPrintableId());
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, leader logic applied, but we are buffering - added to ulog only");
+ }
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, we have a newer version for inplace update");
+ }
return null;
} else {
// We're good, we should apply this update. First, update the bucket's highest.
@@ -507,7 +541,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// This update is a repeat, or was reordered. We need to drop this update.
log.info("Dropping add update due to version docid={} lastVersion={} versionOnUpdate={}", idBytes.utf8ToString(), lastVersion, versionOnUpdate);
-
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, we have a newer version");
+ }
return null;
}
}
@@ -547,7 +583,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// This refresh makes RTG handler aware of this update.q
try {
- log.info("do local add for docid={}", cmd. getPrintableId());
+ if (log.isDebugEnabled()) log.debug("do local add for docid={}", cmd. getPrintableId());
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "do local add");
+ }
doLocalAdd(cmd);
} catch (Exception e) {
Throwable t;
@@ -1517,6 +1556,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
log.trace("Run distrib add collection");
}
try {
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(finalCloneCmd.getPrintableId(), "do distrib update after versionAdd");
+ }
+
doDistribAdd(finalCloneCmd);
if (log.isTraceEnabled()) {
log.trace("after distrib add collection");
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index afb7e05..4f59f8e 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -28,9 +28,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.concurrent.locks.ReentrantLock;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.CloudDescriptor;
@@ -39,6 +37,7 @@ import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkShardTerms;
import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.common.ParWork;
+import org.apache.solr.common.SkyHookDoc;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
@@ -72,7 +71,6 @@ import org.apache.solr.update.SolrCmdDistributor;
import org.apache.solr.update.SolrIndexSplitter;
import org.apache.solr.update.UpdateCommand;
import org.apache.solr.util.TestInjection;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -187,9 +185,6 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
return;
}
- nodes = getCollectionUrls(collection,
- EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT), true);
-
try {
leaderReplica = clusterState.getCollection(collection).getSlice(cloudDesc.getShardId()).getLeader();
} catch (Exception e) {
@@ -220,45 +215,37 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
} else if (req.getParams().get(COMMIT_END_POINT, "").equals("leaders")) {
sendCommitToReplicasAndLocalCommit(cmd, worker, leaderReplica, params);
- } else {
+ } else if (req.getParams().get(COMMIT_END_POINT) == null) {
// zk
+ List<SolrCmdDistributor.Node> useNodes = getCollectionUrls(collection, EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT), true);
if (isLeader) {
SolrCmdDistributor.Node removeNode = null;
- for (SolrCmdDistributor.Node node : nodes) {
+ for (SolrCmdDistributor.Node node : useNodes) {
if (node.getCoreName().equals(this.desc.getName())) {
removeNode = node;
}
}
if (removeNode != null) {
- nodes.remove(removeNode);
+ useNodes.remove(removeNode);
sendCommitToReplicasAndLocalCommit(cmd, worker, leaderReplica, params);
}
}
- List<SolrCmdDistributor.Node> useNodes = null;
- if (req.getParams().get(COMMIT_END_POINT) == null) {
- useNodes = nodes;
+ params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
+ params.set(COMMIT_END_POINT, "leaders");
- params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
- params.set(COMMIT_END_POINT, "leaders");
+ if (useNodes != null && useNodes.size() > 0) {
+ if (log.isDebugEnabled()) log.debug("processCommit - send commit to leaders nodes={}", useNodes);
+ params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
- if (useNodes != null && useNodes.size() > 0) {
- if (log.isDebugEnabled()) log.debug("processCommit - send commit to leaders nodes={}",
- useNodes);
- params.set(DISTRIB_FROM, Replica
- .getCoreUrl(zkController.getBaseUrl(),
- req.getCore().getName()));
-
- List<SolrCmdDistributor.Node> finalUseNodes1 = useNodes;
- worker.collect("distCommit", () -> {
- cmdDistrib.distribCommit(cmd, finalUseNodes1, params);
- });
- }
- return;
+ List<SolrCmdDistributor.Node> finalUseNodes1 = useNodes;
+ worker.collect("distCommit", () -> {
+ cmdDistrib.distribCommit(cmd, finalUseNodes1, params);
+ });
}
-
+ return;
}
}
if (log.isDebugEnabled()) log.debug("processCommit(CommitUpdateCommand) - end");
@@ -326,54 +313,76 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
@Override
protected void doDistribAdd(AddUpdateCommand cmd) throws IOException {
// if (log.isDebugEnabled()) log.debug("Distribute add cmd {} to {} {}", cmd, nodes, isLeader);
- log.info("Distribute add docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
- if (isLeader && !isSubShardLeader) {
- DocCollection coll = clusterState.getCollection(collection);
- List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), cmd.getRootIdUsingRouteParam(), cmd.getSolrInputDocument());
- // the list<node> will actually have only one element for an add request
- if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
- ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
- params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
- params.set(DISTRIB_FROM_PARENT, cloudDesc.getShardId());
- cmdDistrib.distribAdd(cmd, subShardLeaders, params, true);
+ log.info("Distribute add docid={} cmd={} to {} leader={} isSubShardLeader={}", cmd.getPrintableId(), cmd, nodes, isLeader, isSubShardLeader);
+
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "do distrib add isLeader=" + isLeader + " isSubShardLeader=" + isSubShardLeader);
+ }
+ if (isLeader && !isSubShardLeader && !forwardToLeader) {
+
+ DocCollection coll;
+ String routeId;
+ try {
+ coll = clusterState.getCollection(collection);
+ routeId = cmd.getRootIdUsingRouteParam();
+ } catch (Exception e) {
+ log.error("Error getting routeId docId={}", cmd.getPrintableId(), e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Error getting routeId docId=" + cmd.getPrintableId(), e);
}
- final List<SolrCmdDistributor.Node> nodesByRoutingRules = getNodesByRoutingRules(clusterState, coll, cmd.getRootIdUsingRouteParam(), cmd.getSolrInputDocument());
- if (nodesByRoutingRules != null && !nodesByRoutingRules.isEmpty()) {
- ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
- params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
- params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
- params.set(DISTRIB_FROM_COLLECTION, collection);
- params.set(DISTRIB_FROM_SHARD, cloudDesc.getShardId());
+ if (log.isDebugEnabled()) log.debug("going to maybe get sub shard leaders docid={} cmd={} to {} leader={} routeId={}", cmd.getPrintableId(), cmd, nodes, isLeader, routeId);
+ if (routeId != null) {
- try {
- cmdDistrib.distribAdd(cmd, nodesByRoutingRules, params, true);
- } catch (IOException e) {
- log.error("", e);
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), routeId, cmd.getSolrInputDocument(), cmd);
+
+ // the list<node> will actually have only one element for an add request
+ if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
+ ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
+ params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
+ params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
+ params.set(DISTRIB_FROM_PARENT, cloudDesc.getShardId());
+ cmdDistrib.distribAdd(cmd, subShardLeaders, params, true);
}
- return;
- }
+ if (log.isDebugEnabled()) log.debug("Distribute add getNodesByRoutingRules docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
+ final List<SolrCmdDistributor.Node> nodesByRoutingRules = getNodesByRoutingRules(clusterState, coll, cmd.getRootIdUsingRouteParam(), cmd.getSolrInputDocument());
+
+ if (log.isDebugEnabled()) log.debug("Distribute add got NodesByRoutingRules docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodesByRoutingRules, isLeader);
+ if (nodesByRoutingRules != null && !nodesByRoutingRules.isEmpty()) {
+ try {
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "do distrib to replicas with nodesByRoutingRules");
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
+ params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
+ params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
+ params.set(DISTRIB_FROM_COLLECTION, collection);
+ params.set(DISTRIB_FROM_SHARD, cloudDesc.getShardId());
+
+ cmdDistrib.distribAdd(cmd, nodesByRoutingRules, params, true);
+ } catch (IOException e) {
+ log.error("", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ return;
+ }
+ }
} else {
if (log.isDebugEnabled()) {
- log.debug("Not a shard or sub shard leader");
+ log.debug("Not a shard or sub shard leader docId={}", cmd.getPrintableId());
+ }
+ if (!forwardToLeader) {
+ return;
}
}
if (log.isDebugEnabled()) {
log.debug("Using nodes {}", nodes);
}
+
+ if (log.isDebugEnabled()) log.debug("Distribute add using nodes if not null and larger than size 0 docid={} cmd={} to {} isLeader={}", cmd.getPrintableId(), cmd, nodes, isLeader);
if (nodes != null && nodes.size() > 0) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
- if (forwardToLeader) {
- params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
- params.set("h", "forwardToLeaderForShard" + nodes);
- } else {
- params.set(DISTRIB_UPDATE_PARAM, (isLeader || isSubShardLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
- params.set("h", "toReplicasForShard" + nodes);
- }
-
+ params.set(DISTRIB_UPDATE_PARAM, (isLeader || isSubShardLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
if (req.getParams().get(UpdateRequest.MIN_REPFACT) != null) {
@@ -383,7 +392,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
for (SolrCmdDistributor.Node node : nodes) {
if (node.getCoreName().equals(desc.getName())) {
- log.error("IllegalState, trying to send an update to ourself");
+ log.error("docId={} IllegalState, trying to send an update to ourself", cmd.getPrintableId());
throw new IllegalStateException("IllegalState, trying to send an update to ourself");
}
}
@@ -397,7 +406,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// and the current in-place update (that depends on the previous update), if reordered
// in the stream, can result in the current update being bottled up behind the previous
// update in the stream and can lead to degraded performance.
-
+ if (log.isDebugEnabled()) log.debug("Distribute add inplaceupdate docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
try {
cmdDistrib.distribAdd(cmd, nodes, params, true, rollupReplicationTracker, leaderReplicationTracker);
} catch (IOException e) {
@@ -408,6 +417,10 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// if (!isLeader && params.get(DISTRIB_UPDATE_PARAM).equals(DistribPhase.FROMLEADER.toString())) {
// throw new IllegalStateException();
// }
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "send update to cmdDistrib nodes=" + nodes + " cmd=" + cmd);
+ }
+ if (log.isDebugEnabled()) log.debug("Distribute add, std old nodes docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
try {
cmdDistrib.distribAdd(cmd, nodes, params, false, rollupReplicationTracker, leaderReplicationTracker);
} catch (IOException e) {
@@ -441,7 +454,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
protected void doDistribDeleteById(DeleteUpdateCommand cmd) throws IOException {
if (isLeader && !isSubShardLeader) {
DocCollection coll = clusterState.getCollection(collection);
- List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), cmd.getId(), null);
+ List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), cmd.getId(), null, cmd);
// the list<node> will actually have only one element for an add request
if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
@@ -597,7 +610,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted", e);
}
if (leaderLogic) {
- List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), null, null);
+ List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), null, null, cmd);
if (subShardLeaders != null) {
cmdDistrib.distribDelete(cmd, subShardLeaders, params, true, rollupReplicationTracker, leaderReplicationTracker);
}
@@ -760,11 +773,11 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
// that means I want to forward onto my replicas...
// so get the replicas...
forwardToLeader = false;
-
+ String leaderCoreName = leaderReplica.getName();
List<Replica> replicas = clusterState.getCollection(collection)
.getSlice(shardId)
.getReplicas(EnumSet.of(Replica.Type.NRT, Replica.Type.TLOG));
- replicas.removeIf((replica) -> replica.getName().equals(desc.getName()));
+ replicas.removeIf((replica) -> replica.getName().equals(leaderCoreName));
if (replicas.isEmpty()) {
return null;
}
@@ -792,13 +805,13 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
if (log.isInfoEnabled()) {
log.info("check url:{} against:{} result:true", replica.getCoreUrl(), skipListSet);
}
- } else if(zkShardTerms.registered(coreNodeName) && zkShardTerms.skipSendingUpdatesTo(coreNodeName) && replica.getState().equals(Replica.State.ACTIVE)) {
+ } else if(zkShardTerms.registered(coreNodeName) && zkShardTerms.skipSendingUpdatesTo(coreNodeName)) {
log.info("skip url:{} cause its term is less than leader", replica.getCoreUrl());
skippedCoreNodeNames.add(replica.getName());
- } else if (!zkController.getZkStateReader().getLiveNodes().contains(replica.getNodeName())) {
- log.info("skip url:{} cause its not on live node={}", replica.getCoreUrl(), replica.getNodeName());
+ } else if (!zkController.getZkStateReader().getLiveNodes().contains(replica.getNodeName()) || (replica.getState() != Replica.State.ACTIVE &&
+ replica.getState() != Replica.State.BUFFERING)) {
skippedCoreNodeNames.add(replica.getName());
} else {
nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), replica, collection, shardId, maxRetriesToFollowers));
@@ -817,7 +830,10 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
if (log.isDebugEnabled()) log.debug("Forward update to leader {}", nodes);
if (desc.getName().equals(leaderReplica.getName())) {
- throw new IllegalStateException("We were asked to forward an update to ourself, which should not happen name=" + desc.getName() + " isLeader=" + isLeader);
+ IllegalStateException e = new IllegalStateException(
+ "We were asked to forward an update to ourself, which should not happen name=" + desc.getName() + " isLeader=" + isLeader);
+ log.error("Sending an update to ourself id={}", id, e);
+ throw e;
}
return nodes;
}
@@ -831,7 +847,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
@Override
protected boolean shouldCloneCmdDoc() {
boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
- return willDistrib & cloneRequiredOnLeader;
+ return willDistrib;
}
// helper method, processAdd was getting a bit large.
@@ -982,110 +998,115 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
}
/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
- protected List<SolrCmdDistributor.Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
- Collection<Slice> allSlices = coll.getSlices();
- List<SolrCmdDistributor.Node> nodes = null;
- for (Slice aslice : allSlices) {
- final Slice.State state = aslice.getState();
- if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) {
- DocRouter.Range myRange = coll.getSlice(shardId).getRange();
- if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
- boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
- if (isSubset &&
- (docId == null // in case of deletes
- || coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll))) {
- Replica sliceLeader = aslice.getLeader();
- // slice leader can be null because node/shard is created zk before leader election
- if (sliceLeader != null && zkController.getZkStateReader().isNodeLive(sliceLeader.getNodeName())) {
- if (nodes == null) nodes = new ArrayList<>();
- nodes.add(new SolrCmdDistributor.ForwardNode(zkController.getZkStateReader(), sliceLeader, coll.getName(), aslice.getName()));
+ protected List<SolrCmdDistributor.Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc, UpdateCommand cmd) {
+ if (SkyHookDoc.skyHookDoc != null && cmd instanceof AddUpdateCommand) {
+ SkyHookDoc.skyHookDoc.register(((AddUpdateCommand) cmd).getPrintableId(), "getSubShardLeaders isLeader=true");
+ }
+ try {
+ Collection<Slice> allSlices = coll.getSlices();
+ List<SolrCmdDistributor.Node> nodes = null;
+ for (Slice aslice : allSlices) {
+ final Slice.State state = aslice.getState();
+ if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) {
+ DocRouter.Range myRange = coll.getSlice(shardId).getRange();
+ if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
+ boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
+ if (isSubset && (docId == null // in case of deletes
+ || coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll))) {
+ Replica sliceLeader = aslice.getLeader();
+ // slice leader can be null because node/shard is created zk before leader election
+ if (sliceLeader != null && zkController.getZkStateReader().isNodeLive(sliceLeader.getNodeName())) {
+ if (nodes == null) nodes = new ArrayList<>();
+ nodes.add(new SolrCmdDistributor.ForwardNode(zkController.getZkStateReader(), sliceLeader, coll.getName(), aslice.getName()));
+ }
}
}
}
+ } catch (Throwable t) {
+ log.error("Exception getting sub shard leaders", t);
+ if (t instanceof Error) {
+ throw t;
+ }
+ throw new SolrException(ErrorCode.SERVER_ERROR, t);
}
return nodes;
}
/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#MIGRATE} */
protected List<SolrCmdDistributor.Node> getNodesByRoutingRules(ClusterState cstate, DocCollection coll, String id, SolrInputDocument doc) {
- DocRouter router = coll.getRouter();
- List<SolrCmdDistributor.Node> nodes = null;
- if (router instanceof CompositeIdRouter) {
- CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router;
- String myShardId = cloudDesc.getShardId();
- Slice slice = coll.getSlice(myShardId);
- Map<String, RoutingRule> routingRules = slice.getRoutingRules();
- if (routingRules != null) {
-
- // delete by query case
- if (id == null) {
- for (Map.Entry<String, RoutingRule> entry : routingRules.entrySet()) {
- String targetCollectionName = entry.getValue().getTargetCollectionName();
- final DocCollection docCollection = cstate.getCollectionOrNull(targetCollectionName, true);
- if (docCollection != null && docCollection.getActiveSlices().size() > 0) {
- Collection<Slice> activeSlices = docCollection.getActiveSlices();
- Slice any = activeSlices.iterator().next();
- if (nodes == null) nodes = new ArrayList<>();
- nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), any.getLeader()));
+ try {
+ DocRouter router = coll.getRouter();
+ List<SolrCmdDistributor.Node> nodes = null;
+ if (router instanceof CompositeIdRouter) {
+ CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router;
+ String myShardId = cloudDesc.getShardId();
+ Slice slice = coll.getSlice(myShardId);
+ Map<String,RoutingRule> routingRules = slice.getRoutingRules();
+ if (routingRules != null) {
+
+ // delete by query case
+ if (id == null) {
+ for (Map.Entry<String,RoutingRule> entry : routingRules.entrySet()) {
+ String targetCollectionName = entry.getValue().getTargetCollectionName();
+ final DocCollection docCollection = cstate.getCollectionOrNull(targetCollectionName);
+ if (docCollection != null && docCollection.getActiveSlices().size() > 0) {
+ Collection<Slice> activeSlices = docCollection.getActiveSlices();
+ Slice any = activeSlices.iterator().next();
+ if (nodes == null) nodes = new ArrayList<>();
+ nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), any.getLeader()));
+ }
}
+ return nodes;
}
- return nodes;
- }
- String routeKey = SolrIndexSplitter.getRouteKey(id);
- if (routeKey != null) {
- RoutingRule rule = routingRules.get(routeKey + "!");
- if (rule != null) {
- if (! rule.isExpired()) {
- List<DocRouter.Range> ranges = rule.getRouteRanges();
- if (ranges != null && !ranges.isEmpty()) {
- int hash = compositeIdRouter.sliceHash(id, doc, null, coll);
- for (DocRouter.Range range : ranges) {
- if (range.includes(hash)) {
- DocCollection targetColl = cstate.getCollection(rule.getTargetCollectionName());
- Collection<Slice> activeSlices = targetColl.getRouter().getSearchSlicesSingle(id, null, targetColl);
- if (activeSlices == null || activeSlices.isEmpty()) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "No active slices serving " + id + " found for target collection: " + rule.getTargetCollectionName());
+ String routeKey = SolrIndexSplitter.getRouteKey(id);
+ if (routeKey != null) {
+ RoutingRule rule = routingRules.get(routeKey + "!");
+ if (rule != null) {
+ if (!rule.isExpired()) {
+ List<DocRouter.Range> ranges = rule.getRouteRanges();
+ if (ranges != null && !ranges.isEmpty()) {
+ int hash = compositeIdRouter.sliceHash(id, doc, null, coll);
+ for (DocRouter.Range range : ranges) {
+ if (range.includes(hash)) {
+ DocCollection targetColl = cstate.getCollection(rule.getTargetCollectionName());
+ Collection<Slice> activeSlices = targetColl.getRouter().getSearchSlicesSingle(id, null, targetColl);
+ if (activeSlices == null || activeSlices.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No active slices serving " + id + " found for target collection: " + rule.getTargetCollectionName());
+ }
+ Replica targetLeader = targetColl.getLeader(activeSlices.iterator().next().getName());
+ nodes = new ArrayList<>(1);
+ nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), targetLeader));
+ break;
}
- Replica targetLeader = targetColl.getLeader(activeSlices.iterator().next().getName());
- nodes = new ArrayList<>(1);
- nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), targetLeader));
- break;
}
}
- }
- } else {
- ReentrantLock ruleExpiryLock = req.getCore().getRuleExpiryLock();
- if (!ruleExpiryLock.isLocked()) {
+ } else {
+ log.info("Going to expire routing rule");
try {
- if (ruleExpiryLock.tryLock() || ruleExpiryLock.tryLock(10, TimeUnit.MILLISECONDS)) {
- log.info("Going to expire routing rule");
- try {
- // MRM TODO: TODO: needs to use the statepublisher
- Map<String, Object> map = Utils.makeMap(Overseer.QUEUE_OPERATION, OverseerAction.REMOVEROUTINGRULE.toLower(),
- ZkStateReader.COLLECTION_PROP, collection,
- ZkStateReader.SHARD_ID_PROP, myShardId,
- "routeKey", routeKey + "!");
- zkController.getOverseer().offerStateUpdate(Utils.toJSON(map));
- } catch (KeeperException e) {
- log.warn("Exception while removing routing rule for route key: {}", routeKey, e);
- } catch (Exception e) {
- log.error("Exception while removing routing rule for route key: {}", routeKey, e);
- } finally {
- ruleExpiryLock.unlock();
- }
- }
- } catch (InterruptedException e) {
- ParWork.propagateInterrupt(e);
+ // MRM TODO: TODO: needs to use the statepublisher
+ Map<String,Object> map = Utils
+ .makeMap(Overseer.QUEUE_OPERATION, OverseerAction.REMOVEROUTINGRULE.toLower(), ZkStateReader.COLLECTION_PROP, collection,
+ ZkStateReader.SHARD_ID_PROP, myShardId, "routeKey", routeKey + "!");
+ // zkController.getOverseer().offerStateUpdate(Utils.toJSON(map));
+ } catch (Exception e) {
+ log.error("Exception while removing routing rule for route key: {}", routeKey, e);
}
}
}
}
}
}
+ return nodes;
+ } catch (Throwable t) {
+ log.error("Error getting routing rules", t);
+
+ if (t instanceof Error) {
+ throw t;
+ }
+
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Error getting routing rules", t);
}
- return nodes;
}
private void doDefensiveChecks(DistribPhase phase) {
diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java
index a6bb5d2..d627dce 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java
@@ -115,6 +115,10 @@ public class NestedUpdateProcessorFactory extends UpdateRequestProcessorFactory
}
private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) {
+ if (sdoc == parent) {
+ throw new IllegalArgumentException("Parent and child must be different instances");
+ }
+
if(storePath) {
setPathField(sdoc, fullPath);
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java b/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java
index 5052f83..07754cc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java
@@ -76,7 +76,7 @@ public class CleanupOldIndexTest extends SolrCloudTestCase {
int[] maxDocList;
if (TEST_NIGHTLY) {
- maxDocList = new int[] {300, 500, 700};
+ maxDocList = new int[] {100, 200};
} else {
maxDocList = new int[] {30, 50, 70};
}
@@ -89,9 +89,9 @@ public class CleanupOldIndexTest extends SolrCloudTestCase {
// give some time to index...
int[] waitTimes;
if (TEST_NIGHTLY) {
- waitTimes = new int[] {3000, 4000};
+ waitTimes = new int[] {1000, 3000};
} else {
- waitTimes = new int[] {300, 600};
+ waitTimes = new int[] {300};
}
Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java
index fb3dc6e..5e81a76 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java
@@ -16,19 +16,6 @@
*/
package org.apache.solr.cloud;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.JSONTestUtil;
@@ -45,10 +32,10 @@ import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SkyHookDoc;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -59,6 +46,16 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
import static org.apache.solr.update.processor.DistributedUpdateProcessor.DISTRIB_FROM;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
@@ -84,7 +81,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
final String shardId = "s1";
CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 3)
- .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
+ .process(cluster.getSolrClient());
final ZkStateReader stateReader = cluster.getSolrClient().getZkStateReader();
@@ -100,7 +97,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
// verify doc is on the leader and replica
final List<Replica> notLeaders = stateReader.getClusterState().getCollection(COLLECTION).getReplicas()
.stream()
- .filter(r -> r.getName().equals(leader.getName()) == false)
+ .filter(r -> r.getName().equals(leader.getName()) == false && stateReader.isNodeLive(r.getNodeName()) && r.getState() == Replica.State.ACTIVE)
.collect(Collectors.toList());
assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, 1, null);
@@ -156,15 +153,15 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
try {
Thread.sleep(rand.nextInt(30)+1);
} catch (InterruptedException e) {
- return;
+ log.error("Fail", e);
}
- for (int i=0; i < (TEST_NIGHTLY ? 1000 : 100); i++) {
+ for (int i=1; i < (TEST_NIGHTLY ? 1000 : 100); i++) {
if (i % (rand.nextInt(20)+1) == 0) {
try {
Thread.sleep(rand.nextInt(50)+1);
} catch (InterruptedException e) {
- return;
+ log.error("Fail", e);
}
}
@@ -173,7 +170,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
sendDoc(docId);
docsSent.incrementAndGet();
} catch (Exception e) {
- log.error("id=" + docId, e);
+ log.error("Fail id=" + docId, e);
}
}
}
@@ -188,7 +185,9 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
for (int i=0; i < 3; i++) {
try {
reloadCollection(leader, COLLECTION);
- } catch (Exception e) {}
+ } catch (Exception e) {
+ log.error("Fail", e);
+ }
try {
Thread.sleep(rand.nextInt(300)+300);
@@ -216,6 +215,9 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
if (!deletedDocs.contains(docToDelete)) {
SolrTestCaseJ4.delI(String.valueOf(docToDelete));
deletedDocs.add(docToDelete);
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(String.valueOf(docToDelete), "delete from test client");
+ }
}
}
}
@@ -231,7 +233,9 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
for (int i=0; i < 20; i++) {
try {
cluster.getSolrClient().commit(COLLECTION);
- } catch (Exception e) {}
+ } catch (Exception e) {
+ log.error("Fail", e);
+ }
try {
Thread.sleep(rand.nextInt(100)+100);
@@ -257,7 +261,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
log.info("Total of {} docs deleted", deletedDocs.size());
}
- assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, TEST_NIGHTLY ? 1000 : 100, deletedDocs);
+ assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, docsSent.get(), deletedDocs);
maxOnLeader = getMaxVersionFromIndex(leader);
maxOnReplica = getMaxVersionFromIndex(replica);
@@ -342,7 +346,10 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
doc.addField("id", String.valueOf(docId));
doc.addField("a_t", "hello" + docId);
log.info("Send id={}", docId);
- AbstractFullDistribZkTestBase.sendDocs(cluster.getSolrClient(), COLLECTION, Collections.singletonList(doc));
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.register(String.valueOf(docId), "send from test client");
+ }
+ AbstractFullDistribZkTestBase.sendDocs(cluster.getSolrClient(), COLLECTION, doc);
}
/**
@@ -371,10 +378,6 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
boolean reloadedOk = false;
try (Http2SolrClient client = SolrTestCaseJ4.getHttpSolrClient(replica.getBaseUrl())) {
CoreAdminResponse statusResp = CoreAdminRequest.getStatus(coreName, client);
- long leaderCoreStartTime = statusResp.getStartTime(coreName).getTime();
-
- Thread.sleep(1000);
-
// send reload command for the collection
log.info("Sending RELOAD command for {}", testCollectionName);
CollectionAdminRequest.reloadCollection(testCollectionName)
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
index f110724..599aba5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
@@ -132,6 +132,8 @@ public class TestCloudRecovery extends SolrCloudTestCase {
assertTrue("Timeout waiting for all not live", ClusterStateUtil.waitForAllReplicasNotLive(cloudClient.getZkStateReader(), 45000));
ChaosMonkey.start(cluster.getJettySolrRunners());
+ Thread.sleep(150);
+
cluster.waitForActiveCollection(COLLECTION, 2, 2 * (nrtReplicas + tlogReplicas));
resp = cloudClient.query(COLLECTION, params);
diff --git a/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java
index 28d3a83..5a0f6c6 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java
@@ -49,7 +49,7 @@ public class TestRandomCollapseQParserPlugin extends SolrTestCaseJ4 {
= new String[] {NULL_IGNORE, NULL_COLLAPSE, NULL_EXPAND};
@BeforeClass
- public static void buildIndexAndClient() throws Exception {
+ public static void beforeTestRandomCollapseQParserPlugin() throws Exception {
initCore("solrconfig-minimal.xml", "schema-sorts.xml");
final int totalDocs = SolrTestUtil.atLeast(TEST_NIGHTLY ? 500 : 50);
@@ -78,7 +78,7 @@ public class TestRandomCollapseQParserPlugin extends SolrTestCaseJ4 {
}
@AfterClass
- public static void cleanupStatics() throws Exception {
+ public static void afterTestRandomCollapseQParserPlugin() throws Exception {
deleteCore();
SOLR = null;
ALL_SORT_FIELD_NAMES = ALL_COLLAPSE_FIELD_NAMES = null;
@@ -143,7 +143,7 @@ public class TestRandomCollapseQParserPlugin extends SolrTestCaseJ4 {
final SolrParams mainP = params("q", q, "fl", "id,"+collapseField);
final String csize = random().nextBoolean() ?
- "" : " size=" + TestUtil.nextInt(random(),1,TEST_NIGHTLY ? 10000 : 10);
+ "" : " size=" + TestUtil.nextInt(random(),1,TEST_NIGHTLY ? 2500 : 10);
final String nullPolicy = randomNullPolicy();
final String nullPs = NULL_IGNORE.equals(nullPolicy)
diff --git a/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java b/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
index 85b864a..aba3a6d 100644
--- a/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
+++ b/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
@@ -296,7 +296,7 @@ public class BJQParserTest extends SolrTestCaseJ4 {
try (SolrCore core = h.getCore()) {
Gauge parentFilterCache = null;
Gauge filterCache = null;
- TimeOut timeout = new TimeOut(2, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+ TimeOut timeout = new TimeOut(4, TimeUnit.SECONDS, TimeSource.NANO_TIME);
while (!timeout.hasTimedOut()) {
parentFilterCache = (Gauge) (core.getCoreMetricManager().getRegistry().getMetrics().get("CACHE.searcher.perSegFilter"));
filterCache = (Gauge) (core.getCoreMetricManager().getRegistry().getMetrics().get("CACHE.searcher.filterCache"));
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
index c982b21..bf98cb8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
@@ -240,8 +240,7 @@ public class JavaBinUpdateRequestCodec {
}
private SolrInputDocument listToSolrInputDocument(List<NamedList> namedList) {
- SolrInputDocument doc = SolrInputDocument.THREAD_LOCAL_SolrInputDocument.get();
- doc.clear();
+ SolrInputDocument doc = new SolrInputDocument();
for (int i = 0; i < namedList.size(); i++) {
NamedList nl = namedList.get(i);
if (i == 0) {
diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWork.java b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
index def5574..4cdbe06 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ParWork.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
@@ -356,33 +356,29 @@ public class ParWork implements Closeable {
}
if (closeCalls.size() > 0) {
- List<Future<Object>> results = new ArrayList<>(closeCalls.size());
+ List<Future<Object>> results = new ArrayList<>(closeCalls.size());
- for (Callable call : closeCalls) {
- Future future = executor.submit(call);
- results.add(future);
- }
+ for (Callable call : closeCalls) {
+ Future future = executor.submit(call);
+ results.add(future);
+ }
-// List<Future<Object>> results = executor.invokeAll(closeCalls, 8, TimeUnit.SECONDS);
int i = 0;
- for (Future<Object> future : results) {
- try {
- future.get(
- Long.getLong("solr.parwork.task_timeout", TimeUnit.MINUTES.toMillis(10)),
- TimeUnit.MILLISECONDS); // MRM TODO:
- if (!future.isDone() || future.isCancelled()) {
- log.warn("A task did not finish isDone={} isCanceled={}",
- future.isDone(), future.isCancelled());
- // throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A task did nor finish" +future.isDone() + " " + future.isCancelled());
- }
- } catch (TimeoutException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Timeout", e); // TODO: add object info eg ParObject.label
- } catch (InterruptedException e1) {
- log.warn(WORK_WAS_INTERRUPTED);
- // TODO: save interrupted status and reset it at end?
+ for (Future<Object> future : results) {
+ try {
+ future.get(Long.getLong("solr.parwork.task_timeout", TimeUnit.MINUTES.toMillis(10)), TimeUnit.MILLISECONDS); // MRM TODO:
+ if (!future.isDone() || future.isCancelled()) {
+ log.warn("A task did not finish isDone={} isCanceled={}", future.isDone(), future.isCancelled());
+ // throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A task did nor finish" +future.isDone() + " " + future.isCancelled());
}
-
+ } catch (TimeoutException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Timeout", e); // TODO: add object info eg ParObject.label
+ } catch (InterruptedException e1) {
+ log.warn(WORK_WAS_INTERRUPTED);
+ // TODO: save interrupted status and reset it at end?
}
+
+ }
}
}
} finally {
diff --git a/solr/solrj/src/java/org/apache/solr/common/SkyHookDoc.java b/solr/solrj/src/java/org/apache/solr/common/SkyHookDoc.java
new file mode 100644
index 0000000..f59ec39
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/SkyHookDoc.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.common;
+
+import org.apache.commons.io.output.StringBuilderWriter;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.PrintWriter;
+import java.lang.invoke.MethodHandles;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class SkyHookDoc {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ public static final int STACK = 10;
+ public static SkyHookDoc skyHookDoc = null;//new SkyHookDoc();
+
+ private final Map<String,ConcurrentLinkedDeque> map = Collections.synchronizedSortedMap(new TreeMap<>());
+
+ private final ReentrantLock lock = new ReentrantLock();
+
+ public final static Pattern FIELD_ID = Pattern.compile("\\<id\\:(\\d+)\\>");
+
+ public void register(SolrDocumentBase document) {
+ String sId;
+ try {
+ Object id = document.getFieldValue("id");
+
+ if (id instanceof SolrInputField) {
+ Object val = ((SolrInputField) id).getValue();
+ sId = String.valueOf(val);
+ } else {
+ sId = String.valueOf(id);
+ }
+
+ if (sId.contains("=") || sId.contains(",") || sId.contains(":")) {
+ throw new IllegalArgumentException(sId);
+ }
+
+ } catch (Throwable t) {
+ log.error("SkyHook Exception (BAD!)", t);
+ return;
+ }
+
+ register(sId);
+ }
+
+ private void reg(String sId, String line) {
+
+ if (Integer.parseInt(sId) > 400) {
+ log.info("found high one {}", sId);
+ }
+
+ ConcurrentLinkedDeque<String> deque = map.get(sId);
+
+ if (deque != null) {
+ deque.add(line);
+ return;
+ }
+
+ deque = map.get(sId);
+
+ if (deque != null) {
+ deque.add(line);
+ return;
+ }
+
+ deque = new ConcurrentLinkedDeque();
+ deque.add(line);
+
+ ConcurrentLinkedDeque old = map.put(sId, deque);
+
+ if (old != null) {
+ deque.addAll(old);
+ }
+
+ log.info("SkyHook add Send id={} map={}", sId, map.size());
+ }
+
+ public void register(String sId) {
+ register(sId, "");
+ }
+
+ public void register(String sId, String additional) {
+ try {
+ Matcher m = FIELD_ID.matcher(sId);
+
+ if (m.find()) {
+ sId = m.group(1);
+ }
+
+ StringBuilder sb = ObjectReleaseTracker.getThreadLocalStringBuilder();
+ sb.setLength(0);
+ StringBuilderWriter sw = new StringBuilderWriter(sb);
+ PrintWriter pw = new PrintWriter(sw);
+ printStackTrace(new SolrException(SolrException.ErrorCode.UNKNOWN, ""), pw, STACK);
+ String stack = sw.toString();
+ String line = System.currentTimeMillis() + " registered on node=" + MDCLoggingContext.getNodeName() + (additional != null && additional.length() > 0 ?
+ " " + additional :
+ "") + " originStack=" + stack;
+
+ reg(sId, line);
+ } catch (Throwable t) {
+ log.error("SkyHook Exception (BAD!?) " + additional, t);
+ }
+ }
+
+ public void clear() {
+ map.clear();
+ }
+
+ public void logAll() {
+ try {
+ log.info("SkyHookOutput");
+ synchronized (map) {
+ map.forEach((id, deque) -> {
+
+ log.info("⤞⤞⤞⤞⤞⤞⤠⤠⤠⤠⤠⤠⤠⤗⤗⤗⤗⤗");
+ log.info("⤞⤞⤞⤞⤞⤞⤠⤠⤠⤠⤠⤠⤠⤗⤗⤗⤗⤗ docid={}", id);
+
+ deque.forEach(line -> {
+ log.info("docid={} {}", id, line);
+ });
+
+ });
+ }
+ log.info("SkyHookOutput done");
+ } catch (Throwable t) {
+ log.error("SkyHook exception", t);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, t);
+ }
+ }
+
+ private static void printStackTrace(Throwable t, PrintWriter p, int stack) {
+ // Print our stack trace
+ StackTraceElement[] trace = t.getStackTrace();
+ for (int i = 2; i < Math.min(stack, trace.length); i++) {
+ StackTraceElement traceElement = trace[i];
+ p.println(" " + traceElement);
+ }
+ }
+
+}
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
index d7b268b..c3e06a9 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
@@ -42,6 +42,7 @@ import org.apache.solr.common.util.NamedList;
*/
public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> implements Iterable<Map.Entry<String, Object>>
{
+
protected final Map<String,Object> _fields;
private List<SolrDocument> _childDocuments;
@@ -118,7 +119,22 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
}
value = lst;
}
+
_fields.put(name, value);
+
+ if (SkyHookDoc.skyHookDoc != null && "id".equals(name)) {
+ if (value instanceof SolrInputField) {
+ SkyHookDoc.skyHookDoc.register(String.valueOf(((SolrInputField) value).getValue()));
+ } else {
+ try {
+ SkyHookDoc.skyHookDoc.register(String.valueOf(value));
+ } catch (NumberFormatException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "class=" + value.getClass().getName(), e);
+ }
+
+ }
+
+ }
}
/**
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
index 73a2946..3427a75 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
@@ -102,6 +102,9 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
*/
public void addField(String name, Object value)
{
+ if (value == this) {
+ throw new IllegalArgumentException("Parent and child must be different instances");
+ }
SolrInputField field = _fields.get( name );
if( field == null || field.value == null ) {
setField(name, value);
@@ -160,6 +163,10 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
SolrInputField field = new SolrInputField( name );
_fields.put( name, field );
field.setValue( value );
+
+ if (SkyHookDoc.skyHookDoc != null && "id".equals(name)) {
+ SkyHookDoc.skyHookDoc.register(this);
+ }
}
/**
@@ -205,6 +212,9 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
if (_childDocuments != null) {
clone._childDocuments = new ArrayList<>(_childDocuments.size());
for (SolrInputDocument child : _childDocuments) {
+ if (child == clone) {
+ throw new IllegalArgumentException("Document cannot be child of itself");
+ }
clone._childDocuments.add(child.deepCopy());
}
}
@@ -273,6 +283,9 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
@Override
public void addChildDocument(SolrInputDocument child) {
+ if (child == this) {
+ throw new IllegalArgumentException("Document cannot be child of itself");
+ }
if (_childDocuments == null) {
_childDocuments = new ArrayList<>();
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index 131a7bd..eb47118 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -1328,6 +1328,10 @@ public class SolrZkClient implements Closeable {
this.connManager.setDisconnectListener(dl);
}
+ public void addWatch(String basePath, Watcher watcher, AddWatchMode mode) throws KeeperException, InterruptedException {
+ getSolrZooKeeper().addWatch(basePath, watcher == null ? null : wrapWatcher(watcher), mode);
+ }
+
public void addWatch(String basePath, Watcher watcher, AddWatchMode mode, AsyncCallback.VoidCallback cb, Object ctx) {
getSolrZooKeeper().addWatch(basePath, watcher == null ? null : wrapWatcher(watcher), mode, cb, ctx);
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 7da7290..087c982 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -761,29 +761,21 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
gets.incrementAndGet();
if (!allowCached || lastUpdateTime < 0 || System.nanoTime() - lastUpdateTime > LAZY_CACHE_TIME) {
boolean shouldFetch = true;
- DocCollection cached = cachedDocCollection;
- if (cached != null) {
+ if (cachedDocCollection != null) {
Stat exists = null;
try {
exists = zkClient.exists(getCollectionPath(collName), null, true);
} catch (Exception e) {
- ParWork.propagateInterrupt(e);
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
- if (exists != null && exists.getVersion() == cached.getZNodeVersion()) {
+ if (exists != null && exists.getVersion() == cachedDocCollection.getZNodeVersion()) {
shouldFetch = false;
}
}
if (shouldFetch) {
- cached = getCollectionLive(ZkStateReader.this, collName);
+ cachedDocCollection = getCollectionLive(ZkStateReader.this, collName);
lastUpdateTime = System.nanoTime();
- cachedDocCollection = cached;
- return cached;
}
}
- if (log.isDebugEnabled() && cachedDocCollection == null) {
- log.debug("cached collection is null");
- }
return cachedDocCollection;
}
@@ -1422,11 +1414,13 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
public void createWatch() {
String collectionCSNPath = getCollectionSCNPath(coll);
+ CountDownLatch latch = new CountDownLatch(2);
zkClient.addWatch(collectionCSNPath, this, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
if (rc != 0) {
KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
log.error("Exception creating watch for " + path, ex);
}
+ latch.countDown();
}, "collectionStateWatcher:" + coll);
zkClient.addWatch(stateUpdateWatcher.stateUpdatesPath, stateUpdateWatcher, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
@@ -1434,10 +1428,18 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
log.error("Exception creating watch for " + path, ex);
}
+ latch.countDown();
}, "collectionStateUpdatesWatcher:" + coll);
+
+ try {
+ latch.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ ParWork.propagateInterrupt(e);
+ }
}
public void removeWatch() {
+ CountDownLatch latch = new CountDownLatch(2);
String collectionCSNPath = getCollectionSCNPath(coll);
zkClient.removeWatches(collectionCSNPath, this, WatcherType.Any, true, (rc, path, ctx) -> {
if (rc != 0) {
@@ -1445,6 +1447,7 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
if (!(ex instanceof KeeperException.NoWatcherException)) {
log.error("Exception removing watch for " + path, ex);
}
+ latch.countDown();
}
}, "collectionStateWatcher:" + coll);
@@ -1455,7 +1458,14 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
log.error("Exception removing watch for " + path, ex);
}
}
+ latch.countDown();
}, "collectionStateUpdatesWatcher:" + coll);
+
+ try {
+ latch.await(5, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ ParWork.propagateInterrupt(e);
+ }
}
public void refreshStateUpdates() {
@@ -1830,23 +1840,21 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
}
public void createWatch() {
- zkClient.addWatch(COLLECTIONS_ZKNODE, this, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
- if (rc != 0) {
- KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
- log.error("Exception creating watch for " + path, ex);
- }
- }, "collectionsChildWatcher");
+ try {
+ zkClient.addWatch(COLLECTIONS_ZKNODE, this, AddWatchMode.PERSISTENT);
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
}
public void removeWatch() {
- zkClient.removeWatches(COLLECTIONS_ZKNODE, this, WatcherType.Any, true, (rc, path, ctx) -> {
- if (rc != 0) {
- KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
- if (!(ex instanceof KeeperException.NoWatcherException)) {
- log.error("Exception removing watch for " + path, ex);
- }
- }
- }, "collectionsChildWatcher");
+ try {
+ zkClient.removeWatches(COLLECTIONS_ZKNODE, this, WatcherType.Any, true);
+ } catch (KeeperException.NoWatcherException e) {
+
+ } catch (Exception e) {
+ log.warn("Exception removing watch", e);
+ }
}
@Override
@@ -1866,9 +1874,6 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
if (EventType.None.equals(event.getType())) {
return;
}
- if (closed || zkClient.isClosed()) {
- return;
- }
if (node != null) {
MDCLoggingContext.setNode(node);
}
@@ -1882,33 +1887,29 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
public void refresh() {
try {
refreshLiveNodes();
- } catch (KeeperException e) {
+ } catch (Exception e) {
log.error("A ZK error has occurred", e);
- } catch (InterruptedException e) {
- log.warn("Interrupted", e);
}
}
public void createWatch() {
- zkClient.addWatch(LIVE_NODES_ZKNODE, this, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
- if (rc != 0) {
- KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
- log.error("Exception creating watch for " + path, ex);
- }
- }, "liveNodesWatcher");
+ try {
+ zkClient.addWatch(LIVE_NODES_ZKNODE, this, AddWatchMode.PERSISTENT);
+ } catch (Exception e) {
+ log.warn("Exception creating watch", e);
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Exception creating watch", e);
+ }
}
public void removeWatch() {
- zkClient.removeWatches(LIVE_NODES_ZKNODE, this, WatcherType.Any, true, (rc, path, ctx) -> {
- if (rc != 0) {
- KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
- if (!(ex instanceof KeeperException.NoWatcherException)) {
- log.error("Exception removing watch for " + path, ex);
- }
- }
- }, "liveNodesWatcher");
- }
+ try {
+ zkClient.removeWatches(LIVE_NODES_ZKNODE, this, WatcherType.Any, true);
+ } catch (KeeperException.NoWatcherException e) {
+ } catch (Exception e) {
+ log.warn("Exception removing watch", e);
+ }
+ }
@Override
public void close() throws IOException {
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java b/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
index 13e7a50..d5c2f90 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
@@ -20,8 +20,6 @@ import org.apache.commons.io.output.StringBuilderWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Closeable;
-import java.io.IOException;
import java.io.PrintWriter;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@@ -48,13 +46,13 @@ public class ObjectReleaseTracker {
}
public static boolean track(Object object) {
- ObjectTrackerException ote = new ObjectTrackerException(object.getClass().getName());
- OBJECTS.put(object, ote);
+ ObjectTrackerException ote = new ObjectTrackerException(object.getClass().getName() + "@" + Integer.toHexString(object.hashCode()));
+ OBJECTS.put(object.getClass().getName() + "@" + Integer.toHexString(object.hashCode()), ote);
return true;
}
public static boolean release(Object object) {
- OBJECTS.remove(object);
+ OBJECTS.remove(object.getClass().getName() + "@" + Integer.toHexString(object.hashCode()));
return true;
}
@@ -83,13 +81,6 @@ public class ObjectReleaseTracker {
for (Entry<Object,ObjectTrackerException> entry : entries.entrySet()) {
if (object != null && entry.getKey().getClass().getSimpleName().equals(object)) {
entries.remove(entry.getKey());
- if (entry.getKey() instanceof Closeable) {
- try {
- ((Closeable) entry.getKey()).close();
- } catch (IOException e) {
- log.warn("Exception trying to close", e);
- }
- }
continue;
}
objects.add(entry.getKey().getClass().getSimpleName());
@@ -119,7 +110,7 @@ public class ObjectReleaseTracker {
// Print our stack trace
p.println(t);
StackTraceElement[] trace = t.getStackTrace();
- for (int i = 0; i < stack; i++) {
+ for (int i = 0; i < Math.min(stack, trace.length); i++) {
StackTraceElement traceElement = trace[i];
p.println("at " + traceElement);
}
diff --git a/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java b/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java
index d401548..da4689a 100644
--- a/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java
+++ b/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java
@@ -75,6 +75,10 @@ public class MDCLoggingContext {
}
}
+ public static String getNodeName() {
+ return MDC.get(NODE_NAME_PROP);
+ }
+
/**
* Call this in a
* finally.
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java
index b640e52..76d2f19 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java
@@ -46,6 +46,7 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.common.AlreadyClosedException;
import org.apache.solr.common.ParWork;
import org.apache.solr.common.ParWorkExecutor;
+import org.apache.solr.common.SkyHookDoc;
import org.apache.solr.common.StringUtils;
import org.apache.solr.common.TimeTracker;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -588,6 +589,11 @@ public class SolrTestCase extends Assert {
reusedKeys = null;
sslConfig = null;
+ if (SkyHookDoc.skyHookDoc != null) {
+ SkyHookDoc.skyHookDoc.logAll();
+ SkyHookDoc.skyHookDoc.clear();
+ }
+
long testTime = TimeUnit.SECONDS.convert(System.nanoTime() - testStartTime, TimeUnit.NANOSECONDS);
if (!LuceneTestCase.TEST_NIGHTLY && testTime > SOLR_TEST_TIMEOUT) {
log.error("This test suite is too long for non @Nightly runs! Please improve it's performance, break it up, make parts of it @Nightly or make the whole suite @Nightly: {}"
@@ -632,10 +638,9 @@ public class SolrTestCase extends Assert {
log.info("*******************************************************************");
}
- StartupLoggingUtils.shutdown();
-
checkForInterruptRequest();
interuptThreadWithNameContains.clear();
+ StartupLoggingUtils.shutdown();
}
private static void checkForInterruptRequest() {
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index a65cacd..03b3a5f 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -995,21 +995,18 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
}
}
- protected static int sendDocs(CloudHttp2SolrClient cloudClient, String collection, List<SolrInputDocument> batch) throws Exception {
+ protected static void sendDocs(CloudHttp2SolrClient cloudClient, String collection, SolrInputDocument doc) throws Exception {
UpdateRequest up = new UpdateRequest();
- up.add(batch);
+ up.add(doc);
NamedList resp = null;
try {
resp = cloudClient.request(up, collection);
- return cloudClient.getMinAchievedReplicationFactor(cloudClient.getDefaultCollection(), resp);
} catch (Exception exc) {
ParWork.propagateInterrupt(exc);
Throwable rootCause = SolrException.getRootCause(exc);
-
- log.error(" Add batch failed due to: {}", rootCause);
+ log.error(" Add doc failed due to: {}", rootCause);
throw exc;
-
}
}
diff --git a/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml b/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
index d5325e2..b6088b7 100644
--- a/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
+++ b/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
@@ -24,7 +24,7 @@
</Console>
<File name="FILE" fileName="${sys:user.home}/solr-test.log" immediateFlush="false" append="false">
- <PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-5p}{6}} %style{(%t)}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
+ <PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-5p}{6}} %style{(%maxLen{%t}{8})}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
</File>
</Appenders>
@@ -54,8 +54,8 @@
<AsyncLogger name="org.apache.solr.client.solrj.impl.LBSolrClient" level="INFO"/>
<AsyncLogger name="org.apache.solr.cloud.ZkController" level="INFO"/>
<AsyncLogger name="org.apache.solr.common.cloud.ZkMaintenanceUtils" level="INFO"/>
- <AsyncLogger name="org.apache.solr.update.processor.DistributedZkUpdateProcessor" level="INFO"/>
- <AsyncLogger name="org.apache.solr.update.processor.DistributedUpdateProcessor" level="INFO"/>
+ <AsyncLogger name="org.apache.solr.update.processor.DistributedZkUpdateProcessor" level="DEBUG"/>
+ <AsyncLogger name="org.apache.solr.update.processor.DistributedUpdateProcessor" level="DEBUG"/>
<AsyncLogger name="org.apache.solr.update.SolrCmdDistributor" level="INFO"/>
<AsyncLogger name="org.apache.solr.update.processor.LogUpdateProcessorFactory" level="INFO"/>