You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2021/02/27 01:55:51 UTC

[lucene-solr] 01/02: @1408 Zhu Li, the nightly thing please.

This is an automated email from the ASF dual-hosted git repository.

markrmiller pushed a commit to branch reference_impl
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git

commit 9ca26221bf04dbe51db9256f7ea76ed0cffb57a0
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Fri Feb 26 19:47:28 2021 -0600

    @1408 Zhu Li, the nightly thing please.
    
    Took 9 hours 22 minutes
    
    Took 4 minutes
---
 .../src/java/org/apache/solr/cloud/Overseer.java   |  28 +-
 .../src/java/org/apache/solr/core/SolrCore.java    |  11 +-
 .../apache/solr/filestore/NRTCachingDirectory.java |  10 +-
 .../handler/component/RealTimeGetComponent.java    |   3 +-
 .../org/apache/solr/handler/loader/CSVLoader.java  |   3 +-
 .../org/apache/solr/update/AddUpdateCommand.java   |  11 +-
 .../org/apache/solr/update/SolrCmdDistributor.java |  24 +-
 .../org/apache/solr/update/UpdateShardHandler.java |  15 -
 .../processor/DistributedUpdateProcessor.java      |  49 ++-
 .../processor/DistributedZkUpdateProcessor.java    | 329 +++++++++++----------
 .../processor/NestedUpdateProcessorFactory.java    |   4 +
 .../org/apache/solr/cloud/CleanupOldIndexTest.java |   6 +-
 .../solr/cloud/DistributedVersionInfoTest.java     |  59 ++--
 .../org/apache/solr/cloud/TestCloudRecovery.java   |   2 +
 .../search/TestRandomCollapseQParserPlugin.java    |   6 +-
 .../org/apache/solr/search/join/BJQParserTest.java |   2 +-
 .../solrj/request/JavaBinUpdateRequestCodec.java   |   3 +-
 .../src/java/org/apache/solr/common/ParWork.java   |  40 ++-
 .../java/org/apache/solr/common/SkyHookDoc.java    | 165 +++++++++++
 .../java/org/apache/solr/common/SolrDocument.java  |  16 +
 .../org/apache/solr/common/SolrInputDocument.java  |  13 +
 .../org/apache/solr/common/cloud/SolrZkClient.java |   4 +
 .../apache/solr/common/cloud/ZkStateReader.java    |  93 +++---
 .../solr/common/util/ObjectReleaseTracker.java     |  17 +-
 .../org/apache/solr/logging/MDCLoggingContext.java |   4 +
 .../src/java/org/apache/solr/SolrTestCase.java     |   9 +-
 .../solr/cloud/AbstractFullDistribZkTestBase.java  |   9 +-
 .../src/resources/logconf/log4j2-std-debug.xml     |   6 +-
 28 files changed, 608 insertions(+), 333 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 0cb1288..88db849 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -513,9 +513,13 @@ public class Overseer implements SolrCloseable {
       synchronized (this) {
         retry = !zkController.getCoreContainer().isShutDown() && !zkController.isShutdownCalled() && !zkController.isClosed() && !closeAndDone;
       }
-      if (retry) {
+      if (retry && zkController.getZkClient().isAlive()) {
         log.info("rejoining the overseer election after closing");
-        zkController.rejoinOverseerElection(false);
+        try {
+          zkController.rejoinOverseerElection(false);
+        } catch (AlreadyClosedException e) {
+          return;
+        }
       }
 
     }
@@ -880,10 +884,22 @@ public class Overseer implements SolrCloseable {
         overseer.writePendingUpdates();
 
         if (overseer.zkStateWriter != null) {
-          try {
-            zkController.getZkClient().delete(fullPaths, true);
-          } catch (Exception e) {
-            log.warn("Failed deleting processed items", e);
+          if (zkController.getZkClient().isAlive()) {
+            try {
+              zkController.getZkClient().delete(fullPaths, true);
+            } catch (KeeperException.SessionExpiredException | KeeperException.ConnectionLossException e) {
+              if (zkController.getZkClient().isAlive()) {
+                try {
+                  zkController.getZkClient().delete(fullPaths, true);
+                } catch (KeeperException keeperException) {
+                  log.warn("Failed deleting processed items", e);
+                }
+              } else {
+                log.warn("Failed deleting processed items", e);
+              }
+            } catch (Exception e) {
+              log.warn("Failed deleting processed items", e);
+            }
           }
         }
 
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 0bb41b1..ac1bc5f 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -232,7 +232,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
 
   private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
 
-  private final ReentrantLock ruleExpiryLock;
   private final ReentrantLock snapshotDelLock; // A lock instance to guard against concurrent deletions.
 
   private volatile Timer newSearcherTimer;
@@ -734,7 +733,7 @@ public final class SolrCore implements SolrInfoBean, Closeable {
         log.info("Wait for reload lock");
 
         while (!(lock.tryLock() || lock.tryLock(250, TimeUnit.MILLISECONDS))) {
-          if (closing) {
+          if (closing || coreContainer.isShutDown()) {
             log.warn("Skipping reload because we are closed");
             reloadyWaiting.decrementAndGet();
             throw new AlreadyClosedException();
@@ -1148,7 +1147,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
       bufferUpdatesIfConstructing(coreDescriptor);
       timeBufferUpdatesIfConstructing.done();
 
-      this.ruleExpiryLock = new ReentrantLock();
       this.snapshotDelLock = new ReentrantLock();
 
     } catch (Throwable e) {
@@ -1750,7 +1748,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {
 
     if (count < -1) {
       refCount.set(-1);
-      throw new IllegalStateException("Already closed " + count);
+      log.warn("Already closed " + count);
+      return;
     }
 
 //    if (log.isDebugEnabled()) {
@@ -3410,10 +3409,6 @@ public final class SolrCore implements SolrInfoBean, Closeable {
     return snapshotMgr;
   }
 
-  public ReentrantLock getRuleExpiryLock() {
-    return ruleExpiryLock;
-  }
-
   /////////////////////////////////////////////////////////////////////
   // SolrInfoBean stuff: Statistics and Module Info
   /////////////////////////////////////////////////////////////////////
diff --git a/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java b/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java
index 78a2d96..f58d0e2 100644
--- a/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java
+++ b/solr/core/src/java/org/apache/solr/filestore/NRTCachingDirectory.java
@@ -181,9 +181,15 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
     for(String fileName : fileNames) {
       unCache(fileName);
     }
-    //if (Boolean.getBoolean("solr.nrtDirSync")) { // MRM TODO:
+
+    // MRM TODO: we should prob link this setting to tlog sync
+    // with replicas, doesn't make a lot of sense to use, but without replicas
+    // or in certain cases, you would want to enable both - otherwise the cost
+    // is very high and a replica that goes out, by the time it comes back is generally
+    // going to just be recovering from the new leader
+    if (Boolean.getBoolean("solr.nrtDirSync")) {
       in.sync(fileNames);
-    //}
+    }
   }
 
   @Override
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 5f108ee..fe30f00 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -641,7 +641,8 @@ public class RealTimeGetComponent extends SearchComponent
 
         SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
         if (onlyTheseNonStoredDVs != null) {
-          sid = new SolrInputDocument();
+          sid = SolrInputDocument.THREAD_LOCAL_SolrInputDocument.get();
+          sid.clear();
         } else {
           Document luceneDocument = docFetcher.doc(docid);
           sid = toSolrInputDocument(luceneDocument, schema);
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
index 3af21e1..d04ca82 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
@@ -50,7 +50,8 @@ class SingleThreadedCSVLoader extends CSVLoaderBase {
     templateAdd.setReq(req);
     templateAdd.overwrite = overwrite;
     templateAdd.commitWithin = commitWithin;
-    SolrInputDocument doc = new SolrInputDocument();
+    SolrInputDocument doc = SolrInputDocument.THREAD_LOCAL_SolrInputDocument.get();
+    doc.clear();
     doAdd(line, vals, doc, templateAdd);
   }
 }
diff --git a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
index 32e5d62..3bcc3cb 100644
--- a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
+++ b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Document;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.solr.common.SkyHookDoc;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;
@@ -139,14 +140,18 @@ public class AddUpdateCommand extends UpdateCommand {
            int count = field==null ? 0 : field.getValueCount();
            if (count == 0) {
              if (overwrite) {
-               throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Document is missing mandatory uniqueKey field: " + sf.getName());
+               throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Document is missing mandatory uniqueKey field: " + sf.getName() + solrDoc);
              }
            } else if (count  > 1) {
-             throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Document contains multiple values for uniqueKey field: " + field);
+             throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Document contains multiple values for uniqueKey field: " + field);
            } else {
              BytesRefBuilder b = new BytesRefBuilder();
              sf.getType().readableToIndexed(field.getFirstValue().toString(), b);
              indexedId = b.get();
+
+             if (SkyHookDoc.skyHookDoc != null) {
+               SkyHookDoc.skyHookDoc.register(field.getFirstValue().toString());
+             }
            }
          }
        }
@@ -195,7 +200,7 @@ public class AddUpdateCommand extends UpdateCommand {
           if (overwrite) {
             throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
                 "Document is missing mandatory uniqueKey field: "
-                    + sf.getName());
+                    + sf.getName() + " solrDoc=" + solrDoc);
           }
         } else if (count > 1) {
           throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 023231c..f9f74a5 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -237,20 +237,22 @@ public class SolrCmdDistributor implements Closeable {
 
   private void submit(final Req req, String tag) {
 
-    if (cancelExeption != null) {
-      Throwable exp = cancelExeption;
-      cancelExeption = null;
-      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exp);
-    }
+//    if (cancelExeption != null) {
+//      Throwable exp = cancelExeption;
+//      cancelExeption = null;
+//      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exp);
+//    }
 
-    if (log.isDebugEnabled()) {
-      log.debug("sending update to " + req.node.getUrl() + " retry:" + req.retries + " " + req.cmd + " params:" + req.uReq.getParams());
+//    if (log.isDebugEnabled()) {
+//      log.debug("sending update to " + req.node.getUrl() + " retry:" + req.retries + " " + req.cmd + " params:" + req.uReq.getParams());
+//    }
+    if (req.cmd instanceof AddUpdateCommand) {
+      log.info("sending update to " + req.node.getUrl() + " retry:" + req.retries + " docid=" + ((AddUpdateCommand) req.cmd).getPrintableId() + " " + req.cmd + " params:" + req.uReq.getParams());
+    } else {
+      log.info("sending update to " + req.node.getUrl() + " retry:" + req.retries + " docid=" + req.cmd + " params:" + req.uReq.getParams());
     }
 
-    if (isClosed != null && isClosed.isClosed()) {
-      log.warn("Already closed, skippping update");
-      throw new AlreadyClosedException();
-    }
+
 
     req.uReq.setBasePath(req.node.getUrl());
 
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
index f270e0e..6e8ead8 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateShardHandler.java
@@ -24,11 +24,9 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.solr.client.solrj.impl.BaseCloudSolrClient;
 import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
-import org.apache.solr.client.solrj.impl.HttpListenerFactory;
 import org.apache.solr.common.ParWork;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.ModifiableSolrParams;
@@ -42,7 +40,6 @@ import org.apache.solr.update.processor.DistributedUpdateProcessor;
 import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
 import org.apache.solr.util.stats.HttpClientMetricNameStrategy;
 import org.apache.solr.util.stats.InstrumentedHttpRequestExecutor;
-import org.apache.solr.util.stats.InstrumentedPoolingHttpClientConnectionManager;
 import org.apache.solr.util.stats.MetricUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -63,8 +60,6 @@ public class UpdateShardHandler implements SolrInfoBean {
 
   private ExecutorService recoveryExecutor;
 
-  private final InstrumentedPoolingHttpClientConnectionManager defaultConnectionManager;
-
   private final InstrumentedHttpRequestExecutor httpRequestExecutor;
 
   //private final InstrumentedHttpListenerFactory updateHttpListenerFactory;
@@ -79,11 +74,8 @@ public class UpdateShardHandler implements SolrInfoBean {
   public UpdateShardHandler(UpdateShardHandlerConfig cfg) {
     assert ObjectReleaseTracker.track(this);
     assert (closeTracker = new CloseTracker()) != null;
-    defaultConnectionManager = new InstrumentedPoolingHttpClientConnectionManager(HttpClientUtil.getSocketFactoryRegistryProvider().getSocketFactoryRegistry());
     ModifiableSolrParams clientParams = new ModifiableSolrParams();
     if (cfg != null ) {
-      defaultConnectionManager.setMaxTotal(cfg.getMaxUpdateConnections());
-      defaultConnectionManager.setDefaultMaxPerRoute(cfg.getMaxUpdateConnectionsPerHost());
       clientParams.set(HttpClientUtil.PROP_SO_TIMEOUT, cfg.getDistributedSocketTimeout());
       clientParams.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, cfg.getDistributedConnectionTimeout());
       // following is done only for logging complete configuration.
@@ -178,7 +170,6 @@ public class UpdateShardHandler implements SolrInfoBean {
     solrMetricsContext = parentContext.getChildContext(this);
     String expandedScope = SolrMetricManager.mkName(scope, getCategory().name());
     //.initializeMetrics(solrMetricsContext, expandedScope);
-    defaultConnectionManager.initializeMetrics(solrMetricsContext, expandedScope);
     recoveryExecutor = MetricUtils.instrumentedExecutorService(recoveryExecutor, this, solrMetricsContext.getMetricRegistry(),
             SolrMetricManager.mkName("recoveryExecutor", expandedScope, "threadPool"));
   }
@@ -211,11 +202,6 @@ public class UpdateShardHandler implements SolrInfoBean {
     return searchOnlyClient;
   }
 
-
-  public PoolingHttpClientConnectionManager getDefaultConnectionManager() {
-    return defaultConnectionManager;
-  }
-
   /**
    * 
    * @return executor for recovery operations
@@ -235,7 +221,6 @@ public class UpdateShardHandler implements SolrInfoBean {
       closer.collect(recoveryOnlyClient);
       closer.collect(searchOnlyClient);
       closer.collect(updateOnlyClient);
-      closer.collect(defaultConnectionManager);
       closer.collect("SolrInfoBean", () -> {
         SolrInfoBean.super.close();
         return this;
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index b30a584..69fb77f 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -39,6 +39,7 @@ import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.request.GenericSolrRequest;
 import org.apache.solr.client.solrj.response.SimpleSolrResponse;
 import org.apache.solr.common.ParWork;
+import org.apache.solr.common.SkyHookDoc;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
@@ -229,7 +230,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
     // the request right here but for now I think it is better to just return the status
     // to the client that the minRf wasn't reached and let them handle it    
 
-    log.info("docid={} isLeader={} forwardToLeader={} nodes={}, params={}", cmd.getPrintableId(), isLeader, forwardToLeader, getNodes(), req.getParams());
+    if (log.isDebugEnabled()) log.debug("docid={} isLeader={} forwardToLeader={} nodes={}, params={}", cmd.getPrintableId(), isLeader, forwardToLeader, getNodes(), req.getParams());
 
     if (!forwardToLeader) {
       Future distFuture = versionAdd(cmd);
@@ -250,9 +251,15 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
       } else {
         // drop it
         log.info("drop docid={}", cmd.getPrintableId());
+        if (SkyHookDoc.skyHookDoc != null) {
+          SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "final notice of dropping update, returning from processAdd cmd=" + cmd);
+        }
         return;
       }
     } else {
+      if (SkyHookDoc.skyHookDoc != null) {
+        SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "distribute update to replicas if necessary cmd=" + cmd);
+      }
       doDistribAdd(cmd);
     }
 
@@ -303,6 +310,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
     BytesRef idBytes = cmd.getIndexedId();
 
     if (idBytes == null) {
+      if (SkyHookDoc.skyHookDoc != null) {
+        SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "local add due to null idBytes cmd=" + cmd);
+      }
       super.processAdd(cmd);
       return CompletableFuture.completedFuture(null);
     }
@@ -312,6 +322,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
         throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
             "Atomic document updates are not supported unless <updateLog/> is configured");
       } else {
+        if (SkyHookDoc.skyHookDoc != null) {
+          SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "local add due to null vinfo cmd=" + cmd);
+        }
         super.processAdd(cmd);
         return CompletableFuture.completedFuture(null);
       }
@@ -357,6 +370,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
       if (dependentVersionFound == -1) {
         // it means the document has been deleted by now at the leader. drop this update
         log.info("docid={} it means the document has been deleted by now at the leader. drop this update", cmd.getPrintableId());
+        if (SkyHookDoc.skyHookDoc != null) {
+          SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, dependent update has been deleted by the leader cmd=" + cmd);
+        }
         return null;
       }
     }
@@ -412,6 +428,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
           }
           cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
           ulog.add(cmd);
+          if (SkyHookDoc.skyHookDoc != null) {
+            SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, leader logic applied, but we are buffering - added to ulog only");
+          }
           return null;
         }
 
@@ -423,6 +442,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
             // specified it must exist (versionOnUpdate==1) and it does.
           } else {
             if(cmd.getReq().getParams().getBool(CommonParams.FAIL_ON_VERSION_CONFLICTS, true) == false) {
+              if (SkyHookDoc.skyHookDoc != null) {
+                SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, failed on version conflict but set not to fail");
+              }
               return null;
             }
 
@@ -445,6 +467,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
           cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);
           ulog.add(cmd);
           log.info("docid={} dropped because not active and buffering and not a replay update", cmd.getPrintableId());
+          if (SkyHookDoc.skyHookDoc != null) {
+            SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, non logic applied, but we are buffering - added to ulog only");
+          }
           return null;
         }
 
@@ -465,6 +490,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
               }
               versionDelete((DeleteUpdateCommand) fetchedFromLeader);
               log.info("docid={} dropped due to missing dependent update", cmd.getPrintableId());
+              if (SkyHookDoc.skyHookDoc != null) {
+                SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, inplace update with too low a version");
+              }
               return null;
             } else {
               assert fetchedFromLeader instanceof AddUpdateCommand;
@@ -485,6 +513,12 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
               // this means we got a newer full doc update and in that case it makes no sense to apply the older
               // inplace update. Drop this update
               log.info("Update was applied on version: {}, but last version I have is: {}. Dropping current update, docid={}", prev, lastVersion, cmd.getPrintableId());
+              if (SkyHookDoc.skyHookDoc != null) {
+                SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, leader logic applied, but we are buffering - added to ulog only");
+              }
+              if (SkyHookDoc.skyHookDoc != null) {
+                SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, we have a newer version for inplace update");
+              }
               return null;
             } else {
               // We're good, we should apply this update. First, update the bucket's highest.
@@ -507,7 +541,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
               // This update is a repeat, or was reordered. We need to drop this update.
 
               log.info("Dropping add update due to version docid={} lastVersion={} versionOnUpdate={}", idBytes.utf8ToString(), lastVersion, versionOnUpdate);
-
+              if (SkyHookDoc.skyHookDoc != null) {
+                SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "dropping update, we have a newer version");
+              }
               return null;
             }
           }
@@ -547,7 +583,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
     // This refresh makes RTG handler aware of this update.q
 
       try {
-        log.info("do local add for docid={}", cmd. getPrintableId());
+        if (log.isDebugEnabled()) log.debug("do local add for docid={}", cmd. getPrintableId());
+        if (SkyHookDoc.skyHookDoc != null) {
+          SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "do local add");
+        }
         doLocalAdd(cmd);
       } catch (Exception e) {
         Throwable t;
@@ -1517,6 +1556,10 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
         log.trace("Run distrib add collection");
       }
       try {
+        if (SkyHookDoc.skyHookDoc != null) {
+          SkyHookDoc.skyHookDoc.register(finalCloneCmd.getPrintableId(), "do distrib update after versionAdd");
+        }
+
         doDistribAdd(finalCloneCmd);
         if (log.isTraceEnabled()) {
           log.trace("after distrib add collection");
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
index afb7e05..4f59f8e 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java
@@ -28,9 +28,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.cloud.CloudDescriptor;
@@ -39,6 +37,7 @@ import org.apache.solr.cloud.ZkController;
 import org.apache.solr.cloud.ZkShardTerms;
 import org.apache.solr.cloud.overseer.OverseerAction;
 import org.apache.solr.common.ParWork;
+import org.apache.solr.common.SkyHookDoc;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
@@ -72,7 +71,6 @@ import org.apache.solr.update.SolrCmdDistributor;
 import org.apache.solr.update.SolrIndexSplitter;
 import org.apache.solr.update.UpdateCommand;
 import org.apache.solr.util.TestInjection;
-import org.apache.zookeeper.KeeperException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -187,9 +185,6 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
           return;
         }
 
-        nodes = getCollectionUrls(collection,
-            EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT), true);
-
         try {
           leaderReplica = clusterState.getCollection(collection).getSlice(cloudDesc.getShardId()).getLeader();
         } catch (Exception e) {
@@ -220,45 +215,37 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
         } else if (req.getParams().get(COMMIT_END_POINT, "").equals("leaders")) {
 
           sendCommitToReplicasAndLocalCommit(cmd, worker, leaderReplica, params);
-        } else {
+        }  else if (req.getParams().get(COMMIT_END_POINT) == null) {
           // zk
+          List<SolrCmdDistributor.Node> useNodes = getCollectionUrls(collection, EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT), true);
 
           if (isLeader) {
             SolrCmdDistributor.Node removeNode = null;
-            for (SolrCmdDistributor.Node node : nodes) {
+            for (SolrCmdDistributor.Node node : useNodes) {
               if (node.getCoreName().equals(this.desc.getName())) {
                 removeNode = node;
               }
             }
             if (removeNode != null) {
-              nodes.remove(removeNode);
+              useNodes.remove(removeNode);
 
               sendCommitToReplicasAndLocalCommit(cmd, worker, leaderReplica, params);
             }
           }
 
-          List<SolrCmdDistributor.Node> useNodes = null;
-          if (req.getParams().get(COMMIT_END_POINT) == null) {
-            useNodes = nodes;
+          params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
+          params.set(COMMIT_END_POINT, "leaders");
 
-            params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
-            params.set(COMMIT_END_POINT, "leaders");
+          if (useNodes != null && useNodes.size() > 0) {
+            if (log.isDebugEnabled()) log.debug("processCommit - send commit to leaders nodes={}", useNodes);
+            params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
 
-            if (useNodes != null && useNodes.size() > 0) {
-              if (log.isDebugEnabled()) log.debug("processCommit - send commit to leaders nodes={}",
-                  useNodes);
-              params.set(DISTRIB_FROM, Replica
-                  .getCoreUrl(zkController.getBaseUrl(),
-                      req.getCore().getName()));
-
-              List<SolrCmdDistributor.Node> finalUseNodes1 = useNodes;
-              worker.collect("distCommit", () -> {
-                cmdDistrib.distribCommit(cmd, finalUseNodes1, params);
-              });
-            }
-            return;
+            List<SolrCmdDistributor.Node> finalUseNodes1 = useNodes;
+            worker.collect("distCommit", () -> {
+              cmdDistrib.distribCommit(cmd, finalUseNodes1, params);
+            });
           }
-
+          return;
         }
       }
     if (log.isDebugEnabled()) log.debug("processCommit(CommitUpdateCommand) - end");
@@ -326,54 +313,76 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
   @Override
   protected void doDistribAdd(AddUpdateCommand cmd) throws IOException {
    // if (log.isDebugEnabled()) log.debug("Distribute add cmd {} to {} {}", cmd, nodes, isLeader);
-    log.info("Distribute add docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
-    if (isLeader && !isSubShardLeader) {
-      DocCollection coll = clusterState.getCollection(collection);
-      List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), cmd.getRootIdUsingRouteParam(), cmd.getSolrInputDocument());
-      // the list<node> will actually have only one element for an add request
-      if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
-        ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
-        params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
-        params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
-        params.set(DISTRIB_FROM_PARENT, cloudDesc.getShardId());
-        cmdDistrib.distribAdd(cmd, subShardLeaders, params, true);
+    log.info("Distribute add docid={} cmd={} to {} leader={} isSubShardLeader={}", cmd.getPrintableId(), cmd, nodes, isLeader, isSubShardLeader);
+
+    if (SkyHookDoc.skyHookDoc != null) {
+      SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "do distrib add isLeader=" + isLeader + " isSubShardLeader=" + isSubShardLeader);
+    }
+    if (isLeader && !isSubShardLeader && !forwardToLeader) {
+
+      DocCollection coll;
+      String routeId;
+      try {
+        coll = clusterState.getCollection(collection);
+        routeId = cmd.getRootIdUsingRouteParam();
+      } catch (Exception e) {
+        log.error("Error getting routeId docId={}", cmd.getPrintableId(), e);
+        throw new SolrException(ErrorCode.SERVER_ERROR, "Error getting routeId docId=" + cmd.getPrintableId(), e);
       }
 
-      final List<SolrCmdDistributor.Node> nodesByRoutingRules = getNodesByRoutingRules(clusterState, coll, cmd.getRootIdUsingRouteParam(), cmd.getSolrInputDocument());
-      if (nodesByRoutingRules != null && !nodesByRoutingRules.isEmpty()) {
-        ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
-        params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
-        params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
-        params.set(DISTRIB_FROM_COLLECTION, collection);
-        params.set(DISTRIB_FROM_SHARD, cloudDesc.getShardId());
+      if (log.isDebugEnabled()) log.debug("going to maybe get sub shard leaders docid={} cmd={} to {} leader={} routeId={}", cmd.getPrintableId(), cmd, nodes, isLeader, routeId);
+      if (routeId != null) {
 
-        try {
-          cmdDistrib.distribAdd(cmd, nodesByRoutingRules, params, true);
-        } catch (IOException e) {
-          log.error("", e);
-          throw new SolrException(ErrorCode.SERVER_ERROR, e);
+        List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), routeId, cmd.getSolrInputDocument(), cmd);
+
+        // the list<node> will actually have only one element for an add request
+        if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
+          ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
+          params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
+          params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
+          params.set(DISTRIB_FROM_PARENT, cloudDesc.getShardId());
+          cmdDistrib.distribAdd(cmd, subShardLeaders, params, true);
         }
-        return;
-      }
 
+        if (log.isDebugEnabled()) log.debug("Distribute add getNodesByRoutingRules docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
+        final List<SolrCmdDistributor.Node> nodesByRoutingRules = getNodesByRoutingRules(clusterState, coll, cmd.getRootIdUsingRouteParam(), cmd.getSolrInputDocument());
+
+        if (log.isDebugEnabled()) log.debug("Distribute add got NodesByRoutingRules docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodesByRoutingRules, isLeader);
+        if (nodesByRoutingRules != null && !nodesByRoutingRules.isEmpty()) {
+          try {
+            if (SkyHookDoc.skyHookDoc != null) {
+              SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "do distrib to replicas with nodesByRoutingRules");
+            }
+            ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
+            params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
+            params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
+            params.set(DISTRIB_FROM_COLLECTION, collection);
+            params.set(DISTRIB_FROM_SHARD, cloudDesc.getShardId());
+
+            cmdDistrib.distribAdd(cmd, nodesByRoutingRules, params, true);
+          } catch (IOException e) {
+            log.error("", e);
+            throw new SolrException(ErrorCode.SERVER_ERROR, e);
+          }
+          return;
+        }
+      }
     } else {
       if (log.isDebugEnabled()) {
-        log.debug("Not a shard or sub shard leader");
+        log.debug("Not a shard or sub shard leader docId={}",  cmd.getPrintableId());
+      }
+      if (!forwardToLeader) {
+        return;
       }
     }
     if (log.isDebugEnabled()) {
       log.debug("Using nodes {}", nodes);
     }
+
+    if (log.isDebugEnabled()) log.debug("Distribute add using nodes if not null and larger than size 0 docid={} cmd={} to {} isLeader={}", cmd.getPrintableId(), cmd, nodes, isLeader);
     if (nodes != null && nodes.size() > 0) {
       ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
-      if (forwardToLeader) {
-        params.set(DISTRIB_UPDATE_PARAM,  DistribPhase.TOLEADER.toString());
-        params.set("h",  "forwardToLeaderForShard" + nodes);
-      } else {
-        params.set(DISTRIB_UPDATE_PARAM, (isLeader || isSubShardLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
-        params.set("h",  "toReplicasForShard" + nodes);
-      }
-
+      params.set(DISTRIB_UPDATE_PARAM, (isLeader || isSubShardLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString()));
       params.set(DISTRIB_FROM, Replica.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
 
       if (req.getParams().get(UpdateRequest.MIN_REPFACT) != null) {
@@ -383,7 +392,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
 
       for (SolrCmdDistributor.Node node : nodes) {
         if (node.getCoreName().equals(desc.getName())) {
-          log.error("IllegalState, trying to send an update to ourself");
+          log.error("docId={} IllegalState, trying to send an update to ourself", cmd.getPrintableId());
           throw new IllegalStateException("IllegalState, trying to send an update to ourself");
         }
       }
@@ -397,7 +406,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
         // and the current in-place update (that depends on the previous update), if reordered
         // in the stream, can result in the current update being bottled up behind the previous
         // update in the stream and can lead to degraded performance.
-
+        if (log.isDebugEnabled()) log.debug("Distribute add inplaceupdate docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
         try {
           cmdDistrib.distribAdd(cmd, nodes, params, true, rollupReplicationTracker, leaderReplicationTracker);
         } catch (IOException e) {
@@ -408,6 +417,10 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
         //        if (!isLeader && params.get(DISTRIB_UPDATE_PARAM).equals(DistribPhase.FROMLEADER.toString())) {
         //          throw new IllegalStateException();
         //        }
+        if (SkyHookDoc.skyHookDoc != null) {
+          SkyHookDoc.skyHookDoc.register(cmd.getPrintableId(), "send update to cmdDistrib nodes=" + nodes + " cmd=" + cmd);
+        }
+        if (log.isDebugEnabled()) log.debug("Distribute add, std old nodes docid={} cmd={} to {} {}", cmd.getPrintableId(), cmd, nodes, isLeader);
         try {
           cmdDistrib.distribAdd(cmd, nodes, params, false, rollupReplicationTracker, leaderReplicationTracker);
         } catch (IOException e) {
@@ -441,7 +454,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
   protected void doDistribDeleteById(DeleteUpdateCommand cmd) throws IOException {
     if (isLeader && !isSubShardLeader)  {
       DocCollection coll = clusterState.getCollection(collection);
-      List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), cmd.getId(), null);
+      List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), cmd.getId(), null, cmd);
       // the list<node> will actually have only one element for an add request
       if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
         ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
@@ -597,7 +610,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
       throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted", e);
     }
     if (leaderLogic) {
-      List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), null, null);
+      List<SolrCmdDistributor.Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), null, null, cmd);
       if (subShardLeaders != null) {
         cmdDistrib.distribDelete(cmd, subShardLeaders, params, true, rollupReplicationTracker, leaderReplicationTracker);
       }
@@ -760,11 +773,11 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
         // that means I want to forward onto my replicas...
         // so get the replicas...
         forwardToLeader = false;
-
+        String leaderCoreName = leaderReplica.getName();
         List<Replica> replicas = clusterState.getCollection(collection)
             .getSlice(shardId)
             .getReplicas(EnumSet.of(Replica.Type.NRT, Replica.Type.TLOG));
-        replicas.removeIf((replica) -> replica.getName().equals(desc.getName()));
+        replicas.removeIf((replica) -> replica.getName().equals(leaderCoreName));
         if (replicas.isEmpty()) {
           return null;
         }
@@ -792,13 +805,13 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
             if (log.isInfoEnabled()) {
               log.info("check url:{} against:{} result:true", replica.getCoreUrl(), skipListSet);
             }
-          } else if(zkShardTerms.registered(coreNodeName) && zkShardTerms.skipSendingUpdatesTo(coreNodeName) && replica.getState().equals(Replica.State.ACTIVE)) {
+          } else if(zkShardTerms.registered(coreNodeName) && zkShardTerms.skipSendingUpdatesTo(coreNodeName)) {
 
             log.info("skip url:{} cause its term is less than leader", replica.getCoreUrl());
 
             skippedCoreNodeNames.add(replica.getName());
-          } else if (!zkController.getZkStateReader().getLiveNodes().contains(replica.getNodeName())) {
-            log.info("skip url:{} cause its not on live node={}", replica.getCoreUrl(), replica.getNodeName());
+          } else if (!zkController.getZkStateReader().getLiveNodes().contains(replica.getNodeName()) || (replica.getState() != Replica.State.ACTIVE &&
+              replica.getState() != Replica.State.BUFFERING)) {
             skippedCoreNodeNames.add(replica.getName());
           } else {
             nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), replica, collection, shardId, maxRetriesToFollowers));
@@ -817,7 +830,10 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
         if (log.isDebugEnabled()) log.debug("Forward update to leader {}", nodes);
 
         if (desc.getName().equals(leaderReplica.getName())) {
-          throw new IllegalStateException("We were asked to forward an update to ourself, which should not happen name=" + desc.getName() + " isLeader=" + isLeader);
+          IllegalStateException e = new IllegalStateException(
+              "We were asked to forward an update to ourself, which should not happen name=" + desc.getName() + " isLeader=" + isLeader);
+          log.error("Sending an update to ourself id={}", id, e);
+          throw e;
         }
         return nodes;
       }
@@ -831,7 +847,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
   @Override
   protected boolean shouldCloneCmdDoc() {
     boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
-    return willDistrib & cloneRequiredOnLeader;
+    return willDistrib;
   }
 
   // helper method, processAdd was getting a bit large.
@@ -982,110 +998,115 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {
   }
 
   /** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
-  protected List<SolrCmdDistributor.Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
-    Collection<Slice> allSlices = coll.getSlices();
-    List<SolrCmdDistributor.Node> nodes = null;
-    for (Slice aslice : allSlices) {
-      final Slice.State state = aslice.getState();
-      if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY)  {
-        DocRouter.Range myRange = coll.getSlice(shardId).getRange();
-        if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
-        boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
-        if (isSubset &&
-            (docId == null // in case of deletes
-                || coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll))) {
-          Replica sliceLeader = aslice.getLeader();
-          // slice leader can be null because node/shard is created zk before leader election
-          if (sliceLeader != null && zkController.getZkStateReader().isNodeLive(sliceLeader.getNodeName()))  {
-            if (nodes == null) nodes = new ArrayList<>();
-            nodes.add(new SolrCmdDistributor.ForwardNode(zkController.getZkStateReader(), sliceLeader, coll.getName(), aslice.getName()));
+  protected List<SolrCmdDistributor.Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc, UpdateCommand cmd) {
+    if (SkyHookDoc.skyHookDoc != null && cmd instanceof AddUpdateCommand) {
+      SkyHookDoc.skyHookDoc.register(((AddUpdateCommand) cmd).getPrintableId(), "getSubShardLeaders isLeader=true");
+    }
+    try {
+      Collection<Slice> allSlices = coll.getSlices();
+      List<SolrCmdDistributor.Node> nodes = null;
+      for (Slice aslice : allSlices) {
+        final Slice.State state = aslice.getState();
+        if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) {
+          DocRouter.Range myRange = coll.getSlice(shardId).getRange();
+          if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
+          boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
+          if (isSubset && (docId == null // in case of deletes
+              || coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll))) {
+            Replica sliceLeader = aslice.getLeader();
+            // slice leader can be null because node/shard is created zk before leader election
+            if (sliceLeader != null && zkController.getZkStateReader().isNodeLive(sliceLeader.getNodeName())) {
+              if (nodes == null) nodes = new ArrayList<>();
+              nodes.add(new SolrCmdDistributor.ForwardNode(zkController.getZkStateReader(), sliceLeader, coll.getName(), aslice.getName()));
+            }
           }
         }
       }
+    } catch (Throwable t) {
+      log.error("Exception getting sub shard leaders", t);
+      if (t instanceof Error) {
+        throw t;
+      }
+      throw new SolrException(ErrorCode.SERVER_ERROR, t);
     }
     return nodes;
   }
 
   /** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#MIGRATE} */
   protected List<SolrCmdDistributor.Node> getNodesByRoutingRules(ClusterState cstate, DocCollection coll, String id, SolrInputDocument doc)  {
-    DocRouter router = coll.getRouter();
-    List<SolrCmdDistributor.Node> nodes = null;
-    if (router instanceof CompositeIdRouter)  {
-      CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router;
-      String myShardId = cloudDesc.getShardId();
-      Slice slice = coll.getSlice(myShardId);
-      Map<String, RoutingRule> routingRules = slice.getRoutingRules();
-      if (routingRules != null) {
-
-        // delete by query case
-        if (id == null) {
-          for (Map.Entry<String, RoutingRule> entry : routingRules.entrySet()) {
-            String targetCollectionName = entry.getValue().getTargetCollectionName();
-            final DocCollection docCollection = cstate.getCollectionOrNull(targetCollectionName, true);
-            if (docCollection != null && docCollection.getActiveSlices().size() > 0) {
-              Collection<Slice> activeSlices = docCollection.getActiveSlices();
-              Slice any = activeSlices.iterator().next();
-              if (nodes == null) nodes = new ArrayList<>();
-              nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), any.getLeader()));
+    try {
+      DocRouter router = coll.getRouter();
+      List<SolrCmdDistributor.Node> nodes = null;
+      if (router instanceof CompositeIdRouter) {
+        CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router;
+        String myShardId = cloudDesc.getShardId();
+        Slice slice = coll.getSlice(myShardId);
+        Map<String,RoutingRule> routingRules = slice.getRoutingRules();
+        if (routingRules != null) {
+
+          // delete by query case
+          if (id == null) {
+            for (Map.Entry<String,RoutingRule> entry : routingRules.entrySet()) {
+              String targetCollectionName = entry.getValue().getTargetCollectionName();
+              final DocCollection docCollection = cstate.getCollectionOrNull(targetCollectionName);
+              if (docCollection != null && docCollection.getActiveSlices().size() > 0) {
+                Collection<Slice> activeSlices = docCollection.getActiveSlices();
+                Slice any = activeSlices.iterator().next();
+                if (nodes == null) nodes = new ArrayList<>();
+                nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), any.getLeader()));
+              }
             }
+            return nodes;
           }
-          return nodes;
-        }
 
-        String routeKey = SolrIndexSplitter.getRouteKey(id);
-        if (routeKey != null) {
-          RoutingRule rule = routingRules.get(routeKey + "!");
-          if (rule != null) {
-            if (! rule.isExpired()) {
-              List<DocRouter.Range> ranges = rule.getRouteRanges();
-              if (ranges != null && !ranges.isEmpty()) {
-                int hash = compositeIdRouter.sliceHash(id, doc, null, coll);
-                for (DocRouter.Range range : ranges) {
-                  if (range.includes(hash)) {
-                    DocCollection targetColl = cstate.getCollection(rule.getTargetCollectionName());
-                    Collection<Slice> activeSlices = targetColl.getRouter().getSearchSlicesSingle(id, null, targetColl);
-                    if (activeSlices == null || activeSlices.isEmpty()) {
-                      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
-                          "No active slices serving " + id + " found for target collection: " + rule.getTargetCollectionName());
+          String routeKey = SolrIndexSplitter.getRouteKey(id);
+          if (routeKey != null) {
+            RoutingRule rule = routingRules.get(routeKey + "!");
+            if (rule != null) {
+              if (!rule.isExpired()) {
+                List<DocRouter.Range> ranges = rule.getRouteRanges();
+                if (ranges != null && !ranges.isEmpty()) {
+                  int hash = compositeIdRouter.sliceHash(id, doc, null, coll);
+                  for (DocRouter.Range range : ranges) {
+                    if (range.includes(hash)) {
+                      DocCollection targetColl = cstate.getCollection(rule.getTargetCollectionName());
+                      Collection<Slice> activeSlices = targetColl.getRouter().getSearchSlicesSingle(id, null, targetColl);
+                      if (activeSlices == null || activeSlices.isEmpty()) {
+                        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No active slices serving " + id + " found for target collection: " + rule.getTargetCollectionName());
+                      }
+                      Replica targetLeader = targetColl.getLeader(activeSlices.iterator().next().getName());
+                      nodes = new ArrayList<>(1);
+                      nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), targetLeader));
+                      break;
                     }
-                    Replica targetLeader = targetColl.getLeader(activeSlices.iterator().next().getName());
-                    nodes = new ArrayList<>(1);
-                    nodes.add(new SolrCmdDistributor.StdNode(zkController.getZkStateReader(), targetLeader));
-                    break;
                   }
                 }
-              }
-            } else  {
-              ReentrantLock ruleExpiryLock = req.getCore().getRuleExpiryLock();
-              if (!ruleExpiryLock.isLocked()) {
+              } else {
+                log.info("Going to expire routing rule");
                 try {
-                  if (ruleExpiryLock.tryLock() || ruleExpiryLock.tryLock(10, TimeUnit.MILLISECONDS)) {
-                    log.info("Going to expire routing rule");
-                    try {
-                      // MRM TODO: TODO: needs to use the statepublisher
-                      Map<String, Object> map = Utils.makeMap(Overseer.QUEUE_OPERATION, OverseerAction.REMOVEROUTINGRULE.toLower(),
-                          ZkStateReader.COLLECTION_PROP, collection,
-                          ZkStateReader.SHARD_ID_PROP, myShardId,
-                          "routeKey", routeKey + "!");
-                      zkController.getOverseer().offerStateUpdate(Utils.toJSON(map));
-                    } catch (KeeperException e) {
-                      log.warn("Exception while removing routing rule for route key: {}", routeKey, e);
-                    } catch (Exception e) {
-                      log.error("Exception while removing routing rule for route key: {}", routeKey, e);
-                    } finally {
-                      ruleExpiryLock.unlock();
-                    }
-                  }
-                } catch (InterruptedException e) {
-                  ParWork.propagateInterrupt(e);
+                  // MRM TODO: TODO: needs to use the statepublisher
+                  Map<String,Object> map = Utils
+                      .makeMap(Overseer.QUEUE_OPERATION, OverseerAction.REMOVEROUTINGRULE.toLower(), ZkStateReader.COLLECTION_PROP, collection,
+                          ZkStateReader.SHARD_ID_PROP, myShardId, "routeKey", routeKey + "!");
+                  // zkController.getOverseer().offerStateUpdate(Utils.toJSON(map));
+                } catch (Exception e) {
+                  log.error("Exception while removing routing rule for route key: {}", routeKey, e);
                 }
               }
             }
           }
         }
       }
+      return nodes;
+    } catch (Throwable t) {
+      log.error("Error getting routing rules", t);
+
+      if (t instanceof Error) {
+        throw t;
+      }
+
+      throw new SolrException(ErrorCode.SERVER_ERROR, "Error getting routing rules", t);
     }
-    return nodes;
   }
 
   private void doDefensiveChecks(DistribPhase phase) {
diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java
index a6bb5d2..d627dce 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java
@@ -115,6 +115,10 @@ public class NestedUpdateProcessorFactory extends UpdateRequestProcessorFactory
     }
 
     private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) {
+      if (sdoc == parent) {
+        throw new IllegalArgumentException("Parent and child must be different instances");
+      }
+
       if(storePath) {
         setPathField(sdoc, fullPath);
       }
diff --git a/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java b/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java
index 5052f83..07754cc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CleanupOldIndexTest.java
@@ -76,7 +76,7 @@ public class CleanupOldIndexTest extends SolrCloudTestCase {
 
     int[] maxDocList;
     if (TEST_NIGHTLY) {
-      maxDocList = new int[] {300, 500, 700};
+      maxDocList = new int[] {100, 200};
     } else {
       maxDocList = new int[] {30, 50, 70};
     }
@@ -89,9 +89,9 @@ public class CleanupOldIndexTest extends SolrCloudTestCase {
     // give some time to index...
     int[] waitTimes;
     if (TEST_NIGHTLY) {
-      waitTimes = new int[] {3000, 4000};
+      waitTimes = new int[] {1000, 3000};
     } else {
-      waitTimes = new int[] {300, 600};
+      waitTimes = new int[] {300};
     }
 
     Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]);
diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java
index fb3dc6e..5e81a76 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DistributedVersionInfoTest.java
@@ -16,19 +16,6 @@
  */
 package org.apache.solr.cloud;
 
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.JSONTestUtil;
@@ -45,10 +32,10 @@ import org.apache.solr.client.solrj.request.QueryRequest;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.CoreAdminResponse;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SkyHookDoc;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ModifiableSolrParams;
@@ -59,6 +46,16 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
 import static org.apache.solr.update.processor.DistributedUpdateProcessor.DISTRIB_FROM;
 import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
 
@@ -84,7 +81,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
     final String shardId = "s1";
 
     CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 3)
-        .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
+        .process(cluster.getSolrClient());
 
     final ZkStateReader stateReader = cluster.getSolrClient().getZkStateReader();
 
@@ -100,7 +97,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
     // verify doc is on the leader and replica
     final List<Replica> notLeaders = stateReader.getClusterState().getCollection(COLLECTION).getReplicas()
         .stream()
-        .filter(r -> r.getName().equals(leader.getName()) == false)
+        .filter(r -> r.getName().equals(leader.getName()) == false && stateReader.isNodeLive(r.getNodeName()) && r.getState() == Replica.State.ACTIVE)
         .collect(Collectors.toList());
     assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, 1, null);
 
@@ -156,15 +153,15 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
         try {
           Thread.sleep(rand.nextInt(30)+1);
         } catch (InterruptedException e) {
-          return;
+          log.error("Fail", e);
         }
 
-        for (int i=0; i < (TEST_NIGHTLY ? 1000 : 100); i++) {
+        for (int i=1; i < (TEST_NIGHTLY ? 1000 : 100); i++) {
           if (i % (rand.nextInt(20)+1) == 0) {
             try {
               Thread.sleep(rand.nextInt(50)+1);
             } catch (InterruptedException e) {
-              return;
+              log.error("Fail", e);
             }
           }
 
@@ -173,7 +170,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
             sendDoc(docId);
             docsSent.incrementAndGet();
           } catch (Exception e) {
-            log.error("id=" + docId, e);
+            log.error("Fail id=" + docId, e);
           }
         }
       }
@@ -188,7 +185,9 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
         for (int i=0; i < 3; i++) {
           try {
             reloadCollection(leader, COLLECTION);
-          } catch (Exception e) {}
+          } catch (Exception e) {
+            log.error("Fail", e);
+          }
 
           try {
             Thread.sleep(rand.nextInt(300)+300);
@@ -216,6 +215,9 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
             if (!deletedDocs.contains(docToDelete)) {
               SolrTestCaseJ4.delI(String.valueOf(docToDelete));
               deletedDocs.add(docToDelete);
+              if (SkyHookDoc.skyHookDoc != null) {
+                SkyHookDoc.skyHookDoc.register(String.valueOf(docToDelete), "delete from test client");
+              }
             }
           }
         }
@@ -231,7 +233,9 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
         for (int i=0; i < 20; i++) {
           try {
             cluster.getSolrClient().commit(COLLECTION);
-          } catch (Exception e) {}
+          } catch (Exception e) {
+            log.error("Fail", e);
+          }
 
           try {
             Thread.sleep(rand.nextInt(100)+100);
@@ -257,7 +261,7 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
       log.info("Total of {} docs deleted", deletedDocs.size());
     }
 
-    assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, TEST_NIGHTLY ? 1000 : 100, deletedDocs);
+    assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, docsSent.get(), deletedDocs);
 
     maxOnLeader = getMaxVersionFromIndex(leader);
     maxOnReplica = getMaxVersionFromIndex(replica);
@@ -342,7 +346,10 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
     doc.addField("id", String.valueOf(docId));
     doc.addField("a_t", "hello" + docId);
     log.info("Send id={}", docId);
-    AbstractFullDistribZkTestBase.sendDocs(cluster.getSolrClient(), COLLECTION, Collections.singletonList(doc));
+    if (SkyHookDoc.skyHookDoc != null) {
+      SkyHookDoc.skyHookDoc.register(String.valueOf(docId), "send from test client");
+    }
+    AbstractFullDistribZkTestBase.sendDocs(cluster.getSolrClient(), COLLECTION, doc);
   }
 
   /**
@@ -371,10 +378,6 @@ public class DistributedVersionInfoTest extends SolrCloudTestCase {
     boolean reloadedOk = false;
     try (Http2SolrClient client = SolrTestCaseJ4.getHttpSolrClient(replica.getBaseUrl())) {
       CoreAdminResponse statusResp = CoreAdminRequest.getStatus(coreName, client);
-      long leaderCoreStartTime = statusResp.getStartTime(coreName).getTime();
-
-      Thread.sleep(1000);
-
       // send reload command for the collection
       log.info("Sending RELOAD command for {}", testCollectionName);
       CollectionAdminRequest.reloadCollection(testCollectionName)
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
index f110724..599aba5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
@@ -132,6 +132,8 @@ public class TestCloudRecovery extends SolrCloudTestCase {
     assertTrue("Timeout waiting for all not live", ClusterStateUtil.waitForAllReplicasNotLive(cloudClient.getZkStateReader(), 45000));
     ChaosMonkey.start(cluster.getJettySolrRunners());
 
+    Thread.sleep(150);
+
     cluster.waitForActiveCollection(COLLECTION, 2, 2 * (nrtReplicas + tlogReplicas));
 
     resp = cloudClient.query(COLLECTION, params);
diff --git a/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java
index 28d3a83..5a0f6c6 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRandomCollapseQParserPlugin.java
@@ -49,7 +49,7 @@ public class TestRandomCollapseQParserPlugin extends SolrTestCaseJ4 {
     = new String[] {NULL_IGNORE, NULL_COLLAPSE, NULL_EXPAND};
   
   @BeforeClass
-  public static void buildIndexAndClient() throws Exception {
+  public static void beforeTestRandomCollapseQParserPlugin() throws Exception {
     initCore("solrconfig-minimal.xml", "schema-sorts.xml");
     
     final int totalDocs = SolrTestUtil.atLeast(TEST_NIGHTLY ? 500 : 50);
@@ -78,7 +78,7 @@ public class TestRandomCollapseQParserPlugin extends SolrTestCaseJ4 {
   }
   
   @AfterClass
-  public static void cleanupStatics() throws Exception {
+  public static void afterTestRandomCollapseQParserPlugin() throws Exception {
     deleteCore();
     SOLR = null;
     ALL_SORT_FIELD_NAMES = ALL_COLLAPSE_FIELD_NAMES = null;
@@ -143,7 +143,7 @@ public class TestRandomCollapseQParserPlugin extends SolrTestCaseJ4 {
         final SolrParams mainP = params("q", q, "fl", "id,"+collapseField);
 
         final String csize = random().nextBoolean() ?
-          "" : " size=" + TestUtil.nextInt(random(),1,TEST_NIGHTLY ? 10000 : 10);
+          "" : " size=" + TestUtil.nextInt(random(),1,TEST_NIGHTLY ? 2500 : 10);
 
         final String nullPolicy = randomNullPolicy();
         final String nullPs = NULL_IGNORE.equals(nullPolicy)
diff --git a/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java b/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
index 85b864a..aba3a6d 100644
--- a/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
+++ b/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java
@@ -296,7 +296,7 @@ public class BJQParserTest extends SolrTestCaseJ4 {
     try (SolrCore core = h.getCore()) {
       Gauge parentFilterCache = null;
       Gauge filterCache = null;
-      TimeOut timeout = new TimeOut(2, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+      TimeOut timeout = new TimeOut(4, TimeUnit.SECONDS, TimeSource.NANO_TIME);
       while (!timeout.hasTimedOut()) {
         parentFilterCache = (Gauge) (core.getCoreMetricManager().getRegistry().getMetrics().get("CACHE.searcher.perSegFilter"));
         filterCache = (Gauge) (core.getCoreMetricManager().getRegistry().getMetrics().get("CACHE.searcher.filterCache"));
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
index c982b21..bf98cb8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
@@ -240,8 +240,7 @@ public class JavaBinUpdateRequestCodec {
     }
 
     private SolrInputDocument listToSolrInputDocument(List<NamedList> namedList) {
-      SolrInputDocument doc = SolrInputDocument.THREAD_LOCAL_SolrInputDocument.get();
-      doc.clear();
+      SolrInputDocument doc = new SolrInputDocument();
       for (int i = 0; i < namedList.size(); i++) {
         NamedList nl = namedList.get(i);
         if (i == 0) {
diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWork.java b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
index def5574..4cdbe06 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ParWork.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ParWork.java
@@ -356,33 +356,29 @@ public class ParWork implements Closeable {
             }
             if (closeCalls.size() > 0) {
 
-                List<Future<Object>> results = new ArrayList<>(closeCalls.size());
+              List<Future<Object>> results = new ArrayList<>(closeCalls.size());
 
-                for (Callable call : closeCalls) {
-                    Future future = executor.submit(call);
-                    results.add(future);
-                }
+              for (Callable call : closeCalls) {
+                Future future = executor.submit(call);
+                results.add(future);
+              }
 
-//                List<Future<Object>> results = executor.invokeAll(closeCalls, 8, TimeUnit.SECONDS);
               int i = 0;
-                for (Future<Object> future : results) {
-                  try {
-                    future.get(
-                        Long.getLong("solr.parwork.task_timeout", TimeUnit.MINUTES.toMillis(10)),
-                        TimeUnit.MILLISECONDS); // MRM TODO:
-                    if (!future.isDone() || future.isCancelled()) {
-                      log.warn("A task did not finish isDone={} isCanceled={}",
-                          future.isDone(), future.isCancelled());
-                      //  throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A task did nor finish" +future.isDone()  + " " + future.isCancelled());
-                    }
-                  } catch (TimeoutException e) {
-                    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Timeout", e); // TODO: add object info eg ParObject.label
-                  } catch (InterruptedException e1) {
-                    log.warn(WORK_WAS_INTERRUPTED);
-                    // TODO: save interrupted status and reset it at end?
+              for (Future<Object> future : results) {
+                try {
+                  future.get(Long.getLong("solr.parwork.task_timeout", TimeUnit.MINUTES.toMillis(10)), TimeUnit.MILLISECONDS); // MRM TODO:
+                  if (!future.isDone() || future.isCancelled()) {
+                    log.warn("A task did not finish isDone={} isCanceled={}", future.isDone(), future.isCancelled());
+                    //  throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "A task did nor finish" +future.isDone()  + " " + future.isCancelled());
                   }
-
+                } catch (TimeoutException e) {
+                  throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Timeout", e); // TODO: add object info eg ParObject.label
+                } catch (InterruptedException e1) {
+                  log.warn(WORK_WAS_INTERRUPTED);
+                  // TODO: save interrupted status and reset it at end?
                 }
+
+              }
             }
           }
         } finally {
diff --git a/solr/solrj/src/java/org/apache/solr/common/SkyHookDoc.java b/solr/solrj/src/java/org/apache/solr/common/SkyHookDoc.java
new file mode 100644
index 0000000..f59ec39
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/SkyHookDoc.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.common;
+
+import org.apache.commons.io.output.StringBuilderWriter;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.PrintWriter;
+import java.lang.invoke.MethodHandles;
+import java.util.Collections;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class SkyHookDoc {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  public static final int STACK = 10;
+  public static SkyHookDoc skyHookDoc = null;//new SkyHookDoc();
+
+  private final Map<String,ConcurrentLinkedDeque> map = Collections.synchronizedSortedMap(new TreeMap<>());
+
+  private final ReentrantLock lock = new ReentrantLock();
+
+  public final static Pattern FIELD_ID = Pattern.compile("\\<id\\:(\\d+)\\>");
+
+  public void register(SolrDocumentBase document) {
+    String sId;
+    try {
+      Object id = document.getFieldValue("id");
+
+      if (id instanceof SolrInputField) {
+        Object val = ((SolrInputField) id).getValue();
+        sId = String.valueOf(val);
+      } else {
+        sId = String.valueOf(id);
+      }
+
+      if (sId.contains("=") || sId.contains(",") || sId.contains(":")) {
+        throw new IllegalArgumentException(sId);
+      }
+
+    } catch (Throwable t) {
+      log.error("SkyHook Exception (BAD!)", t);
+      return;
+    }
+
+    register(sId);
+  }
+
+  private void reg(String sId, String line) {
+
+    if (Integer.parseInt(sId) > 400) {
+      log.info("found high one {}", sId);
+    }
+
+    ConcurrentLinkedDeque<String> deque = map.get(sId);
+
+    if (deque != null) {
+      deque.add(line);
+      return;
+    }
+
+    deque = map.get(sId);
+
+    if (deque != null) {
+      deque.add(line);
+      return;
+    }
+
+    deque = new ConcurrentLinkedDeque();
+    deque.add(line);
+
+    ConcurrentLinkedDeque old = map.put(sId, deque);
+
+    if (old != null) {
+      deque.addAll(old);
+    }
+
+    log.info("SkyHook add Send id={} map={}", sId, map.size());
+  }
+
+  public void register(String sId) {
+    register(sId, "");
+  }
+
+  public void register(String sId, String additional) {
+    try {
+      Matcher m = FIELD_ID.matcher(sId);
+
+      if (m.find()) {
+        sId = m.group(1);
+      }
+
+      StringBuilder sb = ObjectReleaseTracker.getThreadLocalStringBuilder();
+      sb.setLength(0);
+      StringBuilderWriter sw = new StringBuilderWriter(sb);
+      PrintWriter pw = new PrintWriter(sw);
+      printStackTrace(new SolrException(SolrException.ErrorCode.UNKNOWN, ""), pw, STACK);
+      String stack = sw.toString();
+      String line = System.currentTimeMillis() + " registered on node=" + MDCLoggingContext.getNodeName() + (additional != null && additional.length() > 0 ?
+          " " + additional :
+          "") + " originStack=" + stack;
+
+      reg(sId, line);
+    } catch (Throwable t) {
+      log.error("SkyHook Exception (BAD!?) " + additional, t);
+    }
+  }
+
+  public void clear() {
+    map.clear();
+  }
+
+  public void logAll() {
+    try {
+      log.info("SkyHookOutput");
+      synchronized (map) {
+        map.forEach((id, deque) -> {
+
+          log.info("⤞⤞⤞⤞⤞⤞⤠⤠⤠⤠⤠⤠⤠⤗⤗⤗⤗⤗");
+          log.info("⤞⤞⤞⤞⤞⤞⤠⤠⤠⤠⤠⤠⤠⤗⤗⤗⤗⤗ docid={}", id);
+
+          deque.forEach(line -> {
+            log.info("docid={} {}", id, line);
+          });
+
+        });
+      }
+      log.info("SkyHookOutput done");
+    } catch (Throwable t) {
+      log.error("SkyHook exception", t);
+      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, t);
+    }
+  }
+
+  private static void printStackTrace(Throwable t, PrintWriter p, int stack) {
+    // Print our stack trace
+    StackTraceElement[] trace = t.getStackTrace();
+    for (int i = 2; i < Math.min(stack, trace.length); i++) {
+      StackTraceElement traceElement = trace[i];
+      p.println(" " + traceElement);
+    }
+  }
+
+}
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
index d7b268b..c3e06a9 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
@@ -42,6 +42,7 @@ import org.apache.solr.common.util.NamedList;
  */
 public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> implements Iterable<Map.Entry<String, Object>>
 {
+
   protected final Map<String,Object> _fields;
   
   private List<SolrDocument> _childDocuments;
@@ -118,7 +119,22 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
       }
       value = lst;
     }
+
     _fields.put(name, value);
+
+    if (SkyHookDoc.skyHookDoc != null && "id".equals(name)) {
+      if (value instanceof SolrInputField) {
+        SkyHookDoc.skyHookDoc.register(String.valueOf(((SolrInputField) value).getValue()));
+      } else {
+        try {
+          SkyHookDoc.skyHookDoc.register(String.valueOf(value));
+        } catch (NumberFormatException e) {
+          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "class=" + value.getClass().getName(), e);
+        }
+
+      }
+
+    }
   }
   
   /**
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
index 73a2946..3427a75 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
@@ -102,6 +102,9 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
    */
   public void addField(String name, Object value) 
   {
+    if (value == this) {
+      throw new IllegalArgumentException("Parent and child must be different instances");
+    }
     SolrInputField field = _fields.get( name );
     if( field == null || field.value == null ) {
       setField(name, value);
@@ -160,6 +163,10 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
     SolrInputField field = new SolrInputField( name );
     _fields.put( name, field );
     field.setValue( value );
+
+    if (SkyHookDoc.skyHookDoc != null && "id".equals(name)) {
+      SkyHookDoc.skyHookDoc.register(this);
+    }
   }
 
   /**
@@ -205,6 +212,9 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
     if (_childDocuments != null) {
       clone._childDocuments = new ArrayList<>(_childDocuments.size());
       for (SolrInputDocument child : _childDocuments) {
+        if (child == clone) {
+          throw new IllegalArgumentException("Document cannot be child of itself");
+        }
         clone._childDocuments.add(child.deepCopy());
       }
     }
@@ -273,6 +283,9 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
 
   @Override
   public void addChildDocument(SolrInputDocument child) {
+    if (child == this) {
+      throw new IllegalArgumentException("Document cannot be child of itself");
+    }
    if (_childDocuments == null) {
      _childDocuments = new ArrayList<>();
    }
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
index 131a7bd..eb47118 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -1328,6 +1328,10 @@ public class SolrZkClient implements Closeable {
     this.connManager.setDisconnectListener(dl);
   }
 
+  public void addWatch(String basePath, Watcher watcher, AddWatchMode mode) throws KeeperException, InterruptedException {
+    getSolrZooKeeper().addWatch(basePath, watcher == null ? null : wrapWatcher(watcher), mode);
+  }
+
   public void addWatch(String basePath, Watcher watcher, AddWatchMode mode, AsyncCallback.VoidCallback cb, Object ctx) {
     getSolrZooKeeper().addWatch(basePath, watcher == null ? null : wrapWatcher(watcher), mode, cb, ctx);
   }
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 7da7290..087c982 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -761,29 +761,21 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
       gets.incrementAndGet();
       if (!allowCached || lastUpdateTime < 0 || System.nanoTime() - lastUpdateTime > LAZY_CACHE_TIME) {
         boolean shouldFetch = true;
-        DocCollection cached = cachedDocCollection;
-        if (cached != null) {
+        if (cachedDocCollection != null) {
           Stat exists = null;
           try {
             exists = zkClient.exists(getCollectionPath(collName), null, true);
           } catch (Exception e) {
-            ParWork.propagateInterrupt(e);
-            throw new SolrException(ErrorCode.SERVER_ERROR, e);
           }
-          if (exists != null && exists.getVersion() == cached.getZNodeVersion()) {
+          if (exists != null && exists.getVersion() == cachedDocCollection.getZNodeVersion()) {
             shouldFetch = false;
           }
         }
         if (shouldFetch) {
-          cached = getCollectionLive(ZkStateReader.this, collName);
+          cachedDocCollection = getCollectionLive(ZkStateReader.this, collName);
           lastUpdateTime = System.nanoTime();
-          cachedDocCollection = cached;
-          return cached;
         }
       }
-      if (log.isDebugEnabled() && cachedDocCollection == null) {
-        log.debug("cached collection is null");
-      }
       return cachedDocCollection;
     }
 
@@ -1422,11 +1414,13 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
 
     public void createWatch() {
       String collectionCSNPath = getCollectionSCNPath(coll);
+      CountDownLatch latch = new CountDownLatch(2);
       zkClient.addWatch(collectionCSNPath, this, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
         if (rc != 0) {
           KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
           log.error("Exception creating watch for " + path, ex);
         }
+        latch.countDown();
       }, "collectionStateWatcher:" + coll);
 
       zkClient.addWatch(stateUpdateWatcher.stateUpdatesPath, stateUpdateWatcher, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
@@ -1434,10 +1428,18 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
           KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
           log.error("Exception creating watch for " + path, ex);
         }
+        latch.countDown();
       }, "collectionStateUpdatesWatcher:" + coll);
+
+      try {
+        latch.await(5, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        ParWork.propagateInterrupt(e);
+      }
     }
 
     public void removeWatch() {
+      CountDownLatch latch = new CountDownLatch(2);
       String collectionCSNPath = getCollectionSCNPath(coll);
       zkClient.removeWatches(collectionCSNPath, this, WatcherType.Any, true, (rc, path, ctx) -> {
         if (rc != 0) {
@@ -1445,6 +1447,7 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
           if (!(ex instanceof KeeperException.NoWatcherException)) {
             log.error("Exception removing watch for " + path, ex);
           }
+          latch.countDown();
         }
       }, "collectionStateWatcher:" + coll);
 
@@ -1455,7 +1458,14 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
             log.error("Exception removing watch for " + path, ex);
           }
         }
+        latch.countDown();
       }, "collectionStateUpdatesWatcher:" + coll);
+
+      try {
+        latch.await(5, TimeUnit.SECONDS);
+      } catch (InterruptedException e) {
+        ParWork.propagateInterrupt(e);
+      }
     }
 
     public void refreshStateUpdates() {
@@ -1830,23 +1840,21 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
     }
 
     public void createWatch() {
-      zkClient.addWatch(COLLECTIONS_ZKNODE, this, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
-        if (rc != 0) {
-          KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
-          log.error("Exception creating watch for " + path, ex);
-        }
-      }, "collectionsChildWatcher");
+      try {
+        zkClient.addWatch(COLLECTIONS_ZKNODE, this, AddWatchMode.PERSISTENT);
+      } catch (Exception e) {
+        throw new SolrException(ErrorCode.SERVER_ERROR, e);
+      }
     }
 
     public void removeWatch() {
-      zkClient.removeWatches(COLLECTIONS_ZKNODE, this, WatcherType.Any, true, (rc, path, ctx) -> {
-        if (rc != 0) {
-          KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
-          if (!(ex instanceof KeeperException.NoWatcherException)) {
-            log.error("Exception removing watch for " + path, ex);
-          }
-        }
-      }, "collectionsChildWatcher");
+      try {
+        zkClient.removeWatches(COLLECTIONS_ZKNODE, this, WatcherType.Any, true);
+      } catch (KeeperException.NoWatcherException e) {
+
+      } catch (Exception e) {
+        log.warn("Exception removing watch", e);
+      }
     }
 
     @Override
@@ -1866,9 +1874,6 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
       if (EventType.None.equals(event.getType())) {
         return;
       }
-      if (closed || zkClient.isClosed()) {
-        return;
-      }
       if (node != null) {
         MDCLoggingContext.setNode(node);
       }
@@ -1882,33 +1887,29 @@ public class ZkStateReader implements SolrCloseable, Replica.NodeNameToBaseUrl {
     public void refresh() {
       try {
         refreshLiveNodes();
-      } catch (KeeperException e) {
+      } catch (Exception e) {
         log.error("A ZK error has occurred", e);
-      } catch (InterruptedException e) {
-        log.warn("Interrupted", e);
       }
     }
 
     public void createWatch() {
-      zkClient.addWatch(LIVE_NODES_ZKNODE, this, AddWatchMode.PERSISTENT, (rc, path, ctx) -> {
-        if (rc != 0) {
-          KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
-          log.error("Exception creating watch for " + path, ex);
-        }
-      }, "liveNodesWatcher");
+      try {
+        zkClient.addWatch(LIVE_NODES_ZKNODE, this, AddWatchMode.PERSISTENT);
+      } catch (Exception e) {
+        log.warn("Exception creating watch", e);
+        throw new SolrException(ErrorCode.SERVER_ERROR, "Exception creating watch", e);
+      }
     }
 
     public void removeWatch() {
-      zkClient.removeWatches(LIVE_NODES_ZKNODE, this, WatcherType.Any, true, (rc, path, ctx) -> {
-        if (rc != 0) {
-          KeeperException ex = KeeperException.create(KeeperException.Code.get(rc), path);
-          if (!(ex instanceof KeeperException.NoWatcherException)) {
-            log.error("Exception removing watch for " + path, ex);
-          }
-        }
-      }, "liveNodesWatcher");
-    }
+      try {
+        zkClient.removeWatches(LIVE_NODES_ZKNODE, this, WatcherType.Any, true);
+      } catch (KeeperException.NoWatcherException e) {
 
+      } catch (Exception e) {
+        log.warn("Exception removing watch", e);
+      }
+    }
 
     @Override
     public void close() throws IOException {
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java b/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
index 13e7a50..d5c2f90 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ObjectReleaseTracker.java
@@ -20,8 +20,6 @@ import org.apache.commons.io.output.StringBuilderWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.Closeable;
-import java.io.IOException;
 import java.io.PrintWriter;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
@@ -48,13 +46,13 @@ public class ObjectReleaseTracker {
   }
 
   public static boolean track(Object object) {
-    ObjectTrackerException ote = new ObjectTrackerException(object.getClass().getName());
-    OBJECTS.put(object, ote);
+    ObjectTrackerException ote = new ObjectTrackerException(object.getClass().getName() + "@" + Integer.toHexString(object.hashCode()));
+    OBJECTS.put(object.getClass().getName() + "@" + Integer.toHexString(object.hashCode()), ote);
     return true;
   }
   
   public static boolean release(Object object) {
-    OBJECTS.remove(object);
+    OBJECTS.remove(object.getClass().getName() + "@" + Integer.toHexString(object.hashCode()));
     return true;
   }
   
@@ -83,13 +81,6 @@ public class ObjectReleaseTracker {
       for (Entry<Object,ObjectTrackerException> entry : entries.entrySet()) {
         if (object != null && entry.getKey().getClass().getSimpleName().equals(object)) {
           entries.remove(entry.getKey());
-          if (entry.getKey() instanceof Closeable) {
-            try {
-              ((Closeable) entry.getKey()).close();
-            } catch (IOException e) {
-              log.warn("Exception trying to close", e);
-            }
-          }
           continue;
         }
         objects.add(entry.getKey().getClass().getSimpleName());
@@ -119,7 +110,7 @@ public class ObjectReleaseTracker {
     // Print our stack trace
     p.println(t);
     StackTraceElement[] trace = t.getStackTrace();
-    for (int i = 0; i < stack; i++) {
+    for (int i = 0; i < Math.min(stack, trace.length); i++) {
       StackTraceElement traceElement = trace[i];
       p.println("at " + traceElement);
     }
diff --git a/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java b/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java
index d401548..da4689a 100644
--- a/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java
+++ b/solr/solrj/src/java/org/apache/solr/logging/MDCLoggingContext.java
@@ -75,6 +75,10 @@ public class MDCLoggingContext {
     }
   }
 
+  public static String getNodeName() {
+    return MDC.get(NODE_NAME_PROP);
+  }
+
   /**
    * Call this in a
    * finally.
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java
index b640e52..76d2f19 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCase.java
@@ -46,6 +46,7 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.common.AlreadyClosedException;
 import org.apache.solr.common.ParWork;
 import org.apache.solr.common.ParWorkExecutor;
+import org.apache.solr.common.SkyHookDoc;
 import org.apache.solr.common.StringUtils;
 import org.apache.solr.common.TimeTracker;
 import org.apache.solr.common.params.ModifiableSolrParams;
@@ -588,6 +589,11 @@ public class SolrTestCase extends Assert {
       reusedKeys = null;
       sslConfig = null;
 
+      if (SkyHookDoc.skyHookDoc != null) {
+        SkyHookDoc.skyHookDoc.logAll();
+        SkyHookDoc.skyHookDoc.clear();
+      }
+
       long testTime = TimeUnit.SECONDS.convert(System.nanoTime() - testStartTime, TimeUnit.NANOSECONDS);
       if (!LuceneTestCase.TEST_NIGHTLY && testTime > SOLR_TEST_TIMEOUT) {
         log.error("This test suite is too long for non @Nightly runs! Please improve it's performance, break it up, make parts of it @Nightly or make the whole suite @Nightly: {}"
@@ -632,10 +638,9 @@ public class SolrTestCase extends Assert {
       log.info("*******************************************************************");
     }
 
-    StartupLoggingUtils.shutdown();
-
     checkForInterruptRequest();
     interuptThreadWithNameContains.clear();
+    StartupLoggingUtils.shutdown();
   }
 
   private static void checkForInterruptRequest() {
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index a65cacd..03b3a5f 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -995,21 +995,18 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
     }
   }
 
-  protected static int sendDocs(CloudHttp2SolrClient cloudClient, String collection, List<SolrInputDocument> batch) throws Exception {
+  protected static void sendDocs(CloudHttp2SolrClient cloudClient, String collection, SolrInputDocument doc) throws Exception {
     UpdateRequest up = new UpdateRequest();
-    up.add(batch);
+    up.add(doc);
     NamedList resp = null;
 
     try {
       resp = cloudClient.request(up, collection);
-      return cloudClient.getMinAchievedReplicationFactor(cloudClient.getDefaultCollection(), resp);
     } catch (Exception exc) {
       ParWork.propagateInterrupt(exc);
       Throwable rootCause = SolrException.getRootCause(exc);
-
-      log.error(" Add batch failed due to: {}", rootCause);
+      log.error(" Add doc failed due to: {}", rootCause);
       throw exc;
-
     }
   }
 
diff --git a/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml b/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
index d5325e2..b6088b7 100644
--- a/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
+++ b/solr/test-framework/src/resources/logconf/log4j2-std-debug.xml
@@ -24,7 +24,7 @@
         </Console>
 
         <File name="FILE" fileName="${sys:user.home}/solr-test.log" immediateFlush="false" append="false">
-            <PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-5p}{6}} %style{(%t)}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
+            <PatternLayout pattern="%style{%-4r}{yellow} %highlight{%maxLen{%-5p}{6}} %style{(%maxLen{%t}{8})}{yellow,bold} [%style{%X{node_name} %X{core}}{cyan}] %style{%c{1.}}{cyan} %highlight{%m %notEmpty{%ex}}\n"/>
         </File>
 
     </Appenders>
@@ -54,8 +54,8 @@
         <AsyncLogger name="org.apache.solr.client.solrj.impl.LBSolrClient" level="INFO"/>
         <AsyncLogger name="org.apache.solr.cloud.ZkController" level="INFO"/>
         <AsyncLogger name="org.apache.solr.common.cloud.ZkMaintenanceUtils" level="INFO"/>
-        <AsyncLogger name="org.apache.solr.update.processor.DistributedZkUpdateProcessor" level="INFO"/>
-        <AsyncLogger name="org.apache.solr.update.processor.DistributedUpdateProcessor" level="INFO"/>
+        <AsyncLogger name="org.apache.solr.update.processor.DistributedZkUpdateProcessor" level="DEBUG"/>
+        <AsyncLogger name="org.apache.solr.update.processor.DistributedUpdateProcessor" level="DEBUG"/>
         <AsyncLogger name="org.apache.solr.update.SolrCmdDistributor" level="INFO"/>
         <AsyncLogger name="org.apache.solr.update.processor.LogUpdateProcessorFactory" level="INFO"/>