You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/10/13 16:50:55 UTC

[lucene-solr] branch reference_impl_dev updated: @985 Update how we track collection zk versions in Overseer.

This is an automated email from the ASF dual-hosted git repository.

markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/reference_impl_dev by this push:
     new 1cdd19d  @985 Update how we track collection zk versions in Overseer.
1cdd19d is described below

commit 1cdd19d0bfc181f74edcd90fbd63b8aeded7aa26
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Tue Oct 13 11:49:03 2020 -0500

    @985 Update how we track collection zk versions in Overseer.
---
 .../apache/solr/cloud/overseer/ZkStateWriter.java  | 70 ++++++++++------------
 .../solr/cloud/MissingSegmentRecoveryTest.java     |  2 -
 .../apache/solr/common/cloud/ZkStateReader.java    |  6 +-
 .../solr/common/util/SolrQueuedThreadPool.java     | 29 +++++----
 4 files changed, 50 insertions(+), 57 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
index 32a1ead..4aecdc9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ZkStateWriter.java
@@ -17,6 +17,7 @@
 package org.apache.solr.cloud.overseer;
 
 import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -63,6 +64,7 @@ public class ZkStateWriter {
   // / protected boolean isClusterStateModified = false;
   protected long lastUpdatedTime = 0;
 
+  private Map<String,Integer> trackVersions = new HashMap<>();
 
   private final ZkStateReader reader;
 
@@ -126,11 +128,11 @@ public class ZkStateWriter {
           }
 
           updatesToWrite.remove(name);
+          trackVersions.remove(name);
 
           reader.getZkClient().clean(path);
 
           TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
-          DocCollection coll;
           timeout.waitFor("", () -> {
             DocCollection rc = reader.getClusterState().getCollectionOrNull(name);
             if (rc == null) return true;
@@ -140,17 +142,11 @@ public class ZkStateWriter {
           LinkedHashMap<String,ClusterState.CollectionRef> collStates = new LinkedHashMap<>(prevState.getCollectionStates());
           collStates.remove(name);
           prevState = new ClusterState(prevState.getLiveNodes(), collStates, prevState.getZNodeVersion());
-        } else if (updatesToWrite.get(name) != null || prevState.getCollectionOrNull(name) != null) {
+        } else if (updatesToWrite.get(name) != null) {
           if (log.isDebugEnabled()) {
             log.debug("enqueueUpdate() - going to update_collection {} version: {}", path, c.getZNodeVersion());
           }
 
-          // assert c.getStateFormat() > 1;
-          // stat = reader.getZkClient().getCurator().checkExists().forPath(path);
-          DocCollection coll = updatesToWrite.get(name);
-          if (coll == null) {
-            coll = prevState.getCollectionOrNull(name);
-          }
 
           if (log.isDebugEnabled()) {
             log.debug("The new collection {}", c);
@@ -230,6 +226,7 @@ public class ZkStateWriter {
       //
       //
       Map<String,DocCollection> failedUpdates = new LinkedHashMap<>();
+      Exception lastFailedException = null;
       for (DocCollection c : updatesToWrite.values()) {
         String name = c.getName();
         String path = ZkStateReader.getCollectionPath(c.getName());
@@ -247,21 +244,24 @@ public class ZkStateWriter {
             //}
             // stat = reader.getZkClient().getCurator().setData().withVersion(prevVersion).forPath(path, data);
             try {
-              stat = reader.getZkClient().setData(path, data, c.getZNodeVersion(), false);
-            } catch (KeeperException.BadVersionException bve) {
-
-              if (c.getZNodeVersion() == 1 && stat.getVersion() == 0) {
-                // need to figure out how this case happens
-                stat = reader.getZkClient().setData(path, data, 0, false);
+              int version = c.getZNodeVersion();
+              Integer v = trackVersions.get(c.getName());
+              if (v != null && version == 0) {
+                version = v;
+                trackVersions.put(c.getName(), v + 1);
               } else {
-                // this is a tragic error, we must disallow usage of this instance
-                log.warn(
-                    "Tried to update the cluster state using version={} but we where rejected, found {}",
-                    c.getZNodeVersion(), stat.getVersion(), bve);
-                throw bve;
+                trackVersions.put(c.getName(), version + 1);
               }
-            }
 
+              reader.getZkClient().setData(path, data, version, true);
+            } catch (KeeperException.BadVersionException bve) {
+              lastFailedException = bve;
+              failedUpdates.put(c.getName(), c);
+              stat = reader.getZkClient().exists(path, null);
+              // this is a tragic error, we must disallow usage of this instance
+              log.warn("Tried to update the cluster state using version={} but we where rejected, found {}", c.getZNodeVersion(), stat.getVersion(), bve);
+            }
+            if (log.isDebugEnabled()) log.debug("Set version for local collection {} to {}", c.getName(), c.getZNodeVersion() + 1);
         } catch (InterruptedException | AlreadyClosedException e) {
           ParWork.propagateInterrupt(e);
           throw e;
@@ -269,16 +269,6 @@ public class ZkStateWriter {
           throw e;
         } catch (Exception e) {
           ParWork.propagateInterrupt(e);
-//          if (e instanceof KeeperException.BadVersionException) {
-//            // nocommit invalidState = true;
-//            //if (log.isDebugEnabled())
-//            log.info(
-//                "Tried to update the cluster state using version={} but we where rejected, currently at {}",
-//                prevVersion, c == null ? "null" : c.getZNodeVersion(), e);
-//            prevState = reader.getClusterState();
-//            continue;
-//          }
-          ParWork.propagateInterrupt(e);
           throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
               "Failed processing update=" + c, e);
         }
@@ -290,7 +280,10 @@ public class ZkStateWriter {
           if (c != null && !failedUpdates.containsKey(c.getName())) {
             try {
               //System.out.println("waiting to see state " + prevVersion);
-              Integer finalPrevVersion = c.getZNodeVersion();
+              Integer v = trackVersions.get(c.getName());
+              int version = v - 1;
+              
+              Integer finalPrevVersion = version;
               reader.waitForState(c.getName(), 15, TimeUnit.SECONDS, (l, col) -> {
 
                 //              if (col != null) {
@@ -313,11 +306,11 @@ public class ZkStateWriter {
 
       lastUpdatedTime = System.nanoTime();
 
-      updatesToWrite.clear();
-      if (log.isDebugEnabled()) log.debug("Failed updates {}", failedUpdates.values());
-   //   updatesToWrite.putAll(failedUpdates);
+
+
       if (failedUpdates.size() > 0) {
-        throw new AlreadyClosedException();
+        failedUpdates.clear();
+        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, lastFailedException);
       }
 
       success = true;
@@ -334,8 +327,11 @@ public class ZkStateWriter {
 //      log.debug("writePendingUpdates() - end - New Cluster State is: {}", newClusterState);
 //    }
 
-
-    return state;
+    try {
+      return new ClusterState(state.getLiveNodes(), updatesToWrite);
+    } finally {
+      updatesToWrite.clear();
+    }
   }
 
   public Map<String,DocCollection>  getUpdatesToWrite() {
diff --git a/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java b/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java
index 7b0361d..30de73a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MissingSegmentRecoveryTest.java
@@ -113,8 +113,6 @@ public class MissingSegmentRecoveryTest extends SolrCloudTestCase {
     JettySolrRunner jetty = cluster.getReplicaJetty(replica);
     jetty.stop();
     jetty.start();
-
-    cluster.waitForActiveCollection(collection, 1, 2);
     
     QueryResponse resp = cluster.getSolrClient().query(collection, new SolrQuery("*:*"));
     assertEquals(10, resp.getResults().getNumFound());
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 08b2590..6dc54c5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -852,16 +852,16 @@ public class ZkStateReader implements SolrCloseable {
       notifications.shutdown();
     }
 
+    waitLatches.forEach(c -> { for (int i = 0; i < c.getCount(); i++) c.countDown(); });
+
     try {
       if (closeClient) {
         IOUtils.closeQuietly(zkClient);
       }
 
-      //waitLatches.forEach(c -> { for (int i = 0; i < c.getCount(); i++) c.countDown(); });
-
       if (notifications != null) {
         try {
-          boolean success = notifications.awaitTermination(3, TimeUnit.SECONDS);
+          boolean success = notifications.awaitTermination(1, TimeUnit.SECONDS);
           if (!success) notifications.shutdownNow();
         } catch (InterruptedException e) {
           ParWork.propagateInterrupt(e);
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java b/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java
index f15efc3..b903de0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/SolrQueuedThreadPool.java
@@ -904,7 +904,6 @@ public class SolrQueuedThreadPool extends ContainerLifeCycle implements ThreadFa
 
 
     public void close() {
-        this.closed = true;
         removeBean(_tryExecutor);
         _tryExecutor = TryExecutor.NO_TRY;
 
@@ -913,18 +912,25 @@ public class SolrQueuedThreadPool extends ContainerLifeCycle implements ThreadFa
 
         //setIdleTimeout(1);
 
+        try {
+            super.doStop();
+        } catch (Exception e) {
+            LOG.warn("super.doStop", e);
+            return;
+        }
+
         // Fill the job queue with noop jobs to wakeup idle threads.
         for (int i = 0; i < threads; ++i) {
             jobs.offer(NOOP);
         }
 
         // interrupt threads
-        for (Future thread : _threadFutures)
-        {
-            if (LOG.isDebugEnabled())
-                LOG.debug("Interrupting {}", thread);
-            thread.cancel(true);
-        }
+//        for (Future thread : _threadFutures)
+//        {
+//            if (LOG.isDebugEnabled())
+//                LOG.debug("Interrupting {}", thread);
+//            thread.cancel(true);
+//        }
 
 
         // Close any un-executed jobs
@@ -947,14 +953,7 @@ public class SolrQueuedThreadPool extends ContainerLifeCycle implements ThreadFa
 //        }
 
 
-        try {
-            super.doStop();
-        } catch (Exception e) {
-            LOG.warn("super.doStop", e);
-            return;
-        }
-
-
+        this.closed = true;
         try {
             joinThreads(15000);
         } catch (InterruptedException e) {