You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2021/04/19 19:29:23 UTC

[accumulo] branch 1451-external-compactions-feature updated: WIP refactored how tservers are found, not tested

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1451-external-compactions-feature by this push:
     new 850044d  WIP refactored how tservers are found, not tested
850044d is described below

commit 850044dcaea3fdcf022a78d799d734e06ab91f07
Author: Keith Turner <kt...@apache.org>
AuthorDate: Mon Apr 19 15:27:15 2021 -0400

    WIP refactored how tservers are found, not tested
    
      * Support last tserver for queue+prio so we can return diff tservers
      * Only remove tserver+prio when tserver reports nothing
      * Remove any extra when getting tserver report
---
 .../coordinator/CompactionCoordinator.java         | 129 ++++-----------
 .../accumulo/coordinator/QueueSummaries.java       | 183 +++++++++++++++++++++
 .../coordinator/CompactionCoordinatorTest.java     |   6 +-
 3 files changed, 223 insertions(+), 95 deletions(-)

diff --git a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 3d6f749..b8a8655 100644
--- a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++ b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -23,17 +23,14 @@ import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
 import java.net.UnknownHostException;
 import java.time.Duration;
 import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
-import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.accumulo.coordinator.QueueSummaries.PrioTserver;
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.clientImpl.ThriftTransportPool;
@@ -96,12 +93,7 @@ public class CompactionCoordinator extends AbstractServer
   private static final long FIFTEEN_MINUTES =
       TimeUnit.MILLISECONDS.convert(Duration.of(15, TimeUnit.MINUTES.toChronoUnit()));
 
-  /* Map of external queue name -> priority -> tservers */
-  protected static final Map<String,TreeMap<Long,LinkedHashSet<TServerInstance>>> QUEUES =
-      new ConcurrentHashMap<>();
-  /* index of tserver to queue and priority, exists to provide O(1) lookup into QUEUES */
-  protected static final Map<TServerInstance,HashSet<QueueAndPriority>> INDEX =
-      new ConcurrentHashMap<>();
+  QueueSummaries queueSummaries = new QueueSummaries();
   /* Map of compactionId to RunningCompactions */
   protected static final Map<ExternalCompactionId,RunningCompaction> RUNNING =
       new ConcurrentHashMap<>();
@@ -381,9 +373,7 @@ public class CompactionCoordinator extends AbstractServer
                   QueueAndPriority.get(summary.getQueue().intern(), summary.getPriority());
               synchronized (qp) {
                 TIME_COMPACTOR_LAST_CHECKED.computeIfAbsent(qp.getQueue(), k -> 0L);
-                QUEUES.computeIfAbsent(qp.getQueue(), k -> new TreeMap<>())
-                    .computeIfAbsent(qp.getPriority(), k -> new LinkedHashSet<>()).add(tsi);
-                INDEX.computeIfAbsent(tsi, k -> new HashSet<>()).add(qp);
+                queueSummaries.update(tsi, summaries);
               }
             });
           } finally {
@@ -444,20 +434,7 @@ public class CompactionCoordinator extends AbstractServer
     // run() will iterate over the current and added tservers and add them to the internal
     // data structures. For tservers that are deleted, we need to remove them from QUEUES
     // and INDEX
-    deleted.forEach(tsi -> {
-      INDEX.get(tsi).forEach(qp -> {
-        TreeMap<Long,LinkedHashSet<TServerInstance>> m = QUEUES.get(qp.getQueue());
-        if (null != m) {
-          LinkedHashSet<TServerInstance> tservers = m.get(qp.getPriority());
-          if (null != tservers) {
-            synchronized (qp) {
-              tservers.remove(tsi);
-            }
-          }
-        }
-      });
-      INDEX.remove(tsi);
-    });
+    queueSummaries.remove(deleted);
   }
 
   /**
@@ -483,81 +460,47 @@ public class CompactionCoordinator extends AbstractServer
     TIME_COMPACTOR_LAST_CHECKED.put(queue, System.currentTimeMillis());
 
     TExternalCompactionJob result = null;
-    final TreeMap<Long,LinkedHashSet<TServerInstance>> m = QUEUES.get(queue);
-    if (null != m && !m.isEmpty()) {
-      while (result == null) {
-
-        // m could become empty if we have contacted all tservers in this queue and
-        // there are no compactions
-        if (m.isEmpty()) {
-          LOG.debug("No tservers found for queue {}, returning empty job to compactor {}", queue,
-              compactorAddress);
-          result = new TExternalCompactionJob();
-          break;
-        }
 
-        // Get the first TServerInstance from the highest priority queue
-        final Entry<Long,LinkedHashSet<TServerInstance>> entry = m.firstEntry();
-        final Long priority = entry.getKey();
-        final LinkedHashSet<TServerInstance> tservers = entry.getValue();
-        final QueueAndPriority qp = QueueAndPriority.get(queue, priority);
+    PrioTserver prioTserver = queueSummaries.getNextTserver(queueName);
+
+    while (prioTserver != null) {
+      TServerInstance tserver = prioTserver.tserver;
 
-        if (null == tservers || tservers.isEmpty()) {
-          // Clean up the map entry when no tservers for this queue and priority
-          m.remove(entry.getKey(), entry.getValue());
+      LOG.debug("Getting compaction for queue {} from tserver {}", queue, tserver.getHostAndPort());
+      // Get a compaction from the tserver
+      TabletClientService.Client client = null;
+      try {
+        client = getTabletServerConnection(tserver);
+        TExternalCompactionJob job =
+            client.reserveCompactionJob(TraceUtil.traceInfo(), getContext().rpcCreds(), queue,
+                prioTserver.prio, compactorAddress, externalCompactionId);
+        if (null == job.getExternalCompactionId()) {
+          LOG.debug("No compactions found for queue {} on tserver {}, trying next tserver", queue,
+              tserver.getHostAndPort(), compactorAddress);
+
+          queueSummaries.removeSummary(tserver, queueName, prioTserver.prio);
+          prioTserver = queueSummaries.getNextTserver(queueName);
           continue;
-        } else {
-          synchronized (qp) {
-            final TServerInstance tserver = tservers.iterator().next();
-            LOG.debug("Found tserver {} with priority {} compaction for queue {}",
-                tserver.getHostAndPort(), priority, queue);
-            // Remove the tserver from the list, we are going to run a compaction on this server
-            tservers.remove(tserver);
-            if (tservers.isEmpty()) {
-              // Clean up the map entry when no tservers remaining for this queue and priority
-              // CBUG This may be redundant as cleanup happens in the 'if' clause above
-              m.remove(entry.getKey(), entry.getValue());
-            }
-            final HashSet<QueueAndPriority> queues = INDEX.get(tserver);
-            queues.remove(QueueAndPriority.get(queue, priority));
-            if (queues.isEmpty()) {
-              // Remove the tserver from the index
-              INDEX.remove(tserver);
-            }
-            LOG.debug("Getting compaction for queue {} from tserver {}", queue,
-                tserver.getHostAndPort());
-            // Get a compaction from the tserver
-            TabletClientService.Client client = null;
-            try {
-              client = getTabletServerConnection(tserver);
-              TExternalCompactionJob job = client.reserveCompactionJob(TraceUtil.traceInfo(),
-                  getContext().rpcCreds(), queue, priority, compactorAddress, externalCompactionId);
-              if (null == job.getExternalCompactionId()) {
-                LOG.debug("No compactions found for queue {} on tserver {}, trying next tserver",
-                    queue, tserver.getHostAndPort(), compactorAddress);
-                continue;
-              }
-              RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()),
-                  new RunningCompaction(job, compactorAddress, tserver));
-              LOG.debug("Returning external job {} to {}", job.externalCompactionId,
-                  compactorAddress);
-              result = job;
-              break;
-            } catch (TException e) {
-              LOG.error(
-                  "Error from tserver {} while trying to reserve compaction, trying next tserver",
-                  ExternalCompactionUtil.getHostPortString(tserver.getHostAndPort()), e);
-            } finally {
-              ThriftUtil.returnClient(client);
-            }
-          }
         }
+        RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()),
+            new RunningCompaction(job, compactorAddress, tserver));
+        LOG.debug("Returning external job {} to {}", job.externalCompactionId, compactorAddress);
+        result = job;
+        break;
+      } catch (TException e) {
+        LOG.error("Error from tserver {} while trying to reserve compaction, trying next tserver",
+            ExternalCompactionUtil.getHostPortString(tserver.getHostAndPort()), e);
+      } finally {
+        ThriftUtil.returnClient(client);
       }
-    } else {
+    }
+
+    if (result == null) {
       LOG.debug("No tservers found for queue {}, returning empty job to compactor {}", queue,
           compactorAddress);
       result = new TExternalCompactionJob();
     }
+
     return result;
 
   }
diff --git a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
new file mode 100644
index 0000000..fced9a0
--- /dev/null
+++ b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.coordinator;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary;
+
+import com.google.common.collect.Sets;
+
+public class QueueSummaries {
+
+  // keep track of the last tserver retunred for qeueue
+ final Map<String, PrioTserver> LAST = new HashMap<>();
+
+ //CBUG may not need concurrent hash map depending on how sync is done
+  /* Map of external queue name -> priority -> tservers */
+  final Map<String,TreeMap<Long,TreeSet<TServerInstance>>> QUEUES =
+      new ConcurrentHashMap<>();
+  /* index of tserver to queue and priority, exists to provide O(1) lookup into QUEUES */
+  final Map<TServerInstance,Set<QueueAndPriority>> INDEX =
+      new ConcurrentHashMap<>();
+
+  private Entry<Long,TreeSet<TServerInstance>> getNextTserverEntry(String queue) {
+    TreeMap<Long,TreeSet<TServerInstance>> m = QUEUES.get(queue);
+    if(m == null) {
+      return null;
+    }
+
+    Iterator<Entry<Long,TreeSet<TServerInstance>>> iter = m.entrySet().iterator();
+
+    while(iter.hasNext()) {
+      Entry<Long,TreeSet<TServerInstance>> next = iter.next();
+      if(next.getValue().isEmpty()) {
+        iter.remove();
+      } else {
+        return next;
+      }
+    }
+
+    QUEUES.remove(queue);
+
+    return null;
+
+  }
+
+
+  static class PrioTserver {
+
+    final TServerInstance tserver;
+    final long prio;
+
+    public PrioTserver(TServerInstance t, long p) {
+      this.tserver = t;
+      this.prio = p;
+    }
+  }
+
+  //CBUG may want to to do finer grained sync...
+  synchronized PrioTserver getNextTserver(String queue) {
+
+    Entry<Long,TreeSet<TServerInstance>> entry = getNextTserverEntry(queue);
+
+    if(entry == null) {
+      // no tserver, so remove any last entry if it exists
+      LAST.remove(queue);
+      return null;
+    }
+
+    final Long priority = entry.getKey();
+    final TreeSet<TServerInstance> tservers = entry.getValue();
+
+    PrioTserver last = LAST.get(queue);
+
+    TServerInstance nextTserver = null;
+
+    if(last != null && last.prio == priority) {
+      TServerInstance higher = tservers.higher(last.tserver);
+      if(higher == null) {
+        nextTserver = tservers.first();
+      } else {
+        nextTserver = higher;
+      }
+    } else {
+      nextTserver = tservers.first();
+    }
+
+    PrioTserver result = new PrioTserver(nextTserver, priority);
+
+    LAST.put(queue, result);
+
+    return result;
+  }
+
+
+  synchronized void update(TServerInstance tsi, List<TCompactionQueueSummary> summaries) {
+
+    Set<QueueAndPriority> newQP = new HashSet<>();
+    summaries.forEach(summary -> {
+      QueueAndPriority qp =
+          QueueAndPriority.get(summary.getQueue().intern(), summary.getPriority());
+      newQP.add(qp);
+    });
+
+    Set<QueueAndPriority> currentQP = INDEX.getOrDefault(tsi, Set.of());
+
+    // remove anything the tserver did not report
+    for(QueueAndPriority qp : Sets.difference(currentQP, newQP)) {
+      removeSummary(tsi, qp.getQueue(), qp.getPriority());
+    }
+
+    INDEX.put(tsi, newQP);
+
+    newQP.forEach(qp -> {
+      QUEUES.computeIfAbsent(qp.getQueue(), k -> new TreeMap<>())
+      .computeIfAbsent(qp.getPriority(), k -> new TreeSet<>()).add(tsi);
+    });
+  }
+
+  synchronized void removeSummary(TServerInstance tsi, String queue, long priority) {
+    TreeMap<Long,TreeSet<TServerInstance>> m = QUEUES.get(queue);
+    if(m != null) {
+      TreeSet<TServerInstance> s = m.get(priority);
+      if(s != null) {
+        if(s.remove(tsi) && s.isEmpty()) {
+          m.remove(priority);
+        }
+      }
+
+      if(m.isEmpty()) {
+        QUEUES.remove(queue);
+      }
+    }
+
+    Set<QueueAndPriority> qaps = INDEX.get(tsi);
+    if(qaps != null) {
+      if(qaps.remove(QueueAndPriority.get(queue, priority)) && qaps.isEmpty()) {
+        INDEX.remove(tsi);
+      }
+    }
+  }
+
+  synchronized void remove(Set<TServerInstance> deleted) {
+    deleted.forEach(tsi -> {
+      INDEX.get(tsi).forEach(qp -> {
+        TreeMap<Long,TreeSet<TServerInstance>> m = QUEUES.get(qp.getQueue());
+        if (null != m) {
+          TreeSet<TServerInstance> tservers = m.get(qp.getPriority());
+          if (null != tservers) {
+              tservers.remove(tsi);
+          }
+        }
+      });
+      INDEX.remove(tsi);
+    });
+  }
+}
diff --git a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
index a1522b8..6436297 100644
--- a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
+++ b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
@@ -152,11 +152,13 @@ public class CompactionCoordinatorTest {
         TKeyExtent extent) throws TException {}
 
     public Map<String,TreeMap<Long,LinkedHashSet<TServerInstance>>> getQueues() {
-      return QUEUES;
+      // CBUG todo refactor test
+      return null;
     }
 
     public Map<TServerInstance,HashSet<QueueAndPriority>> getIndex() {
-      return INDEX;
+      // CBUG todo refactor test
+      return null;
     }
 
     public Map<ExternalCompactionId,RunningCompaction> getRunning() {