You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2021/04/19 19:29:23 UTC
[accumulo] branch 1451-external-compactions-feature updated: WIP
refactored how tservers are found, not tested
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1451-external-compactions-feature by this push:
new 850044d WIP refactored how tservers are found, not tested
850044d is described below
commit 850044dcaea3fdcf022a78d799d734e06ab91f07
Author: Keith Turner <kt...@apache.org>
AuthorDate: Mon Apr 19 15:27:15 2021 -0400
WIP refactored how tservers are found, not tested
* Support last tserver for queue+prio so we can return diff tservers
* Only remove tserver+prio when tserver reports nothing
* Remove any extra when getting tserver report
---
.../coordinator/CompactionCoordinator.java | 129 ++++-----------
.../accumulo/coordinator/QueueSummaries.java | 183 +++++++++++++++++++++
.../coordinator/CompactionCoordinatorTest.java | 6 +-
3 files changed, 223 insertions(+), 95 deletions(-)
diff --git a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 3d6f749..b8a8655 100644
--- a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++ b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -23,17 +23,14 @@ import static org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly;
import java.net.UnknownHostException;
import java.time.Duration;
import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
-import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import org.apache.accumulo.coordinator.QueueSummaries.PrioTserver;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.clientImpl.ThriftTransportPool;
@@ -96,12 +93,7 @@ public class CompactionCoordinator extends AbstractServer
private static final long FIFTEEN_MINUTES =
TimeUnit.MILLISECONDS.convert(Duration.of(15, TimeUnit.MINUTES.toChronoUnit()));
- /* Map of external queue name -> priority -> tservers */
- protected static final Map<String,TreeMap<Long,LinkedHashSet<TServerInstance>>> QUEUES =
- new ConcurrentHashMap<>();
- /* index of tserver to queue and priority, exists to provide O(1) lookup into QUEUES */
- protected static final Map<TServerInstance,HashSet<QueueAndPriority>> INDEX =
- new ConcurrentHashMap<>();
+ QueueSummaries queueSummaries = new QueueSummaries();
/* Map of compactionId to RunningCompactions */
protected static final Map<ExternalCompactionId,RunningCompaction> RUNNING =
new ConcurrentHashMap<>();
@@ -381,9 +373,7 @@ public class CompactionCoordinator extends AbstractServer
QueueAndPriority.get(summary.getQueue().intern(), summary.getPriority());
synchronized (qp) {
TIME_COMPACTOR_LAST_CHECKED.computeIfAbsent(qp.getQueue(), k -> 0L);
- QUEUES.computeIfAbsent(qp.getQueue(), k -> new TreeMap<>())
- .computeIfAbsent(qp.getPriority(), k -> new LinkedHashSet<>()).add(tsi);
- INDEX.computeIfAbsent(tsi, k -> new HashSet<>()).add(qp);
+ queueSummaries.update(tsi, summaries);
}
});
} finally {
@@ -444,20 +434,7 @@ public class CompactionCoordinator extends AbstractServer
// run() will iterate over the current and added tservers and add them to the internal
// data structures. For tservers that are deleted, we need to remove them from QUEUES
// and INDEX
- deleted.forEach(tsi -> {
- INDEX.get(tsi).forEach(qp -> {
- TreeMap<Long,LinkedHashSet<TServerInstance>> m = QUEUES.get(qp.getQueue());
- if (null != m) {
- LinkedHashSet<TServerInstance> tservers = m.get(qp.getPriority());
- if (null != tservers) {
- synchronized (qp) {
- tservers.remove(tsi);
- }
- }
- }
- });
- INDEX.remove(tsi);
- });
+ queueSummaries.remove(deleted);
}
/**
@@ -483,81 +460,47 @@ public class CompactionCoordinator extends AbstractServer
TIME_COMPACTOR_LAST_CHECKED.put(queue, System.currentTimeMillis());
TExternalCompactionJob result = null;
- final TreeMap<Long,LinkedHashSet<TServerInstance>> m = QUEUES.get(queue);
- if (null != m && !m.isEmpty()) {
- while (result == null) {
-
- // m could become empty if we have contacted all tservers in this queue and
- // there are no compactions
- if (m.isEmpty()) {
- LOG.debug("No tservers found for queue {}, returning empty job to compactor {}", queue,
- compactorAddress);
- result = new TExternalCompactionJob();
- break;
- }
- // Get the first TServerInstance from the highest priority queue
- final Entry<Long,LinkedHashSet<TServerInstance>> entry = m.firstEntry();
- final Long priority = entry.getKey();
- final LinkedHashSet<TServerInstance> tservers = entry.getValue();
- final QueueAndPriority qp = QueueAndPriority.get(queue, priority);
+ PrioTserver prioTserver = queueSummaries.getNextTserver(queueName);
+
+ while (prioTserver != null) {
+ TServerInstance tserver = prioTserver.tserver;
- if (null == tservers || tservers.isEmpty()) {
- // Clean up the map entry when no tservers for this queue and priority
- m.remove(entry.getKey(), entry.getValue());
+ LOG.debug("Getting compaction for queue {} from tserver {}", queue, tserver.getHostAndPort());
+ // Get a compaction from the tserver
+ TabletClientService.Client client = null;
+ try {
+ client = getTabletServerConnection(tserver);
+ TExternalCompactionJob job =
+ client.reserveCompactionJob(TraceUtil.traceInfo(), getContext().rpcCreds(), queue,
+ prioTserver.prio, compactorAddress, externalCompactionId);
+ if (null == job.getExternalCompactionId()) {
+ LOG.debug("No compactions found for queue {} on tserver {}, trying next tserver", queue,
+ tserver.getHostAndPort(), compactorAddress);
+
+ queueSummaries.removeSummary(tserver, queueName, prioTserver.prio);
+ prioTserver = queueSummaries.getNextTserver(queueName);
continue;
- } else {
- synchronized (qp) {
- final TServerInstance tserver = tservers.iterator().next();
- LOG.debug("Found tserver {} with priority {} compaction for queue {}",
- tserver.getHostAndPort(), priority, queue);
- // Remove the tserver from the list, we are going to run a compaction on this server
- tservers.remove(tserver);
- if (tservers.isEmpty()) {
- // Clean up the map entry when no tservers remaining for this queue and priority
- // CBUG This may be redundant as cleanup happens in the 'if' clause above
- m.remove(entry.getKey(), entry.getValue());
- }
- final HashSet<QueueAndPriority> queues = INDEX.get(tserver);
- queues.remove(QueueAndPriority.get(queue, priority));
- if (queues.isEmpty()) {
- // Remove the tserver from the index
- INDEX.remove(tserver);
- }
- LOG.debug("Getting compaction for queue {} from tserver {}", queue,
- tserver.getHostAndPort());
- // Get a compaction from the tserver
- TabletClientService.Client client = null;
- try {
- client = getTabletServerConnection(tserver);
- TExternalCompactionJob job = client.reserveCompactionJob(TraceUtil.traceInfo(),
- getContext().rpcCreds(), queue, priority, compactorAddress, externalCompactionId);
- if (null == job.getExternalCompactionId()) {
- LOG.debug("No compactions found for queue {} on tserver {}, trying next tserver",
- queue, tserver.getHostAndPort(), compactorAddress);
- continue;
- }
- RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()),
- new RunningCompaction(job, compactorAddress, tserver));
- LOG.debug("Returning external job {} to {}", job.externalCompactionId,
- compactorAddress);
- result = job;
- break;
- } catch (TException e) {
- LOG.error(
- "Error from tserver {} while trying to reserve compaction, trying next tserver",
- ExternalCompactionUtil.getHostPortString(tserver.getHostAndPort()), e);
- } finally {
- ThriftUtil.returnClient(client);
- }
- }
}
+ RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()),
+ new RunningCompaction(job, compactorAddress, tserver));
+ LOG.debug("Returning external job {} to {}", job.externalCompactionId, compactorAddress);
+ result = job;
+ break;
+ } catch (TException e) {
+ LOG.error("Error from tserver {} while trying to reserve compaction, trying next tserver",
+ ExternalCompactionUtil.getHostPortString(tserver.getHostAndPort()), e);
+ } finally {
+ ThriftUtil.returnClient(client);
}
- } else {
+ }
+
+ if (result == null) {
LOG.debug("No tservers found for queue {}, returning empty job to compactor {}", queue,
compactorAddress);
result = new TExternalCompactionJob();
}
+
return result;
}
diff --git a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
new file mode 100644
index 0000000..fced9a0
--- /dev/null
+++ b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/QueueSummaries.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.coordinator;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary;
+
+import com.google.common.collect.Sets;
+
+public class QueueSummaries {
+
+ // keep track of the last tserver retunred for qeueue
+ final Map<String, PrioTserver> LAST = new HashMap<>();
+
+ //CBUG may not need concurrent hash map depending on how sync is done
+ /* Map of external queue name -> priority -> tservers */
+ final Map<String,TreeMap<Long,TreeSet<TServerInstance>>> QUEUES =
+ new ConcurrentHashMap<>();
+ /* index of tserver to queue and priority, exists to provide O(1) lookup into QUEUES */
+ final Map<TServerInstance,Set<QueueAndPriority>> INDEX =
+ new ConcurrentHashMap<>();
+
+ private Entry<Long,TreeSet<TServerInstance>> getNextTserverEntry(String queue) {
+ TreeMap<Long,TreeSet<TServerInstance>> m = QUEUES.get(queue);
+ if(m == null) {
+ return null;
+ }
+
+ Iterator<Entry<Long,TreeSet<TServerInstance>>> iter = m.entrySet().iterator();
+
+ while(iter.hasNext()) {
+ Entry<Long,TreeSet<TServerInstance>> next = iter.next();
+ if(next.getValue().isEmpty()) {
+ iter.remove();
+ } else {
+ return next;
+ }
+ }
+
+ QUEUES.remove(queue);
+
+ return null;
+
+ }
+
+
+ static class PrioTserver {
+
+ final TServerInstance tserver;
+ final long prio;
+
+ public PrioTserver(TServerInstance t, long p) {
+ this.tserver = t;
+ this.prio = p;
+ }
+ }
+
+ //CBUG may want to to do finer grained sync...
+ synchronized PrioTserver getNextTserver(String queue) {
+
+ Entry<Long,TreeSet<TServerInstance>> entry = getNextTserverEntry(queue);
+
+ if(entry == null) {
+ // no tserver, so remove any last entry if it exists
+ LAST.remove(queue);
+ return null;
+ }
+
+ final Long priority = entry.getKey();
+ final TreeSet<TServerInstance> tservers = entry.getValue();
+
+ PrioTserver last = LAST.get(queue);
+
+ TServerInstance nextTserver = null;
+
+ if(last != null && last.prio == priority) {
+ TServerInstance higher = tservers.higher(last.tserver);
+ if(higher == null) {
+ nextTserver = tservers.first();
+ } else {
+ nextTserver = higher;
+ }
+ } else {
+ nextTserver = tservers.first();
+ }
+
+ PrioTserver result = new PrioTserver(nextTserver, priority);
+
+ LAST.put(queue, result);
+
+ return result;
+ }
+
+
+ synchronized void update(TServerInstance tsi, List<TCompactionQueueSummary> summaries) {
+
+ Set<QueueAndPriority> newQP = new HashSet<>();
+ summaries.forEach(summary -> {
+ QueueAndPriority qp =
+ QueueAndPriority.get(summary.getQueue().intern(), summary.getPriority());
+ newQP.add(qp);
+ });
+
+ Set<QueueAndPriority> currentQP = INDEX.getOrDefault(tsi, Set.of());
+
+ // remove anything the tserver did not report
+ for(QueueAndPriority qp : Sets.difference(currentQP, newQP)) {
+ removeSummary(tsi, qp.getQueue(), qp.getPriority());
+ }
+
+ INDEX.put(tsi, newQP);
+
+ newQP.forEach(qp -> {
+ QUEUES.computeIfAbsent(qp.getQueue(), k -> new TreeMap<>())
+ .computeIfAbsent(qp.getPriority(), k -> new TreeSet<>()).add(tsi);
+ });
+ }
+
+ synchronized void removeSummary(TServerInstance tsi, String queue, long priority) {
+ TreeMap<Long,TreeSet<TServerInstance>> m = QUEUES.get(queue);
+ if(m != null) {
+ TreeSet<TServerInstance> s = m.get(priority);
+ if(s != null) {
+ if(s.remove(tsi) && s.isEmpty()) {
+ m.remove(priority);
+ }
+ }
+
+ if(m.isEmpty()) {
+ QUEUES.remove(queue);
+ }
+ }
+
+ Set<QueueAndPriority> qaps = INDEX.get(tsi);
+ if(qaps != null) {
+ if(qaps.remove(QueueAndPriority.get(queue, priority)) && qaps.isEmpty()) {
+ INDEX.remove(tsi);
+ }
+ }
+ }
+
+ synchronized void remove(Set<TServerInstance> deleted) {
+ deleted.forEach(tsi -> {
+ INDEX.get(tsi).forEach(qp -> {
+ TreeMap<Long,TreeSet<TServerInstance>> m = QUEUES.get(qp.getQueue());
+ if (null != m) {
+ TreeSet<TServerInstance> tservers = m.get(qp.getPriority());
+ if (null != tservers) {
+ tservers.remove(tsi);
+ }
+ }
+ });
+ INDEX.remove(tsi);
+ });
+ }
+}
diff --git a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
index a1522b8..6436297 100644
--- a/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
+++ b/server/compaction-coordinator/src/test/java/org/apache/accumulo/coordinator/CompactionCoordinatorTest.java
@@ -152,11 +152,13 @@ public class CompactionCoordinatorTest {
TKeyExtent extent) throws TException {}
public Map<String,TreeMap<Long,LinkedHashSet<TServerInstance>>> getQueues() {
- return QUEUES;
+ // CBUG todo refactor test
+ return null;
}
public Map<TServerInstance,HashSet<QueueAndPriority>> getIndex() {
- return INDEX;
+ // CBUG todo refactor test
+ return null;
}
public Map<ExternalCompactionId,RunningCompaction> getRunning() {