You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2024/04/26 19:33:04 UTC

(accumulo) branch elasticity updated (ee04ad3aa5 -> 07bf1bc6fd)

This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a change to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git


    from ee04ad3aa5 Fixed check in TabletResourceGroupBalanceIT to use Wait.waitFor (#4479)
     add 6c1e453f53 Merge branch '2.1'
     add 8521cb3178 improves efficiency of migration set clean up (#4474)
     add 4839d3bf22 logs when closing tablets are waiting on compactions (#4472)
     add 714fdb7e1a logs when tablet decides there is nothing to do for user compaction (#4473)
     add 3c5326c89f Add visibility to grep searches (#4468)
     add 0958a5a456 Add scan server prefix to valid zooProp list (#4487)
     add 6d0b476852 Additional metrics changes (#4459)
     add 94b8d96e5c Merge branch '2.1'
     add 65dd34fad4 Fix PausedCompactionMetrics caused by merge issue (#4491)
     new 07bf1bc6fd Merge branch 'main' into elasticity

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 core/pom.xml                                       |   1 +
 .../org/apache/accumulo/core/conf/Property.java    |  33 +-
 .../accumulo/core/iterators/user/GrepIterator.java |   2 +-
 .../core/metrics/MeterRegistryFactory.java         |   5 +
 .../apache/accumulo/core/metrics/MetricsInfo.java  | 127 ++++++++
 .../accumulo/core/metrics/MetricsProducer.java     |  43 ---
 .../apache/accumulo/core/metrics/MetricsUtil.java  | 126 +-------
 .../spi/metrics/LoggingMeterRegistryFactory.java   |  81 +++++
 .../core/spi/metrics/MeterRegistryFactory.java     |  72 +++++
 .../apache/accumulo/core/util/cache/Caches.java    |   6 +-
 .../accumulo/core/util/threads/ThreadPools.java    |  38 ++-
 .../core/iterators/user/GrepIteratorTest.java      |  18 ++
 .../metrics/LoggingMeterRegistryFactoryTest.java}  |  43 +--
 .../miniclusterImpl/MiniAccumuloConfigImpl.java    |   3 +
 .../org/apache/accumulo/server/AbstractServer.java |   9 +-
 .../org/apache/accumulo/server/ServerContext.java  |  13 +
 .../server/compaction/PausedCompactionMetrics.java |   5 +-
 .../conf/store/impl/PropCacheCaffeineImpl.java     |  18 +-
 .../server/conf/store/impl/PropStoreMetrics.java   |  92 ------
 .../server/conf/store/impl/ZooPropLoader.java      |  19 +-
 .../server/conf/store/impl/ZooPropStore.java       |  15 +-
 .../server/metrics/MeterRegistryEnvPropImpl.java   |  75 +++++
 .../accumulo/server/metrics/MetricsInfoImpl.java   | 334 +++++++++++++++++++++
 .../accumulo/server/metrics/ProcessMetrics.java    |   5 +-
 .../apache/accumulo/server/rpc/TServerUtils.java   |  11 +-
 .../apache/accumulo/server/rpc/TimedProcessor.java |   6 +-
 .../conf/store/impl/PropCacheCaffeineImplTest.java |   6 +-
 .../server/conf/store/impl/PropStoreEventTest.java |   8 +-
 .../server/conf/store/impl/ZooPropLoaderTest.java  |  78 ++---
 .../metrics/MeterRegistryEnvPropImplTest.java      |  57 ++++
 .../server/metrics/MetricsInfoImplTest.java        |  84 ++++++
 .../accumulo/server/rpc/TServerUtilsTest.java      |  19 +-
 .../org/apache/accumulo/compactor/Compactor.java   |  20 +-
 .../apache/accumulo/gc/SimpleGarbageCollector.java |  20 +-
 .../java/org/apache/accumulo/manager/Manager.java  |  78 ++---
 .../coordinator/CompactionCoordinator.java         |  10 +-
 .../compaction/coordinator/QueueMetrics.java       |  26 +-
 .../accumulo/manager/metrics/ManagerMetrics.java   |  26 +-
 .../accumulo/manager/metrics/fate/FateMetrics.java |  71 +++--
 .../compaction/CompactionCoordinatorTest.java      |  37 ++-
 .../java/org/apache/accumulo/monitor/Monitor.java  |   6 +
 .../org/apache/accumulo/tserver/ScanServer.java    |  18 +-
 .../org/apache/accumulo/tserver/TabletServer.java  |  30 +-
 .../tserver/TabletServerResourceManager.java       |  23 +-
 .../tserver/metrics/TabletServerMinCMetrics.java   |   8 +-
 .../tserver/metrics/TabletServerScanMetrics.java   |  24 +-
 .../accumulo/shell/commands/GrepCommand.java       |   2 +-
 .../test/conf/store/PropCacheCaffeineImplZkIT.java |  10 +-
 .../accumulo/test/functional/ZombieTServer.java    |   2 +-
 .../apache/accumulo/test/metrics/MetricsIT.java    |  48 +--
 .../test/metrics/TestStatsDRegistryFactory.java    |   6 +-
 .../accumulo/test/performance/NullTserver.java     |   2 +-
 .../org/apache/accumulo/test/shell/ShellIT.java    |   1 +
 test/src/main/resources/log4j2-test.properties     |  23 --
 54 files changed, 1259 insertions(+), 684 deletions(-)
 create mode 100644 core/src/main/java/org/apache/accumulo/core/metrics/MetricsInfo.java
 create mode 100644 core/src/main/java/org/apache/accumulo/core/spi/metrics/LoggingMeterRegistryFactory.java
 create mode 100644 core/src/main/java/org/apache/accumulo/core/spi/metrics/MeterRegistryFactory.java
 copy core/src/test/java/org/apache/accumulo/core/{cli/TestHelp.java => spi/metrics/LoggingMeterRegistryFactoryTest.java} (50%)
 delete mode 100644 server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropStoreMetrics.java
 create mode 100644 server/base/src/main/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImpl.java
 create mode 100644 server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java
 create mode 100644 server/base/src/test/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImplTest.java
 create mode 100644 server/base/src/test/java/org/apache/accumulo/server/metrics/MetricsInfoImplTest.java


(accumulo) 01/01: Merge branch 'main' into elasticity

Posted by ct...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 07bf1bc6fd3c72ad77df0ba363175e34c474e3ec
Merge: ee04ad3aa5 65dd34fad4
Author: Christopher Tubbs <ct...@apache.org>
AuthorDate: Fri Apr 26 15:22:37 2024 -0400

    Merge branch 'main' into elasticity

 core/pom.xml                                       |   1 +
 .../org/apache/accumulo/core/conf/Property.java    |  33 +-
 .../accumulo/core/iterators/user/GrepIterator.java |   2 +-
 .../core/metrics/MeterRegistryFactory.java         |   5 +
 .../apache/accumulo/core/metrics/MetricsInfo.java  | 127 ++++++++
 .../accumulo/core/metrics/MetricsProducer.java     |  43 ---
 .../apache/accumulo/core/metrics/MetricsUtil.java  | 126 +-------
 .../spi/metrics/LoggingMeterRegistryFactory.java   |  81 +++++
 .../core/spi/metrics/MeterRegistryFactory.java     |  72 +++++
 .../apache/accumulo/core/util/cache/Caches.java    |   6 +-
 .../accumulo/core/util/threads/ThreadPools.java    |  38 ++-
 .../core/iterators/user/GrepIteratorTest.java      |  18 ++
 .../metrics/LoggingMeterRegistryFactoryTest.java   |  52 ++++
 .../miniclusterImpl/MiniAccumuloConfigImpl.java    |   3 +
 .../org/apache/accumulo/server/AbstractServer.java |   9 +-
 .../org/apache/accumulo/server/ServerContext.java  |  13 +
 .../server/compaction/PausedCompactionMetrics.java |   5 +-
 .../conf/store/impl/PropCacheCaffeineImpl.java     |  18 +-
 .../server/conf/store/impl/PropStoreMetrics.java   |  92 ------
 .../server/conf/store/impl/ZooPropLoader.java      |  19 +-
 .../server/conf/store/impl/ZooPropStore.java       |  15 +-
 .../server/metrics/MeterRegistryEnvPropImpl.java   |  75 +++++
 .../accumulo/server/metrics/MetricsInfoImpl.java   | 334 +++++++++++++++++++++
 .../accumulo/server/metrics/ProcessMetrics.java    |   5 +-
 .../apache/accumulo/server/rpc/TServerUtils.java   |  11 +-
 .../apache/accumulo/server/rpc/TimedProcessor.java |   6 +-
 .../conf/store/impl/PropCacheCaffeineImplTest.java |   6 +-
 .../server/conf/store/impl/PropStoreEventTest.java |   8 +-
 .../server/conf/store/impl/ZooPropLoaderTest.java  |  78 ++---
 .../metrics/MeterRegistryEnvPropImplTest.java      |  57 ++++
 .../server/metrics/MetricsInfoImplTest.java        |  84 ++++++
 .../accumulo/server/rpc/TServerUtilsTest.java      |  19 +-
 .../org/apache/accumulo/compactor/Compactor.java   |  20 +-
 .../apache/accumulo/gc/SimpleGarbageCollector.java |  20 +-
 .../java/org/apache/accumulo/manager/Manager.java  |  78 ++---
 .../coordinator/CompactionCoordinator.java         |  10 +-
 .../compaction/coordinator/QueueMetrics.java       |  26 +-
 .../accumulo/manager/metrics/ManagerMetrics.java   |  26 +-
 .../accumulo/manager/metrics/fate/FateMetrics.java |  71 +++--
 .../compaction/CompactionCoordinatorTest.java      |  37 ++-
 .../java/org/apache/accumulo/monitor/Monitor.java  |   6 +
 .../org/apache/accumulo/tserver/ScanServer.java    |  18 +-
 .../org/apache/accumulo/tserver/TabletServer.java  |  30 +-
 .../tserver/TabletServerResourceManager.java       |  23 +-
 .../tserver/metrics/TabletServerMinCMetrics.java   |   8 +-
 .../tserver/metrics/TabletServerScanMetrics.java   |  24 +-
 .../accumulo/shell/commands/GrepCommand.java       |   2 +-
 .../test/conf/store/PropCacheCaffeineImplZkIT.java |  10 +-
 .../accumulo/test/functional/ZombieTServer.java    |   2 +-
 .../apache/accumulo/test/metrics/MetricsIT.java    |  48 +--
 .../test/metrics/TestStatsDRegistryFactory.java    |   6 +-
 .../accumulo/test/performance/NullTserver.java     |   2 +-
 .../org/apache/accumulo/test/shell/ShellIT.java    |   1 +
 test/src/main/resources/log4j2-test.properties     |  23 --
 54 files changed, 1288 insertions(+), 664 deletions(-)

diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index b05e0aba4e,463b37cbcf..a708b14be1
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -339,12 -322,15 +339,19 @@@ public enum Property 
        "Enables metrics functionality using Micrometer.", "2.1.0"),
    GENERAL_MICROMETER_JVM_METRICS_ENABLED("general.micrometer.jvm.metrics.enabled", "false",
        PropertyType.BOOLEAN, "Enables JVM metrics functionality using Micrometer.", "2.1.0"),
-   GENERAL_MICROMETER_FACTORY("general.micrometer.factory", "", PropertyType.CLASSNAME,
-       "Name of class that implements MeterRegistryFactory.", "2.1.0"),
+   GENERAL_MICROMETER_FACTORY("general.micrometer.factory",
+       "org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory",
+       PropertyType.CLASSNAMELIST,
+       "A comma separated list of one or more class names that implements"
+           + " org.apache.accumulo.core.spi.metrics.MeterRegistryFactory. Prior to"
+           + " 2.1.3 this was a single value and the default was an empty string.  In 2.1.3 the default"
+           + " was changed and it now can accept multiple class names. The metrics spi was introduced in 2.1.3,"
+           + " the deprecated factory is org.apache.accumulo.core.metrics.MeterRegistryFactory.",
+       "2.1.0"),
 +  GENERAL_MICROMETER_USER_TAGS("general.micrometer.user.tags", "", PropertyType.STRING,
 +      "A comma separated list of tags to emit with all metrics from the process. Example:"
 +          + "\"tag1=value1,tag2=value2\".",
 +      "4.0.0"),
    GENERAL_PROCESS_BIND_ADDRESS("general.process.bind.addr", "0.0.0.0", PropertyType.STRING,
        "The local IP address to which this server should bind for sending and receiving network traffic.",
        "3.0.0"),
@@@ -1414,11 -1503,28 +1421,26 @@@
  
    public static final EnumSet<Property> fixedProperties = EnumSet.of(
        // port options
-       GC_PORT, MANAGER_CLIENTPORT, TSERV_CLIENTPORT,
+       GC_PORT, MANAGER_CLIENTPORT, TSERV_CLIENTPORT, SSERV_CLIENTPORT, SSERV_PORTSEARCH,
+       COMPACTOR_PORTSEARCH, TSERV_PORTSEARCH,
+ 
+       // max message options
+       SSERV_MAX_MESSAGE_SIZE, TSERV_MAX_MESSAGE_SIZE, COMPACTOR_MAX_MESSAGE_SIZE,
 -      COMPACTION_COORDINATOR_MAX_MESSAGE_SIZE,
  
-       // tserver cache options
+       // block cache options
        TSERV_CACHE_MANAGER_IMPL, TSERV_DATACACHE_SIZE, TSERV_INDEXCACHE_SIZE,
-       TSERV_SUMMARYCACHE_SIZE,
+       TSERV_SUMMARYCACHE_SIZE, SSERV_DATACACHE_SIZE, SSERV_INDEXCACHE_SIZE, SSERV_SUMMARYCACHE_SIZE,
+ 
+       // blocksize options
+       TSERV_DEFAULT_BLOCKSIZE, SSERV_DEFAULT_BLOCKSIZE,
+ 
+       // sserver specific options
+       SSERVER_SCAN_REFERENCE_EXPIRATION_TIME, SSERV_CACHED_TABLET_METADATA_EXPIRATION,
+ 
+       // thread options
+       TSERV_MINTHREADS, TSERV_MINTHREADS_TIMEOUT, SSERV_MINTHREADS, SSERV_MINTHREADS_TIMEOUT,
 -      COMPACTION_COORDINATOR_MINTHREADS, COMPACTION_COORDINATOR_MINTHREADS_TIMEOUT,
+       MANAGER_MINTHREADS, MANAGER_MINTHREADS_TIMEOUT, COMPACTOR_MINTHREADS,
+       COMPACTOR_MINTHREADS_TIMEOUT,
  
        // others
        TSERV_NATIVEMAP_ENABLED, TSERV_SCAN_MAX_OPENFILES, MANAGER_RECOVERY_WAL_EXISTENCE_CACHE_TIME);
diff --cc core/src/main/java/org/apache/accumulo/core/metrics/MetricsInfo.java
index 0000000000,b6ef72b1df..18ad925882
mode 000000,100644..100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsInfo.java
@@@ -1,0 -1,114 +1,127 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *   https://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing,
+  * software distributed under the License is distributed on an
+  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  * KIND, either express or implied.  See the License for the
+  * specific language governing permissions and limitations
+  * under the License.
+  */
+ package org.apache.accumulo.core.metrics;
+ 
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.List;
+ import java.util.Objects;
+ 
+ import com.google.common.net.HostAndPort;
+ 
+ import io.micrometer.core.instrument.MeterRegistry;
+ import io.micrometer.core.instrument.Tag;
+ 
+ public interface MetricsInfo {
+ 
+   /**
+    * Convenience method to create tag name / value pair for the instance name
+    *
+    * @param instanceName the instance name
+    */
+   static Tag instanceNameTag(final String instanceName) {
+     Objects.requireNonNull(instanceName,
+         "cannot create the tag without providing the instance name");
+     return Tag.of("instance.name", instanceName);
+   }
+ 
+   /**
+    * Convenience method to create tag name / value pair for the process name
+    *
+    * @param processName the process name
+    */
+   static Tag processTag(final String processName) {
+     Objects.requireNonNull(processName, "cannot create the tag without providing the process name");
+     return Tag.of("process.name", processName);
+   }
+ 
++  /**
++   * Convenience method to create tag name / value pair for the resource group name
++   *
++   * @param resourceGroupName the resource group name
++   */
++  static Tag resourceGroupTag(final String resourceGroupName) {
++    if (resourceGroupName == null || resourceGroupName.isEmpty()) {
++      return Tag.of("resource.group", "NOT_PROVIDED");
++    }
++    return Tag.of("resource.group", resourceGroupName);
++  }
++
+   /**
+    * Convenience method to create tag name / value pairs for the host and port from address
+    * host:port pair.
+    *
+    * @param hostAndPort the host:port pair
+    */
+   static List<Tag> addressTags(final HostAndPort hostAndPort) {
+     Objects.requireNonNull(hostAndPort, "cannot create the tag without providing the hostAndPort");
+     List<Tag> tags = new ArrayList<>(2);
+     tags.add(Tag.of("host", hostAndPort.getHost()));
+     int port = hostAndPort.getPort();
+     if (port != 0) {
+       tags.add(Tag.of("port", Integer.toString(hostAndPort.getPort())));
+     }
+     return Collections.unmodifiableList(tags);
+   }
+ 
+   boolean isMetricsEnabled();
+ 
+   /**
+    * Convenience method to set the common tags for application (process), host and port.
+    *
+    * @param applicationName the application (process) name.
+    * @param hostAndPort the host:port pair
+    */
 -  void addServiceTags(final String applicationName, final HostAndPort hostAndPort);
++  void addServiceTags(final String applicationName, final HostAndPort hostAndPort,
++      final String resourceGroup);
+ 
+   /**
+    * Add the list of tag name / value pair to the common tags that will be emitted with all metrics.
+    * Common tags must ne added before initialization of any registries. Tags that are added after a
+    * registry is initialized may not be emitted by the underlying metrics system. This would cause
+    * inconsistent grouping and filtering based on tags,
+    *
+    * @param updates list of tags (name / value pairs)
+    */
+   void addCommonTags(final List<Tag> updates);
+ 
+   /**
+    * Get the current list of common tags.
+    */
+   Collection<Tag> getCommonTags();
+ 
+   void addRegistry(MeterRegistry registry);
+ 
+   void addMetricsProducers(MetricsProducer... producer);
+ 
+   /**
+    * Initialize the metrics system. This sets the list of common tags that are emitted with the
+    * metrics.
+    */
+   void init();
+ 
+   MeterRegistry getRegistry();
+ 
+   /**
+    * Close the underlying registry and release resources. The registry will not accept new meters
+    * and will stop publishing metrics.
+    */
+   void close();
+ }
diff --cc core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index f1d39986dd,e50d2d9c03..a2dba3957b
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -592,63 -556,6 +592,27 @@@ import io.micrometer.core.instrument.Me
   * <td>Distribution Summary</td>
   * <td></td>
   * </tr>
-  * <!-- ZooKeeper property cache -->
-  * <tr>
-  * <td>N/A</td>
-  * <td>N/A</td>
-  * <td>{@value #METRICS_PROPSTORE_LOAD_TIMER}</td>
-  * <td>Timer</td>
-  * <td></td>
-  * </tr>
-  * <tr>
-  * <td>N/A</td>
-  * <td>N/A</td>
-  * <td>{@value #METRICS_PROPSTORE_REFRESH_COUNT}</td>
-  * <td>Counter</td>
-  * <td></td>
-  * </tr>
-  * <tr>
-  * <td>N/A</td>
-  * <td>N/A</td>
-  * <td>{@value #METRICS_PROPSTORE_REFRESH_LOAD_COUNT}</td>
-  * <td>Counter</td>
-  * <td></td>
-  * </tr>
-  * <tr>
-  * <td>N/A</td>
-  * <td>N/A</td>
-  * <td>{@value #METRICS_PROPSTORE_EVICTION_COUNT}</td>
-  * <td>Counter</td>
-  * <td></td>
-  * </tr>
-  * <tr>
-  * <td>N/A</td>
-  * <td>N/A</td>
-  * <td>{@value #METRICS_PROPSTORE_ZK_ERROR_COUNT}</td>
-  * <td>Counter</td>
-  * <td></td>
-  * </tr>
 + * <tr>
 + * <td>N/A</td>
 + * <td>N/A</td>
 + * <td>{@link #METRICS_MANAGER_ROOT_TGW_ERRORS}</td>
 + * <td>Gauge</td>
 + * <td></td>
 + * </tr>
 + * <tr>
 + * <td>N/A</td>
 + * <td>N/A</td>
 + * <td>{@link #METRICS_MANAGER_META_TGW_ERRORS}</td>
 + * <td>Gauge</td>
 + * <td></td>
 + * </tr>
 + * <tr>
 + * <td>N/A</td>
 + * <td>N/A</td>
 + * <td>{@link #METRICS_MANAGER_USER_TGW_ERRORS}</td>
 + * <td>Gauge</td>
 + * <td></td>
 + * </tr>
   * </table>
   *
   * @since 2.1.0
diff --cc core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
index 311bb5e190,0000000000..2608dbab7d
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
@@@ -1,182 -1,0 +1,58 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   https://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.accumulo.core.metrics;
 +
- import java.lang.reflect.InvocationTargetException;
- import java.util.ArrayList;
- import java.util.Collections;
- import java.util.List;
- import java.util.concurrent.ExecutorService;
 +import java.util.regex.Matcher;
 +import java.util.regex.Pattern;
 +
- import org.apache.accumulo.core.classloader.ClassLoaderUtil;
- import org.apache.accumulo.core.conf.AccumuloConfiguration;
- import org.apache.accumulo.core.conf.Property;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- 
- import com.google.common.net.HostAndPort;
- 
- import io.micrometer.core.instrument.MeterRegistry;
- import io.micrometer.core.instrument.Metrics;
- import io.micrometer.core.instrument.Tag;
- import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
- import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;
- import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
- import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
- import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
- import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
- 
 +public class MetricsUtil {
 +
-   private static final Logger LOG = LoggerFactory.getLogger(MetricsUtil.class);
- 
-   private static JvmGcMetrics gc;
-   private static List<Tag> commonTags;
-   private static Pattern camelCasePattern = Pattern.compile("[a-z][A-Z][a-z]");
- 
-   public static void initializeMetrics(final AccumuloConfiguration conf, final String appName,
-       final HostAndPort address, final String instanceName, final String resourceGroup)
-       throws ClassNotFoundException, InstantiationException, IllegalAccessException,
-       IllegalArgumentException, InvocationTargetException, NoSuchMethodException,
-       SecurityException {
-     initializeMetrics(conf.getBoolean(Property.GENERAL_MICROMETER_ENABLED),
-         conf.getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED),
-         conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address, instanceName,
-         resourceGroup, conf.get(Property.GENERAL_MICROMETER_USER_TAGS));
-   }
- 
-   private static void initializeMetrics(boolean enabled, boolean jvmMetricsEnabled,
-       String factoryClass, String appName, HostAndPort address, String instanceName,
-       String resourceGroup, String userTags) throws ClassNotFoundException, InstantiationException,
-       IllegalAccessException, IllegalArgumentException, InvocationTargetException,
-       NoSuchMethodException, SecurityException {
- 
-     LOG.info("initializing metrics, enabled:{}, class:{}", enabled, factoryClass);
- 
-     if (enabled && factoryClass != null && !factoryClass.isEmpty()) {
- 
-       String processName = appName;
-       String serviceInstance = System.getProperty("accumulo.metrics.service.instance", "");
-       if (!serviceInstance.isBlank()) {
-         processName += serviceInstance;
-       }
- 
-       List<Tag> tags = new ArrayList<>();
-       tags.add(Tag.of("instance.name", instanceName));
-       tags.add(Tag.of("process.name", processName));
-       tags.add(Tag.of("resource.group", resourceGroup));
-       if (address != null) {
-         if (!address.getHost().isEmpty()) {
-           tags.add(Tag.of("host", address.getHost()));
-         }
-         if (address.getPort() > 0) {
-           tags.add(Tag.of("port", Integer.toString(address.getPort())));
-         }
-       }
- 
-       if (!userTags.isEmpty()) {
-         String[] userTagList = userTags.split(",");
-         for (String userTag : userTagList) {
-           String[] tagParts = userTag.split("=");
-           if (tagParts.length == 2) {
-             tags.add(Tag.of(tagParts[0], tagParts[1]));
-           } else {
-             LOG.warn("Malformed user metric tag: {} in property {}", userTag,
-                 Property.GENERAL_MICROMETER_USER_TAGS.getKey());
-           }
-         }
-       }
- 
-       commonTags = Collections.unmodifiableList(tags);
- 
-       Class<? extends MeterRegistryFactory> clazz =
-           ClassLoaderUtil.loadClass(factoryClass, MeterRegistryFactory.class);
-       MeterRegistryFactory factory = clazz.getDeclaredConstructor().newInstance();
- 
-       MeterRegistry registry = factory.create();
-       registry.config().commonTags(commonTags);
-       Metrics.addRegistry(registry);
- 
-       if (jvmMetricsEnabled) {
-         new ClassLoaderMetrics(commonTags).bindTo(Metrics.globalRegistry);
-         new JvmMemoryMetrics(commonTags).bindTo(Metrics.globalRegistry);
-         gc = new JvmGcMetrics(commonTags);
-         gc.bindTo(Metrics.globalRegistry);
-         new ProcessorMetrics(commonTags).bindTo(Metrics.globalRegistry);
-         new JvmThreadMetrics(commonTags).bindTo(Metrics.globalRegistry);
-       }
-     }
-   }
- 
-   public static void initializeProducers(MetricsProducer... producer) {
-     for (MetricsProducer p : producer) {
-       p.registerMetrics(Metrics.globalRegistry);
-       LOG.info("Metric producer {} initialized", p.getClass().getSimpleName());
-     }
-   }
- 
-   public static void addExecutorServiceMetrics(ExecutorService executor, String name) {
-     new ExecutorServiceMetrics(executor, name, commonTags).bindTo(Metrics.globalRegistry);
-   }
- 
-   public static List<Tag> getCommonTags() {
-     return commonTags;
-   }
++  private final static Pattern camelCasePattern = Pattern.compile("[a-z][A-Z][a-z]");
 +
 +  /**
 +   * This method replaces any intended delimiters with the "." delimiter that is used by micrometer.
 +   * Micrometer will then transform these delimiters to the metric producer's delimiter. Example:
 +   * "compactorQueue" becomes "compactor.queue" in micrometer. When using Prometheus,
 +   * "compactor.queue" would become "compactor_queue".
-    *
 +   */
 +  public static String formatString(String name) {
 +
 +    // Replace spaces with dot delimiter
 +    name = name.replace(" ", ".");
 +    // Replace snake_case with dot delimiter
 +    name = name.replace("_", ".");
 +    // Replace hyphens with dot delimiter
 +    name = name.replace("-", ".");
 +
 +    // Insert a dot delimiter before each capital letter found in the regex pattern.
 +    Matcher matcher = camelCasePattern.matcher(name);
 +    StringBuilder output = new StringBuilder(name);
 +    int insertCount = 0;
 +    while (matcher.find()) {
 +      // Pattern matches on a "aAa" pattern and inserts the dot before the uppercase character.
 +      // Results in "aAa" becoming "a.Aa".
 +      output.insert(matcher.start() + 1 + insertCount, ".");
 +      // The correct index position will shift as inserts occur.
 +      insertCount++;
 +    }
 +    name = output.toString();
 +    // remove all capital letters after the dot delimiters have been inserted.
 +    return name.toLowerCase();
 +  }
- 
-   public static void close() {
-     if (gc != null) {
-       gc.close();
-     }
-   }
- 
 +}
diff --cc core/src/main/java/org/apache/accumulo/core/util/cache/Caches.java
index f5ef8e4c8f,0000000000..8fb51f1dee
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/accumulo/core/util/cache/Caches.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/cache/Caches.java
@@@ -1,110 -1,0 +1,108 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   https://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.accumulo.core.util.cache;
 +
 +import static com.google.common.base.Suppliers.memoize;
 +
 +import java.util.function.Supplier;
 +
 +import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.core.metrics.MetricsUtil;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.github.benmanes.caffeine.cache.Caffeine;
 +
 +import io.micrometer.core.instrument.MeterRegistry;
 +import io.micrometer.core.instrument.binder.cache.CaffeineStatsCounter;
 +
 +public class Caches implements MetricsProducer {
 +
 +  public enum CacheName {
 +    BULK_IMPORT_FILE_LENGTHS,
 +    CLASSLOADERS,
 +    COMBINER_LOGGED_MSGS,
 +    COMPACTIONS_COMPLETED,
 +    COMPACTION_CONFIGS,
 +    COMPACTION_DIR_CACHE,
 +    COMPACTION_DISPATCHERS,
 +    COMPACTION_SERVICE_UNKNOWN,
 +    COMPACTOR_GROUP_ID,
 +    COMPRESSION_ALGORITHM,
 +    CRYPT_PASSWORDS,
 +    HOST_REGEX_BALANCER_TABLE_REGEX,
 +    INSTANCE_ID,
 +    NAMESPACE_ID,
 +    NAMESPACE_CONFIGS,
 +    PROP_CACHE,
 +    RECOVERY_MANAGER_PATH_CACHE,
 +    SCAN_SERVER_TABLET_METADATA,
 +    SERVICE_ENVIRONMENT_TABLE_CONFIGS,
 +    SPACE_AWARE_VOLUME_CHOICE,
 +    SPLITTER_FILES,
 +    SPLITTER_STARTING,
 +    SPLITTER_UNSPLITTABLE,
 +    TABLE_CONFIGS,
 +    TABLE_ID,
 +    TABLE_PARENT_CONFIGS,
 +    TABLE_ZOO_HELPER_CACHE,
 +    TSRM_FILE_LENGTHS,
 +    TINYLFU_BLOCK_CACHE,
-     VOLUME_HDFS_CONFIGS;
++    VOLUME_HDFS_CONFIGS
 +  }
 +
 +  private static final Logger LOG = LoggerFactory.getLogger(Caches.class);
 +  private static final Supplier<Caches> CACHES = memoize(() -> new Caches());
 +
 +  public static Caches getInstance() {
 +    return CACHES.get();
 +  }
 +
 +  private MeterRegistry registry = null;
 +
 +  private Caches() {}
 +
 +  @Override
 +  public void registerMetrics(MeterRegistry registry) {
 +    this.registry = registry;
 +  }
 +
 +  private boolean setupMicrometerMetrics(Caffeine<Object,Object> cacheBuilder, String name) {
 +    if (registry != null) {
 +      try {
-         cacheBuilder.recordStats(
-             () -> new CaffeineStatsCounter(registry, name, MetricsUtil.getCommonTags()));
++        cacheBuilder.recordStats(() -> new CaffeineStatsCounter(registry, name));
 +        LOG.trace("Metrics enabled for {} cache.", name);
 +        return true;
 +      } catch (IllegalStateException e) {
 +        // recordStats was already called by the cacheBuilder.
 +      }
 +    }
 +    return false;
 +  }
 +
 +  public Caffeine<Object,Object> createNewBuilder(CacheName name, boolean emitMetricsIfEnabled) {
 +    Caffeine<Object,Object> cacheBuilder = Caffeine.newBuilder();
 +    boolean metricsConfigured = false;
 +    if (emitMetricsIfEnabled) {
 +      metricsConfigured = setupMicrometerMetrics(cacheBuilder, name.name());
 +    }
 +    LOG.trace("Caffeine builder created for {}, metrics enabled: {}", name, metricsConfigured);
 +    return cacheBuilder;
 +  }
 +
 +}
diff --cc server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java
index f7e545eef3,bd66689d0f..354f89b63b
--- a/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/AbstractServer.java
@@@ -28,9 -27,7 +28,8 @@@ import org.apache.accumulo.core.classlo
  import org.apache.accumulo.core.cli.ConfigOpts;
  import org.apache.accumulo.core.conf.AccumuloConfiguration;
  import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.conf.SiteConfiguration;
  import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.core.metrics.MetricsUtil;
  import org.apache.accumulo.core.trace.TraceUtil;
  import org.apache.accumulo.core.util.threads.ThreadPools;
  import org.apache.accumulo.server.mem.LowMemoryDetector;
diff --cc server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImpl.java
index 1f04878899,d55c9465b1..b7e87b546e
--- a/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/conf/store/impl/PropCacheCaffeineImpl.java
@@@ -53,28 -51,20 +53,21 @@@ public class PropCacheCaffeineImpl impl
    private final LoadingCache<PropStoreKey<?>,VersionedProperties> cache;
  
    private PropCacheCaffeineImpl(final CacheLoader<PropStoreKey<?>,VersionedProperties> cacheLoader,
-       final PropStoreMetrics metrics, final Ticker ticker, boolean runTasksInline) {
-     this.metrics = metrics;
+       final Ticker ticker, boolean runTasksInline) {
 -    var builder = Caffeine.newBuilder().expireAfterAccess(EXPIRE_MIN, BASE_TIME_UNITS)
 -        .evictionListener(this::evictionNotifier);
 +    Caffeine<Object,Object> caffeine =
 +        Caches.getInstance().createNewBuilder(CacheName.PROP_CACHE, true)
 +            .expireAfterAccess(EXPIRE_MIN, BASE_TIME_UNITS);
      if (runTasksInline) {
 -      builder.executor(Runnable::run);
 +      caffeine.executor(Runnable::run);
      } else {
 -      builder.executor(executor);
 +      caffeine.executor(executor);
      }
      if (ticker != null) {
 -      builder.ticker(ticker);
 +      caffeine.ticker(ticker);
      }
 -    cache = builder.build(cacheLoader);
 +    cache = caffeine.evictionListener(this::evictionNotifier).build(cacheLoader);
    }
  
-   public PropStoreMetrics getMetrics() {
-     return metrics;
-   }
- 
    void evictionNotifier(PropStoreKey<?> propStoreKey, VersionedProperties value,
        RemovalCause cause) {
      log.trace("Evicted: ID: {} was evicted from cache. Reason: {}", propStoreKey, cause);
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java
index 0000000000,988bf38f63..61c7d9831b
mode 000000,100644..100644
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/MetricsInfoImpl.java
@@@ -1,0 -1,319 +1,334 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *   https://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing,
+  * software distributed under the License is distributed on an
+  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  * KIND, either express or implied.  See the License for the
+  * specific language governing permissions and limitations
+  * under the License.
+  */
+ package org.apache.accumulo.server.metrics;
+ 
+ import static org.apache.hadoop.util.StringUtils.getTrimmedStrings;
+ 
+ import java.lang.reflect.InvocationTargetException;
+ import java.time.Duration;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.concurrent.locks.Lock;
+ import java.util.concurrent.locks.ReentrantLock;
+ 
+ import org.apache.accumulo.core.classloader.ClassLoaderUtil;
+ import org.apache.accumulo.core.conf.Property;
+ import org.apache.accumulo.core.metrics.MetricsInfo;
+ import org.apache.accumulo.core.metrics.MetricsProducer;
+ import org.apache.accumulo.server.ServerContext;
+ import org.checkerframework.checker.nullness.qual.NonNull;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ 
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.net.HostAndPort;
+ 
+ import io.micrometer.core.instrument.Meter;
+ import io.micrometer.core.instrument.MeterRegistry;
+ import io.micrometer.core.instrument.Metrics;
+ import io.micrometer.core.instrument.Tag;
+ import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
+ import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
+ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
+ import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
+ import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
+ import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
+ import io.micrometer.core.instrument.config.MeterFilter;
+ import io.micrometer.core.instrument.distribution.DistributionStatisticConfig;
+ 
+ public class MetricsInfoImpl implements MetricsInfo {
+ 
+   private static final Logger LOG = LoggerFactory.getLogger(MetricsInfoImpl.class);
+ 
+   private final ServerContext context;
+ 
+   private final Lock lock = new ReentrantLock();
+ 
+   private final Map<String,Tag> commonTags;
+ 
+   // JvmGcMetrics are declared with AutoCloseable - keep reference to use with close()
+   private JvmGcMetrics jvmGcMetrics;
+ 
+   private final boolean metricsEnabled;
+ 
+   private CompositeMeterRegistry composite = null;
+   private final List<MeterRegistry> pendingRegistries = new ArrayList<>();
+ 
+   private final List<MetricsProducer> producers = new ArrayList<>();
+ 
+   public MetricsInfoImpl(final ServerContext context) {
+     this.context = context;
+     metricsEnabled = context.getConfiguration().getBoolean(Property.GENERAL_MICROMETER_ENABLED);
+     printMetricsConfig();
+     commonTags = new HashMap<>();
 -    Tag t = MetricsInfo.instanceNameTag(context.getInstanceName());
 -    commonTags.put(t.getKey(), t);
++
++    Tag iidTag = MetricsInfo.instanceNameTag(context.getInstanceName());
++    commonTags.put(iidTag.getKey(), iidTag);
++
++    var userTags = context.getConfiguration().get(Property.GENERAL_MICROMETER_USER_TAGS);
++    if (!userTags.isEmpty()) {
++      String[] userTagList = userTags.split(",");
++      for (String userTag : userTagList) {
++        String[] tagParts = userTag.split("=");
++        if (tagParts.length == 2) {
++          Tag tag = Tag.of(tagParts[0], tagParts[1]);
++          commonTags.put(tag.getKey(), tag);
++        } else {
++          LOG.warn("Malformed user metric tag: {} in property {}", userTag,
++              Property.GENERAL_MICROMETER_USER_TAGS.getKey());
++        }
++      }
++    }
+   }
+ 
+   private void printMetricsConfig() {
+     final boolean jvmMetricsEnabled =
+         context.getConfiguration().getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED);
+     LOG.info("micrometer metrics enabled: {}", metricsEnabled);
+     if (jvmMetricsEnabled) {
+       if (metricsEnabled) {
+         LOG.info("detailed jvm metrics enabled: {}", jvmMetricsEnabled);
+       } else {
+         LOG.info("requested jvm metrics, but micrometer metrics are disabled.");
+       }
+     }
+     if (metricsEnabled) {
+       LOG.info("metrics registry factories: {}",
+           context.getConfiguration().get(Property.GENERAL_MICROMETER_FACTORY));
+     }
+   }
+ 
+   @Override
+   public boolean isMetricsEnabled() {
+     return metricsEnabled;
+   }
+ 
+   /**
+    * Common tags for all services.
+    */
+   @Override
 -  public void addServiceTags(final String applicationName, final HostAndPort hostAndPort) {
++  public void addServiceTags(final String applicationName, final HostAndPort hostAndPort,
++      final String resourceGroupName) {
+     List<Tag> tags = new ArrayList<>();
+ 
 -    if (applicationName != null && !applicationName.isEmpty()) {
 -      tags.add(MetricsInfo.processTag(applicationName));
 -    }
 -    if (hostAndPort != null) {
 -      tags.addAll(MetricsInfo.addressTags(hostAndPort));
 -    }
++    tags.add(MetricsInfo.processTag(applicationName));
++    tags.addAll(MetricsInfo.addressTags(hostAndPort));
++    tags.add(MetricsInfo.resourceGroupTag(resourceGroupName));
++
+     addCommonTags(tags);
+   }
+ 
+   @Override
+   public void addCommonTags(List<Tag> updates) {
+     lock.lock();
+     try {
+       if (composite != null) {
+         LOG.warn(
 -            "Common tags after registry has been initialized may be ignored. Current common tags: {}",
 -            commonTags);
++            "Common tags after registry has been initialized may be ignored. Current common tags: {}, Adding: {}",
++            commonTags, updates);
+         return;
+       }
+       updates.forEach(t -> commonTags.put(t.getKey(), t));
+     } finally {
+       lock.unlock();
+     }
+   }
+ 
+   @Override
+   public Collection<Tag> getCommonTags() {
+     lock.lock();
+     try {
+       return Collections.unmodifiableCollection(commonTags.values());
+     } finally {
+       lock.unlock();
+     }
+   }
+ 
+   @Override
+   public void addRegistry(MeterRegistry registry) {
+     lock.lock();
+     try {
+       if (composite != null) {
+         composite.add(registry);
+       } else {
+         // defer until composite is initialized
+         pendingRegistries.add(registry);
+       }
+ 
+     } finally {
+       lock.unlock();
+     }
+   }
+ 
+   @Override
+   public void addMetricsProducers(MetricsProducer... producer) {
+     if (producer.length == 0) {
+       LOG.debug(
+           "called addMetricsProducers() without providing at least one producer - this has no effect");
+       return;
+     }
+     lock.lock();
+     try {
+       if (composite == null) {
+         producers.addAll(Arrays.asList(producer));
+       } else {
+         Arrays.stream(producer).forEach(p -> p.registerMetrics(composite));
+       }
+     } finally {
+       lock.unlock();
+     }
+   }
+ 
+   @Override
+   public MeterRegistry getRegistry() {
+     lock.lock();
+     try {
+       if (composite == null) {
+         throw new IllegalStateException("metrics have not been initialized, call init() first");
+       }
+     } finally {
+       lock.unlock();
+     }
+     return composite;
+   }
+ 
+   @Override
+   public void init() {
+     lock.lock();
+     try {
+       if (composite != null) {
+         LOG.warn("metrics registry has already been initialized");
+         return;
+       }
+       composite = new CompositeMeterRegistry();
+       composite.config().commonTags(commonTags.values());
+ 
+       LOG.info("Metrics initialization. common tags: {}", commonTags);
+ 
+       boolean jvmMetricsEnabled =
+           context.getConfiguration().getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED);
+ 
+       if (jvmMetricsEnabled) {
+         LOG.info("enabling detailed jvm, classloader, jvm gc and process metrics");
+         new ClassLoaderMetrics().bindTo(composite);
+         new JvmMemoryMetrics().bindTo(composite);
+         jvmGcMetrics = new JvmGcMetrics();
+         jvmGcMetrics.bindTo(composite);
+         new ProcessorMetrics().bindTo(composite);
+         new JvmThreadMetrics().bindTo(composite);
+       }
+ 
+       MeterFilter replicationFilter = new MeterFilter() {
+         @Override
+         public DistributionStatisticConfig configure(Meter.Id id,
+             @NonNull DistributionStatisticConfig config) {
+           if (id.getName().equals("replicationQueue")) {
+             return DistributionStatisticConfig.builder().percentiles(0.5, 0.75, 0.9, 0.95, 0.99)
+                 .expiry(Duration.ofMinutes(10)).build().merge(config);
+           }
+           return config;
+         }
+       };
+ 
+       if (isMetricsEnabled()) {
+         // user specified registries
+         String userRegistryFactories =
+             context.getConfiguration().get(Property.GENERAL_MICROMETER_FACTORY);
+ 
+         for (String factoryName : getTrimmedStrings(userRegistryFactories)) {
+           try {
+             MeterRegistry registry = getRegistryFromFactory(factoryName, context);
+             registry.config().commonTags(commonTags.values());
+             registry.config().meterFilter(replicationFilter);
+             addRegistry(registry);
+           } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException
+               | InstantiationException | IllegalAccessException ex) {
+             LOG.warn("Could not load registry {}", factoryName, ex);
+           }
+         }
+       }
+ 
+       pendingRegistries.forEach(registry -> composite.add(registry));
+ 
+       LOG.info("Metrics initialization. Register producers: {}", producers);
+       producers.forEach(p -> p.registerMetrics(composite));
+ 
+       Metrics.globalRegistry.add(composite);
+ 
+     } finally {
+       lock.unlock();
+     }
+   }
+ 
+   @VisibleForTesting
+   @SuppressWarnings({"deprecation",
+       "support for org.apache.accumulo.core.metrics.MeterRegistryFactory can be removed in 3.1"})
+   static MeterRegistry getRegistryFromFactory(final String factoryName, final ServerContext context)
+       throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException,
+       InstantiationException, IllegalAccessException {
+     try {
+       LOG.info("look for meter spi registry factory {}", factoryName);
+       Class<? extends org.apache.accumulo.core.spi.metrics.MeterRegistryFactory> clazz =
+           ClassLoaderUtil.loadClass(factoryName,
+               org.apache.accumulo.core.spi.metrics.MeterRegistryFactory.class);
+       org.apache.accumulo.core.spi.metrics.MeterRegistryFactory factory =
+           clazz.getDeclaredConstructor().newInstance();
+       org.apache.accumulo.core.spi.metrics.MeterRegistryFactory.InitParameters initParameters =
+           new MeterRegistryEnvPropImpl(context);
+       return factory.create(initParameters);
+     } catch (ClassCastException ex) {
+       // empty. On exception try deprecated version
+     }
+     try {
+       LOG.info("find legacy meter registry factory {}", factoryName);
+       Class<? extends org.apache.accumulo.core.metrics.MeterRegistryFactory> clazz = ClassLoaderUtil
+           .loadClass(factoryName, org.apache.accumulo.core.metrics.MeterRegistryFactory.class);
+       org.apache.accumulo.core.metrics.MeterRegistryFactory factory =
+           clazz.getDeclaredConstructor().newInstance();
+       return factory.create();
+     } catch (ClassCastException ex) {
+       // empty. No valid metrics factory, fall through and then throw exception.
+     }
+     throw new ClassNotFoundException(
+         "Could not find appropriate class implementing a MetricsFactory for: " + factoryName);
+   }
+ 
+   @Override
+   public synchronized void close() {
+     LOG.info("Closing metrics registry");
+     if (jvmGcMetrics != null) {
+       jvmGcMetrics.close();
+       jvmGcMetrics = null;
+     }
+     if (composite != null) {
+       composite.close();
+       composite = null;
+     }
+   }
+ 
+   @Override
+   public String toString() {
+     return "MetricsCommonTags{tags=" + getCommonTags() + '}';
+   }
+ }
diff --cc server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java
index 8b2d8c9ee2,8b44525c53..17e8e42197
--- a/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/metrics/ProcessMetrics.java
@@@ -18,11 -18,11 +18,10 @@@
   */
  package org.apache.accumulo.server.metrics;
  
 -import java.util.List;
 -
  import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.core.metrics.MetricsUtil;
  import org.apache.accumulo.server.ServerContext;
  
 +import io.micrometer.core.instrument.Counter;
  import io.micrometer.core.instrument.MeterRegistry;
  
  public class ProcessMetrics implements MetricsProducer {
@@@ -36,14 -35,7 +35,14 @@@
  
    @Override
    public void registerMetrics(MeterRegistry registry) {
-     registry.gauge(METRICS_LOW_MEMORY, MetricsUtil.getCommonTags(), this, this::lowMemDetected);
-     idleCounter = registry.counter(METRICS_SERVER_IDLE, MetricsUtil.getCommonTags());
 -    registry.gauge(METRICS_LOW_MEMORY, List.of(), this, this::lowMemDetected);
++    registry.gauge(METRICS_LOW_MEMORY, this, this::lowMemDetected);
++    idleCounter = registry.counter(METRICS_SERVER_IDLE);
 +  }
 +
 +  public void incrementIdleCounter() {
 +    if (idleCounter != null) {
 +      idleCounter.increment();
 +    }
    }
  
    private int lowMemDetected(ProcessMetrics processMetrics) {
diff --cc server/base/src/test/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImplTest.java
index 0000000000,ff0f958dd5..9929a8acbb
mode 000000,100644..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImplTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/metrics/MeterRegistryEnvPropImplTest.java
@@@ -1,0 -1,50 +1,57 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *   https://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing,
+  * software distributed under the License is distributed on an
+  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  * KIND, either express or implied.  See the License for the
+  * specific language governing permissions and limitations
+  * under the License.
+  */
+ package org.apache.accumulo.server.metrics;
+ 
++import static org.easymock.EasyMock.createMock;
+ import static org.easymock.EasyMock.expect;
+ import static org.easymock.EasyMock.replay;
++import static org.easymock.EasyMock.verify;
+ import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertNotNull;
+ 
+ import java.util.Map;
+ 
+ import org.apache.accumulo.core.conf.ConfigurationCopy;
++import org.apache.accumulo.core.util.cache.Caches;
+ import org.apache.accumulo.server.ServerContext;
 -import org.easymock.EasyMock;
+ import org.junit.jupiter.api.Test;
+ 
+ public class MeterRegistryEnvPropImplTest {
+ 
+   @Test
+   public void customParamsTest() {
 -    ServerContext context = EasyMock.createMock(ServerContext.class);
++    ServerContext context = createMock(ServerContext.class);
+     ConfigurationCopy conf = new ConfigurationCopy();
+     conf.set("unknown", "none");
+     conf.set("general.custom.metrics.opts.sample.p1", "v1");
+     conf.set("general.custom.metrics.opts.sample.p2", "v2");
+ 
+     expect(context.getConfiguration()).andReturn(conf).anyTimes();
+ 
++    var c = Caches.getInstance();
++    expect(context.getCaches()).andReturn(c).anyTimes();
+     replay(context);
+     MeterRegistryEnvPropImpl env = new MeterRegistryEnvPropImpl(context);
+     assertEquals(Map.of("sample.p1", "v1", "sample.p2", "v2"), env.getOptions());
+     assertNotNull(env.getServiceEnv());
++
++    verify(context);
++
+   }
+ }
diff --cc server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index db21c2605e,ebd195a22f..39cb8697f0
--- a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++ b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -593,16 -585,11 +592,11 @@@ public class Compactor extends Abstract
        throw new RuntimeException("Error registering compactor in ZooKeeper", e);
      }
  
-     try {
-       MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName,
-           clientAddress, getContext().getInstanceName(), this.getResourceGroup());
-       pausedMetrics = new PausedCompactionMetrics();
-       MetricsUtil.initializeProducers(this, pausedMetrics);
-     } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
-         | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
-         | SecurityException e1) {
-       LOG.error("Error initializing metrics, metrics will not be emitted.", e1);
-     }
+     MetricsInfo metricsInfo = getContext().getMetricsInfo();
 -    metricsInfo.addServiceTags(getApplicationName(), clientAddress);
++    metricsInfo.addServiceTags(getApplicationName(), clientAddress, getResourceGroup());
+ 
+     metricsInfo.addMetricsProducers(this, pausedMetrics);
+     metricsInfo.init();
  
      var watcher = new CompactionWatcher(getConfiguration());
      var schedExecutor = ThreadPools.getServerThreadPools()
diff --cc server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index ec1f623377,e9c23b1729..0a40057890
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@@ -22,12 -22,6 +22,11 @@@ import static com.google.common.util.co
  
  import java.io.FileNotFoundException;
  import java.io.IOException;
- import java.lang.reflect.InvocationTargetException;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Set;
  import java.util.UUID;
  import java.util.concurrent.TimeUnit;
  import java.util.stream.IntStream;
@@@ -49,14 -42,11 +48,14 @@@ import org.apache.accumulo.core.lock.Se
  import org.apache.accumulo.core.lock.ServiceLockData.ThriftService;
  import org.apache.accumulo.core.metadata.AccumuloTable;
  import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
- import org.apache.accumulo.core.metrics.MetricsUtil;
+ import org.apache.accumulo.core.metrics.MetricsInfo;
  import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
 +import org.apache.accumulo.core.spi.balancer.TableLoadBalancer;
  import org.apache.accumulo.core.trace.TraceUtil;
  import org.apache.accumulo.core.util.Halt;
 +import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
  import org.apache.accumulo.core.util.threads.ThreadPools;
 +import org.apache.accumulo.core.util.time.NanoTime;
  import org.apache.accumulo.gc.metrics.GcCycleMetrics;
  import org.apache.accumulo.gc.metrics.GcMetrics;
  import org.apache.accumulo.server.AbstractServer;
@@@ -167,16 -154,11 +166,10 @@@ public class SimpleGarbageCollector ext
        System.exit(1);
      }
  
-     try {
-       MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, address,
-           getContext().getInstanceName(), this.getResourceGroup());
-       MetricsUtil.initializeProducers(this, new GcMetrics(this));
-     } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
-         | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
-         | SecurityException e1) {
-       log.error("Error initializing metrics, metrics will not be emitted.", e1);
-     }
- 
+     MetricsInfo metricsInfo = getContext().getMetricsInfo();
 -    metricsInfo.addServiceTags(getApplicationName(), address);
 -
++    metricsInfo.addServiceTags(getApplicationName(), address, getResourceGroup());
+     metricsInfo.addMetricsProducers(this, new GcMetrics(this));
+     metricsInfo.init();
      try {
        long delay = getStartDelay();
        log.debug("Sleeping for {} milliseconds before beginning garbage collection cycles", delay);
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index d17f5f570c,5c6cc2f436..8ec092f5fc
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -103,9 -94,12 +102,11 @@@ import org.apache.accumulo.core.manager
  import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
  import org.apache.accumulo.core.metadata.AccumuloTable;
  import org.apache.accumulo.core.metadata.TServerInstance;
 -import org.apache.accumulo.core.metadata.TabletLocationState;
  import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
  import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
- import org.apache.accumulo.core.metrics.MetricsUtil;
+ import org.apache.accumulo.core.metrics.MetricsInfo;
+ import org.apache.accumulo.core.metrics.MetricsProducer;
  import org.apache.accumulo.core.security.Authorizations;
  import org.apache.accumulo.core.spi.balancer.BalancerEnvironment;
  import org.apache.accumulo.core.spi.balancer.SimpleLoadBalancer;
@@@ -944,7 -1074,6 +948,9 @@@ public class Manager extends AbstractSe
      // Start the Manager's Fate Service
      fateServiceHandler = new FateServiceHandler(this);
      managerClientHandler = new ManagerClientServiceHandler(this);
-     compactionCoordinator = new CompactionCoordinator(context, security, fateRefs);
++    compactionCoordinator =
++        new CompactionCoordinator(context, security, fateRefs, getResourceGroup());
++
      // Start the Manager's Client service
      // Ensure that calls before the manager gets the lock fail
      ManagerClientService.Iface haProxy =
@@@ -979,16 -1108,12 +985,12 @@@
        managerUpgrading.set(true);
      }
  
-     ManagerMetrics mm = new ManagerMetrics(getConfiguration(), this);
-     try {
-       MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName,
-           sa.getAddress(), getContext().getInstanceName(), this.getResourceGroup());
-       MetricsUtil.initializeProducers(this, mm);
-     } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
-         | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
-         | SecurityException e1) {
-       log.error("Error initializing metrics, metrics will not be emitted.", e1);
-     }
+     MetricsInfo metricsInfo = getContext().getMetricsInfo();
 -    metricsInfo.addServiceTags(getApplicationName(), sa.getAddress());
 -
 -    var producers = ManagerMetrics.getProducers(getConfiguration(), this);
++    metricsInfo.addServiceTags(getApplicationName(), sa.getAddress(), getResourceGroup());
++    ManagerMetrics managerMetrics = new ManagerMetrics(getConfiguration(), this);
++    var producers = managerMetrics.getProducers(getConfiguration(), this);
+     metricsInfo.addMetricsProducers(producers.toArray(new MetricsProducer[0]));
+     metricsInfo.init();
  
      recoveryManager = new RecoveryManager(this, timeToCacheRecoveryWalExistence);
  
@@@ -1037,10 -1154,8 +1039,10 @@@
        throw new IllegalStateException("Unable to read " + zroot + Constants.ZRECOVERY, e);
      }
  
 -    watchers.add(new TabletGroupWatcher(this,
 -        TabletStateStore.getStoreForLevel(DataLevel.USER, context, this), null) {
 +    this.splitter = new Splitter(context);
 +    this.splitter.start();
 +
-     watchers.add(new TabletGroupWatcher(this, this.userTabletStore, null, mm) {
++    watchers.add(new TabletGroupWatcher(this, this.userTabletStore, null, managerMetrics) {
        @Override
        boolean canSuspendTablets() {
          // Always allow user data tablets to enter suspended state.
@@@ -1048,24 -1163,26 +1050,26 @@@
        }
      });
  
-     watchers.add(new TabletGroupWatcher(this, this.metadataTabletStore, watchers.get(0), mm) {
 -    watchers.add(new TabletGroupWatcher(this,
 -        TabletStateStore.getStoreForLevel(DataLevel.METADATA, context, this), watchers.get(0)) {
--      @Override
--      boolean canSuspendTablets() {
--        // Allow metadata tablets to enter suspended state only if so configured. Generally
--        // we'll want metadata tablets to
--        // be immediately reassigned, even if there's a global table.suspension.duration
--        // setting.
--        return getConfiguration().getBoolean(Property.MANAGER_METADATA_SUSPENDABLE);
--      }
--    });
--
-     watchers.add(new TabletGroupWatcher(this, this.rootTabletStore, watchers.get(1), mm) {
 -    watchers.add(new TabletGroupWatcher(this,
 -        TabletStateStore.getStoreForLevel(DataLevel.ROOT, context), watchers.get(1)) {
--      @Override
--      boolean canSuspendTablets() {
--        // Never allow root tablet to enter suspended state.
--        return false;
--      }
--    });
++    watchers.add(
++        new TabletGroupWatcher(this, this.metadataTabletStore, watchers.get(0), managerMetrics) {
++          @Override
++          boolean canSuspendTablets() {
++            // Allow metadata tablets to enter suspended state only if so configured. Generally
++            // we'll want metadata tablets to
++            // be immediately reassigned, even if there's a global table.suspension.duration
++            // setting.
++            return getConfiguration().getBoolean(Property.MANAGER_METADATA_SUSPENDABLE);
++          }
++        });
++
++    watchers
++        .add(new TabletGroupWatcher(this, this.rootTabletStore, watchers.get(1), managerMetrics) {
++          @Override
++          boolean canSuspendTablets() {
++            // Never allow root tablet to enter suspended state.
++            return false;
++          }
++        });
      for (TabletGroupWatcher watcher : watchers) {
        watcher.start();
      }
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
index 87f8fc01d8,0000000000..8fcddc530f
mode 100644,000000..100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java
@@@ -1,1051 -1,0 +1,1055 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   https://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.accumulo.manager.compaction.coordinator;
 +
 +import static java.util.concurrent.TimeUnit.SECONDS;
 +import static java.util.stream.Collectors.groupingBy;
 +import static java.util.stream.Collectors.toList;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.COMPACTED;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
 +
 +import java.io.FileNotFoundException;
 +import java.io.IOException;
 +import java.io.UncheckedIOException;
 +import java.time.Duration;
 +import java.util.ArrayList;
 +import java.util.Collection;
 +import java.util.Collections;
 +import java.util.HashMap;
 +import java.util.HashSet;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Optional;
 +import java.util.Set;
 +import java.util.concurrent.ConcurrentHashMap;
 +import java.util.concurrent.CountDownLatch;
 +import java.util.concurrent.ScheduledFuture;
 +import java.util.concurrent.ScheduledThreadPoolExecutor;
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.atomic.AtomicReference;
 +import java.util.function.Consumer;
 +import java.util.stream.Collectors;
 +
 +import org.apache.accumulo.core.Constants;
 +import org.apache.accumulo.core.client.AccumuloSecurityException;
 +import org.apache.accumulo.core.client.TableDeletedException;
 +import org.apache.accumulo.core.client.TableNotFoundException;
 +import org.apache.accumulo.core.client.admin.CompactionConfig;
 +import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
 +import org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode;
 +import org.apache.accumulo.core.clientImpl.thrift.TInfo;
 +import org.apache.accumulo.core.clientImpl.thrift.TableOperation;
 +import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType;
 +import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException;
 +import org.apache.accumulo.core.compaction.thrift.CompactionCoordinatorService;
 +import org.apache.accumulo.core.compaction.thrift.TCompactionState;
 +import org.apache.accumulo.core.compaction.thrift.TCompactionStatusUpdate;
 +import org.apache.accumulo.core.compaction.thrift.TExternalCompaction;
 +import org.apache.accumulo.core.compaction.thrift.TExternalCompactionList;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.NamespaceId;
 +import org.apache.accumulo.core.dataImpl.KeyExtent;
 +import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
 +import org.apache.accumulo.core.fate.Fate;
 +import org.apache.accumulo.core.fate.FateId;
 +import org.apache.accumulo.core.fate.FateInstanceType;
 +import org.apache.accumulo.core.fate.FateKey;
 +import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
 +import org.apache.accumulo.core.iteratorsImpl.system.SystemIteratorUtil;
 +import org.apache.accumulo.core.logging.TabletLogger;
 +import org.apache.accumulo.core.manager.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.CompactableFileImpl;
 +import org.apache.accumulo.core.metadata.ReferencedTabletFile;
 +import org.apache.accumulo.core.metadata.StoredTabletFile;
 +import org.apache.accumulo.core.metadata.schema.Ample;
 +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
 +import org.apache.accumulo.core.metadata.schema.Ample.RejectionHandler;
 +import org.apache.accumulo.core.metadata.schema.CompactionMetadata;
 +import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
 +import org.apache.accumulo.core.metadata.schema.TabletMetadata;
 +import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
 +import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 +import org.apache.accumulo.core.metadata.schema.filters.HasExternalCompactionsFilter;
 +import org.apache.accumulo.core.metrics.MetricsProducer;
 +import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
 +import org.apache.accumulo.core.spi.compaction.CompactionJob;
 +import org.apache.accumulo.core.spi.compaction.CompactionKind;
 +import org.apache.accumulo.core.spi.compaction.CompactorGroupId;
 +import org.apache.accumulo.core.tabletserver.thrift.InputFile;
 +import org.apache.accumulo.core.tabletserver.thrift.IteratorConfig;
 +import org.apache.accumulo.core.tabletserver.thrift.TCompactionKind;
 +import org.apache.accumulo.core.tabletserver.thrift.TCompactionStats;
 +import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob;
 +import org.apache.accumulo.core.util.Retry;
 +import org.apache.accumulo.core.util.UtilWaitThread;
 +import org.apache.accumulo.core.util.cache.Caches.CacheName;
 +import org.apache.accumulo.core.util.compaction.ExternalCompactionUtil;
 +import org.apache.accumulo.core.util.compaction.RunningCompaction;
 +import org.apache.accumulo.core.util.threads.ThreadPools;
 +import org.apache.accumulo.core.util.threads.Threads;
 +import org.apache.accumulo.core.volume.Volume;
 +import org.apache.accumulo.manager.Manager;
 +import org.apache.accumulo.manager.compaction.coordinator.commit.CommitCompaction;
 +import org.apache.accumulo.manager.compaction.coordinator.commit.CompactionCommitData;
 +import org.apache.accumulo.manager.compaction.coordinator.commit.RenameCompactionFile;
 +import org.apache.accumulo.manager.compaction.queue.CompactionJobQueues;
 +import org.apache.accumulo.server.ServerContext;
 +import org.apache.accumulo.server.compaction.CompactionConfigStorage;
 +import org.apache.accumulo.server.compaction.CompactionPluginUtils;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.accumulo.server.tablets.TabletNameGenerator;
 +import org.apache.hadoop.fs.FileStatus;
 +import org.apache.hadoop.fs.FileSystem;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.thrift.TException;
 +import org.apache.zookeeper.KeeperException;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.github.benmanes.caffeine.cache.Cache;
 +import com.github.benmanes.caffeine.cache.CacheLoader;
 +import com.github.benmanes.caffeine.cache.LoadingCache;
 +import com.github.benmanes.caffeine.cache.Weigher;
 +import com.google.common.annotations.VisibleForTesting;
 +import com.google.common.base.Preconditions;
 +import com.google.common.collect.Sets;
 +import com.google.common.net.HostAndPort;
 +
 +import io.micrometer.core.instrument.Gauge;
 +import io.micrometer.core.instrument.MeterRegistry;
 +
 +public class CompactionCoordinator
 +    implements CompactionCoordinatorService.Iface, Runnable, MetricsProducer {
 +
 +  private static final Logger LOG = LoggerFactory.getLogger(CompactionCoordinator.class);
 +
 +  /*
 +   * Map of compactionId to RunningCompactions. This is an informational cache of what external
 +   * compactions may be running. Its possible it may contain external compactions that are not
 +   * actually running. It may not contain compactions that are actually running. The metadata table
 +   * is the most authoritative source of what external compactions are currently running, but it
 +   * does not have the stats that this map has.
 +   */
 +  protected final Map<ExternalCompactionId,RunningCompaction> RUNNING_CACHE =
 +      new ConcurrentHashMap<>();
 +
 +  /* Map of group name to last time compactor called to get a compaction job */
 +  // ELASTICITY_TODO need to clean out groups that are no longer configured..
 +  private final Map<CompactorGroupId,Long> TIME_COMPACTOR_LAST_CHECKED = new ConcurrentHashMap<>();
 +
 +  private final ServerContext ctx;
 +  private final SecurityOperation security;
++  private final String resourceGroupName;
 +  private final CompactionJobQueues jobQueues;
 +  private final AtomicReference<Map<FateInstanceType,Fate<Manager>>> fateInstances;
 +  // Exposed for tests
 +  protected CountDownLatch shutdown = new CountDownLatch(1);
 +
 +  private final ScheduledThreadPoolExecutor schedExecutor;
 +
 +  private final Cache<ExternalCompactionId,RunningCompaction> completed;
 +  private final LoadingCache<FateId,CompactionConfig> compactionConfigCache;
 +  private final Cache<Path,Integer> tabletDirCache;
 +  private final DeadCompactionDetector deadCompactionDetector;
 +
-   private final QueueMetrics queueMetrics;
++  private QueueMetrics queueMetrics;
 +
 +  public CompactionCoordinator(ServerContext ctx, SecurityOperation security,
-       AtomicReference<Map<FateInstanceType,Fate<Manager>>> fateInstances) {
++      AtomicReference<Map<FateInstanceType,Fate<Manager>>> fateInstances,
++      final String resourceGroupName) {
 +    this.ctx = ctx;
 +    this.schedExecutor = this.ctx.getScheduledExecutor();
 +    this.security = security;
++    this.resourceGroupName = resourceGroupName;
 +
 +    this.jobQueues = new CompactionJobQueues(
 +        ctx.getConfiguration().getCount(Property.MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE));
 +
 +    this.queueMetrics = new QueueMetrics(jobQueues);
 +
 +    this.fateInstances = fateInstances;
 +
 +    completed = ctx.getCaches().createNewBuilder(CacheName.COMPACTIONS_COMPLETED, true)
 +        .maximumSize(200).expireAfterWrite(10, TimeUnit.MINUTES).build();
 +
 +    CacheLoader<FateId,CompactionConfig> loader =
 +        fateId -> CompactionConfigStorage.getConfig(ctx, fateId);
 +
 +    // Keep a small short lived cache of compaction config. Compaction config never changes, however
 +    // when a compaction is canceled it is deleted which is why there is a time limit. It does not
 +    // hurt to let a job that was canceled start, it will be canceled later. Caching this immutable
 +    // config will help avoid reading the same data over and over.
 +    compactionConfigCache = ctx.getCaches().createNewBuilder(CacheName.COMPACTION_CONFIGS, true)
 +        .expireAfterWrite(30, SECONDS).maximumSize(100).build(loader);
 +
 +    Weigher<Path,Integer> weigher = (path, count) -> {
 +      return path.toUri().toString().length();
 +    };
 +
 +    tabletDirCache = ctx.getCaches().createNewBuilder(CacheName.COMPACTION_DIR_CACHE, true)
 +        .maximumWeight(10485760L).weigher(weigher).build();
 +
 +    deadCompactionDetector =
 +        new DeadCompactionDetector(this.ctx, this, schedExecutor, fateInstances);
 +    // At this point the manager does not have its lock so no actions should be taken yet
 +  }
 +
 +  private volatile Thread serviceThread = null;
 +
 +  public void start() {
 +    serviceThread = Threads.createThread("CompactionCoordinator Thread", this);
 +    serviceThread.start();
 +  }
 +
 +  public void shutdown() {
 +    shutdown.countDown();
 +    var localThread = serviceThread;
 +    if (localThread != null) {
 +      try {
 +        localThread.join();
 +      } catch (InterruptedException e) {
 +        LOG.error("Exception stopping compaction coordinator thread", e);
 +      }
 +    }
 +  }
 +
 +  protected void startCompactionCleaner(ScheduledThreadPoolExecutor schedExecutor) {
 +    ScheduledFuture<?> future =
 +        schedExecutor.scheduleWithFixedDelay(this::cleanUpCompactors, 0, 5, TimeUnit.MINUTES);
 +    ThreadPools.watchNonCriticalScheduledTask(future);
 +  }
 +
 +  protected void startRunningCleaner(ScheduledThreadPoolExecutor schedExecutor) {
 +    ScheduledFuture<?> future =
 +        schedExecutor.scheduleWithFixedDelay(this::cleanUpRunning, 0, 5, TimeUnit.MINUTES);
 +    ThreadPools.watchNonCriticalScheduledTask(future);
 +  }
 +
 +  protected void startIdleCompactionWatcher() {
 +
 +    ScheduledFuture<?> future = schedExecutor.scheduleWithFixedDelay(this::idleCompactionWarning,
 +        getTServerCheckInterval(), getTServerCheckInterval(), TimeUnit.MILLISECONDS);
 +    ThreadPools.watchNonCriticalScheduledTask(future);
 +  }
 +
 +  private void idleCompactionWarning() {
 +
 +    long now = System.currentTimeMillis();
 +    Map<String,Set<HostAndPort>> idleCompactors = getIdleCompactors();
 +    TIME_COMPACTOR_LAST_CHECKED.forEach((groupName, lastCheckTime) -> {
 +      if ((now - lastCheckTime) > getMissingCompactorWarningTime()
 +          && jobQueues.getQueuedJobs(groupName) > 0
 +          && idleCompactors.containsKey(groupName.canonical())) {
 +        LOG.warn("No compactors have checked in with coordinator for group {} in {}ms", groupName,
 +            getMissingCompactorWarningTime());
 +      }
 +    });
 +
 +  }
 +
 +  @Override
 +  public void run() {
- 
 +    startCompactionCleaner(schedExecutor);
 +    startRunningCleaner(schedExecutor);
 +
 +    // On a re-start of the coordinator it's possible that external compactions are in-progress.
 +    // Attempt to get the running compactions on the compactors and then resolve which tserver
 +    // the external compaction came from to re-populate the RUNNING collection.
 +    LOG.info("Checking for running external compactions");
 +    // On re-start contact the running Compactors to try and seed the list of running compactions
 +    List<RunningCompaction> running = getCompactionsRunningOnCompactors();
 +    if (running.isEmpty()) {
 +      LOG.info("No running external compactions found");
 +    } else {
 +      LOG.info("Found {} running external compactions", running.size());
 +      running.forEach(rc -> {
 +        TCompactionStatusUpdate update = new TCompactionStatusUpdate();
 +        update.setState(TCompactionState.IN_PROGRESS);
 +        update.setMessage("Coordinator restarted, compaction found in progress");
 +        rc.addUpdate(System.currentTimeMillis(), update);
 +        RUNNING_CACHE.put(ExternalCompactionId.of(rc.getJob().getExternalCompactionId()), rc);
 +      });
 +    }
 +
 +    startDeadCompactionDetector();
 +
 +    startIdleCompactionWatcher();
 +
 +    try {
 +      shutdown.await();
 +    } catch (InterruptedException e) {
 +      LOG.warn("Interrupted waiting for shutdown latch.", e);
 +    }
 +
 +    LOG.info("Shutting down");
 +  }
 +
 +  private Map<String,Set<HostAndPort>> getIdleCompactors() {
 +
 +    Map<String,Set<HostAndPort>> allCompactors = new HashMap<>();
 +    ExternalCompactionUtil.getCompactorAddrs(ctx)
 +        .forEach((group, compactorList) -> allCompactors.put(group, new HashSet<>(compactorList)));
 +
 +    Set<String> emptyQueues = new HashSet<>();
 +
 +    // Remove all of the compactors that are running a compaction
 +    RUNNING_CACHE.values().forEach(rc -> {
 +      Set<HostAndPort> busyCompactors = allCompactors.get(rc.getGroupName());
 +      if (busyCompactors != null
 +          && busyCompactors.remove(HostAndPort.fromString(rc.getCompactorAddress()))) {
 +        if (busyCompactors.isEmpty()) {
 +          emptyQueues.add(rc.getGroupName());
 +        }
 +      }
 +    });
 +    // Remove entries with empty queues
 +    emptyQueues.forEach(e -> allCompactors.remove(e));
 +    return allCompactors;
 +  }
 +
 +  protected void startDeadCompactionDetector() {
 +    deadCompactionDetector.start();
 +  }
 +
 +  protected long getMissingCompactorWarningTime() {
 +    return this.ctx.getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME) * 3;
 +  }
 +
 +  protected long getTServerCheckInterval() {
 +    return this.ctx.getConfiguration()
 +        .getTimeInMillis(Property.COMPACTION_COORDINATOR_TSERVER_COMPACTION_CHECK_INTERVAL);
 +  }
 +
 +  public long getNumRunningCompactions() {
 +    return RUNNING_CACHE.size();
 +  }
 +
 +  /**
 +   * Return the next compaction job from the queue to a Compactor
 +   *
 +   * @param groupName group
 +   * @param compactorAddress compactor address
 +   * @throws ThriftSecurityException when permission error
 +   * @return compaction job
 +   */
 +  @Override
 +  public TExternalCompactionJob getCompactionJob(TInfo tinfo, TCredentials credentials,
 +      String groupName, String compactorAddress, String externalCompactionId)
 +      throws ThriftSecurityException {
 +
 +    // do not expect users to call this directly, expect compactors to call this method
 +    if (!security.canPerformSystemActions(credentials)) {
 +      throw new AccumuloSecurityException(credentials.getPrincipal(),
 +          SecurityErrorCode.PERMISSION_DENIED).asThriftException();
 +    }
 +    CompactorGroupId groupId = CompactorGroupId.of(groupName);
 +    LOG.trace("getCompactionJob called for group {} by compactor {}", groupId, compactorAddress);
 +    TIME_COMPACTOR_LAST_CHECKED.put(groupId, System.currentTimeMillis());
 +
 +    TExternalCompactionJob result = null;
 +
 +    CompactionJobQueues.MetaJob metaJob = jobQueues.poll(groupId);
 +
 +    while (metaJob != null) {
 +
 +      Optional<CompactionConfig> compactionConfig = getCompactionConfig(metaJob);
 +
 +      // this method may reread the metadata, do not use the metadata in metaJob for anything after
 +      // this method
 +      CompactionMetadata ecm = null;
 +
 +      var kind = metaJob.getJob().getKind();
 +
 +      // Only reserve user compactions when the config is present. When compactions are canceled the
 +      // config is deleted.
 +      var cid = ExternalCompactionId.from(externalCompactionId);
 +      if (kind == CompactionKind.SYSTEM
 +          || (kind == CompactionKind.USER && compactionConfig.isPresent())) {
 +        ecm = reserveCompaction(metaJob, compactorAddress, cid);
 +      }
 +
 +      if (ecm != null) {
 +        result = createThriftJob(externalCompactionId, ecm, metaJob, compactionConfig);
 +        // It is possible that by the time this added that the the compactor that made this request
 +        // is dead. In this cases the compaction is not actually running.
 +        RUNNING_CACHE.put(ExternalCompactionId.of(result.getExternalCompactionId()),
 +            new RunningCompaction(result, compactorAddress, groupName));
 +        TabletLogger.compacting(metaJob.getTabletMetadata(), cid, compactorAddress,
 +            metaJob.getJob());
 +        break;
 +      } else {
 +        LOG.debug(
 +            "Unable to reserve compaction job for {}, pulling another off the queue for group {}",
 +            metaJob.getTabletMetadata().getExtent(), groupName);
 +        metaJob = jobQueues.poll(CompactorGroupId.of(groupName));
 +      }
 +    }
 +
 +    if (metaJob == null) {
 +      LOG.debug("No jobs found in group {} ", groupName);
 +    }
 +
 +    if (result == null) {
 +      LOG.trace("No jobs found for group {}, returning empty job to compactor {}", groupName,
 +          compactorAddress);
 +      result = new TExternalCompactionJob();
 +    }
 +
 +    return result;
 +
 +  }
 +
 +  @VisibleForTesting
 +  public static boolean canReserveCompaction(TabletMetadata tablet, CompactionKind kind,
 +      Set<StoredTabletFile> jobFiles, ServerContext ctx) {
 +
 +    if (tablet == null) {
 +      // the tablet no longer exist
 +      return false;
 +    }
 +
 +    if (tablet.getOperationId() != null) {
 +      return false;
 +    }
 +
 +    if (ctx.getTableState(tablet.getTableId()) != TableState.ONLINE) {
 +      return false;
 +    }
 +
 +    if (!tablet.getFiles().containsAll(jobFiles)) {
 +      return false;
 +    }
 +
 +    var currentlyCompactingFiles = tablet.getExternalCompactions().values().stream()
 +        .flatMap(ecm -> ecm.getJobFiles().stream()).collect(Collectors.toSet());
 +
 +    if (!Collections.disjoint(jobFiles, currentlyCompactingFiles)) {
 +      return false;
 +    }
 +
 +    switch (kind) {
 +      case SYSTEM:
 +        var userRequestedCompactions = tablet.getUserCompactionsRequested().size();
 +        if (userRequestedCompactions > 0) {
 +          LOG.debug(
 +              "Unable to reserve {} for system compaction, tablet has {} pending requested user compactions",
 +              tablet.getExtent(), userRequestedCompactions);
 +          return false;
 +        } else if (tablet.getSelectedFiles() != null
 +            && !Collections.disjoint(jobFiles, tablet.getSelectedFiles().getFiles())) {
 +          return false;
 +        }
 +        break;
 +      case USER:
 +        if (tablet.getSelectedFiles() == null
 +            || !tablet.getSelectedFiles().getFiles().containsAll(jobFiles)) {
 +          return false;
 +        }
 +        break;
 +      default:
 +        throw new UnsupportedOperationException("Not currently handling " + kind);
 +    }
 +
 +    return true;
 +  }
 +
 +  private void checkTabletDir(KeyExtent extent, Path path) {
 +    try {
 +      if (tabletDirCache.getIfPresent(path) == null) {
 +        FileStatus[] files = null;
 +        try {
 +          files = ctx.getVolumeManager().listStatus(path);
 +        } catch (FileNotFoundException ex) {
 +          // ignored
 +        }
 +
 +        if (files == null) {
 +          LOG.debug("Tablet {} had no dir, creating {}", extent, path);
 +
 +          ctx.getVolumeManager().mkdirs(path);
 +        }
 +        tabletDirCache.put(path, 1);
 +      }
 +    } catch (IOException e) {
 +      throw new UncheckedIOException(e);
 +    }
 +  }
 +
 +  protected CompactionMetadata createExternalCompactionMetadata(CompactionJob job,
 +      Set<StoredTabletFile> jobFiles, TabletMetadata tablet, String compactorAddress,
 +      ExternalCompactionId externalCompactionId) {
 +    boolean propDels;
 +
 +    FateId fateId = null;
 +
 +    switch (job.getKind()) {
 +      case SYSTEM: {
 +        boolean compactingAll = tablet.getFiles().equals(jobFiles);
 +        propDels = !compactingAll;
 +      }
 +        break;
 +      case USER: {
 +        boolean compactingAll = tablet.getSelectedFiles().initiallySelectedAll()
 +            && tablet.getSelectedFiles().getFiles().equals(jobFiles);
 +        propDels = !compactingAll;
 +        fateId = tablet.getSelectedFiles().getFateId();
 +      }
 +        break;
 +      default:
 +        throw new IllegalArgumentException();
 +    }
 +
 +    Consumer<String> directoryCreator = dir -> checkTabletDir(tablet.getExtent(), new Path(dir));
 +    ReferencedTabletFile newFile = TabletNameGenerator.getNextDataFilenameForMajc(propDels, ctx,
 +        tablet, directoryCreator, externalCompactionId);
 +
 +    return new CompactionMetadata(jobFiles, newFile, compactorAddress, job.getKind(),
 +        job.getPriority(), job.getGroup(), propDels, fateId);
 +
 +  }
 +
 +  protected CompactionMetadata reserveCompaction(CompactionJobQueues.MetaJob metaJob,
 +      String compactorAddress, ExternalCompactionId externalCompactionId) {
 +
 +    Preconditions.checkArgument(metaJob.getJob().getKind() == CompactionKind.SYSTEM
 +        || metaJob.getJob().getKind() == CompactionKind.USER);
 +
 +    var tabletMetadata = metaJob.getTabletMetadata();
 +
 +    var jobFiles = metaJob.getJob().getFiles().stream().map(CompactableFileImpl::toStoredTabletFile)
 +        .collect(Collectors.toSet());
 +
 +    Retry retry = Retry.builder().maxRetries(5).retryAfter(Duration.ofMillis(100))
 +        .incrementBy(Duration.ofMillis(100)).maxWait(Duration.ofSeconds(10)).backOffFactor(1.5)
 +        .logInterval(Duration.ofMinutes(3)).createRetry();
 +
 +    while (retry.canRetry()) {
 +      try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) {
 +        var extent = metaJob.getTabletMetadata().getExtent();
 +
 +        if (!canReserveCompaction(tabletMetadata, metaJob.getJob().getKind(), jobFiles, ctx)) {
 +          return null;
 +        }
 +
 +        var ecm = createExternalCompactionMetadata(metaJob.getJob(), jobFiles, tabletMetadata,
 +            compactorAddress, externalCompactionId);
 +
 +        // any data that is read from the tablet to make a decision about if it can compact or not
 +        // must be included in the requireSame call
 +        var tabletMutator = tabletsMutator.mutateTablet(extent).requireAbsentOperation()
 +            .requireSame(tabletMetadata, FILES, SELECTED, ECOMP);
 +
 +        tabletMutator.putExternalCompaction(externalCompactionId, ecm);
 +        tabletMutator.submit(tm -> tm.getExternalCompactions().containsKey(externalCompactionId));
 +
 +        var result = tabletsMutator.process().get(extent);
 +
 +        if (result.getStatus() == Ample.ConditionalResult.Status.ACCEPTED) {
 +          return ecm;
 +        } else {
 +          tabletMetadata = result.readMetadata();
 +        }
 +      }
 +
 +      retry.useRetry();
 +      try {
 +        retry.waitForNextAttempt(LOG,
 +            "Reserved compaction for " + metaJob.getTabletMetadata().getExtent());
 +      } catch (InterruptedException e) {
 +        throw new RuntimeException(e);
 +      }
 +    }
 +
 +    return null;
 +  }
 +
 +  protected TExternalCompactionJob createThriftJob(String externalCompactionId,
 +      CompactionMetadata ecm, CompactionJobQueues.MetaJob metaJob,
 +      Optional<CompactionConfig> compactionConfig) {
 +
 +    Set<CompactableFile> selectedFiles;
 +    if (metaJob.getJob().getKind() == CompactionKind.SYSTEM) {
 +      selectedFiles = Set.of();
 +    } else {
 +      selectedFiles = metaJob.getTabletMetadata().getSelectedFiles().getFiles().stream()
 +          .map(file -> new CompactableFileImpl(file,
 +              metaJob.getTabletMetadata().getFilesMap().get(file)))
 +          .collect(Collectors.toUnmodifiableSet());
 +    }
 +
 +    Map<String,String> overrides = CompactionPluginUtils.computeOverrides(compactionConfig, ctx,
 +        metaJob.getTabletMetadata().getExtent(), metaJob.getJob().getFiles(), selectedFiles);
 +
 +    IteratorConfig iteratorSettings = SystemIteratorUtil
 +        .toIteratorConfig(compactionConfig.map(CompactionConfig::getIterators).orElse(List.of()));
 +
 +    var files = ecm.getJobFiles().stream().map(storedTabletFile -> {
 +      var dfv = metaJob.getTabletMetadata().getFilesMap().get(storedTabletFile);
 +      return new InputFile(storedTabletFile.getMetadata(), dfv.getSize(), dfv.getNumEntries(),
 +          dfv.getTime());
 +    }).collect(toList());
 +
 +    // The fateId here corresponds to the Fate transaction that is driving a user initiated
 +    // compaction. A system initiated compaction has no Fate transaction driving it so its ok to set
 +    // it to null. If anything tries to use the id for a system compaction and triggers a NPE it's
 +    // probably a bug that needs to be fixed.
 +    FateId fateId = null;
 +    if (metaJob.getJob().getKind() == CompactionKind.USER) {
 +      fateId = metaJob.getTabletMetadata().getSelectedFiles().getFateId();
 +    }
 +
 +    return new TExternalCompactionJob(externalCompactionId,
 +        metaJob.getTabletMetadata().getExtent().toThrift(), files, iteratorSettings,
 +        ecm.getCompactTmpName().getNormalizedPathStr(), ecm.getPropagateDeletes(),
 +        TCompactionKind.valueOf(ecm.getKind().name()), fateId == null ? null : fateId.toThrift(),
 +        overrides);
 +  }
 +
 +  @Override
 +  public void registerMetrics(MeterRegistry registry) {
 +    Gauge.builder(METRICS_MAJC_QUEUED, jobQueues, CompactionJobQueues::getQueuedJobCount)
++        .tag("subprocess", "compaction.coordinator")
 +        .description("Number of queued major compactions").register(registry);
 +    Gauge.builder(METRICS_MAJC_RUNNING, this, CompactionCoordinator::getNumRunningCompactions)
++        .tag("subprocess", "compaction.coordinator")
 +        .description("Number of running major compactions").register(registry);
 +
 +    queueMetrics.registerMetrics(registry);
 +  }
 +
 +  public void addJobs(TabletMetadata tabletMetadata, Collection<CompactionJob> jobs) {
 +    jobQueues.add(tabletMetadata, jobs);
 +  }
 +
 +  public CompactionCoordinatorService.Iface getThriftService() {
 +    return this;
 +  }
 +
 +  private Optional<CompactionConfig> getCompactionConfig(CompactionJobQueues.MetaJob metaJob) {
 +    if (metaJob.getJob().getKind() == CompactionKind.USER
 +        && metaJob.getTabletMetadata().getSelectedFiles() != null) {
 +      var cconf =
 +          compactionConfigCache.get(metaJob.getTabletMetadata().getSelectedFiles().getFateId());
 +      return Optional.ofNullable(cconf);
 +    }
 +    return Optional.empty();
 +  }
 +
 +  /**
 +   * Compactors calls this method when they have finished a compaction. This method does the
 +   * following.
 +   *
 +   * <ol>
 +   * <li>Reads the tablets metadata and determines if the compaction can commit. Its possible that
 +   * things changed while the compaction was running and it can no longer commit.</li>
 +   * <li>Commit the compaction using a conditional mutation. If the tablets files or location
 +   * changed since reading the tablets metadata, then conditional mutation will fail. When this
 +   * happens it will reread the metadata and go back to step 1 conceptually. When committing a
 +   * compaction the compacted files are removed and scan entries are added to the tablet in case the
 +   * files are in use, this prevents GC from deleting the files between updating tablet metadata and
 +   * refreshing the tablet. The scan entries are only added when a tablet has a location.</li>
 +   * <li>After successful commit a refresh request is sent to the tablet if it has a location. This
 +   * will cause the tablet to start using the newly compacted files for future scans. Also the
 +   * tablet can delete the scan entries if there are no active scans using them.</li>
 +   * </ol>
 +   *
 +   * <p>
 +   * User compactions will be refreshed as part of the fate operation. The user compaction fate
 +   * operation will see the compaction was committed after this code updates the tablet metadata,
 +   * however if it were to rely on this code to do the refresh it would not be able to know when the
 +   * refresh was actually done. Therefore, user compactions will refresh as part of the fate
 +   * operation so that it's known to be done before the fate operation returns. Since the fate
 +   * operation will do it, there is no need to do it here for user compactions.
 +   *
 +   * @param tinfo trace info
 +   * @param credentials tcredentials object
 +   * @param externalCompactionId compaction id
 +   * @param textent tablet extent
 +   * @param stats compaction stats
 +   * @throws ThriftSecurityException when permission error
 +   */
 +  @Override
 +  public void compactionCompleted(TInfo tinfo, TCredentials credentials,
 +      String externalCompactionId, TKeyExtent textent, TCompactionStats stats)
 +      throws ThriftSecurityException {
 +    // do not expect users to call this directly, expect other tservers to call this method
 +    if (!security.canPerformSystemActions(credentials)) {
 +      throw new AccumuloSecurityException(credentials.getPrincipal(),
 +          SecurityErrorCode.PERMISSION_DENIED).asThriftException();
 +    }
 +
 +    // maybe fate has not started yet
 +    var localFates = fateInstances.get();
 +    while (localFates == null) {
 +      UtilWaitThread.sleep(100);
 +      if (shutdown.getCount() == 0) {
 +        return;
 +      }
 +      localFates = fateInstances.get();
 +    }
 +
 +    var extent = KeyExtent.fromThrift(textent);
 +    var localFate = localFates.get(FateInstanceType.fromTableId(extent.tableId()));
 +
 +    LOG.info("Compaction completed, id: {}, stats: {}, extent: {}", externalCompactionId, stats,
 +        extent);
 +    final var ecid = ExternalCompactionId.of(externalCompactionId);
 +
 +    var tabletMeta =
 +        ctx.getAmple().readTablet(extent, ECOMP, SELECTED, LOCATION, FILES, COMPACTED, OPID);
 +
 +    if (!CommitCompaction.canCommitCompaction(ecid, tabletMeta)) {
 +      return;
 +    }
 +
 +    // Start a fate transaction to commit the compaction.
 +    CompactionMetadata ecm = tabletMeta.getExternalCompactions().get(ecid);
 +    var renameOp = new RenameCompactionFile(new CompactionCommitData(ecid, extent, ecm, stats));
 +    var txid = localFate.seedTransaction("COMMIT_COMPACTION", FateKey.forCompactionCommit(ecid),
 +        renameOp, true, "Commit compaction " + ecid);
 +
 +    txid.ifPresentOrElse(fateId -> LOG.debug("initiated compaction commit {} {}", ecid, fateId),
 +        () -> LOG.debug("compaction commit already initiated for {}", ecid));
 +  }
 +
 +  @Override
 +  public void compactionFailed(TInfo tinfo, TCredentials credentials, String externalCompactionId,
 +      TKeyExtent extent) throws ThriftSecurityException {
 +    // do not expect users to call this directly, expect other tservers to call this method
 +    if (!security.canPerformSystemActions(credentials)) {
 +      throw new AccumuloSecurityException(credentials.getPrincipal(),
 +          SecurityErrorCode.PERMISSION_DENIED).asThriftException();
 +    }
 +    KeyExtent fromThriftExtent = KeyExtent.fromThrift(extent);
 +    LOG.info("Compaction failed, id: {}, extent: {}", externalCompactionId, fromThriftExtent);
 +    final var ecid = ExternalCompactionId.of(externalCompactionId);
 +    compactionsFailed(Map.of(ecid, KeyExtent.fromThrift(extent)));
 +  }
 +
 +  void compactionsFailed(Map<ExternalCompactionId,KeyExtent> compactions) {
 +    // Need to process each level by itself because the conditional tablet mutator does not support
 +    // mutating multiple data levels at the same time
 +    compactions.entrySet().stream()
 +        .collect(groupingBy(entry -> DataLevel.of(entry.getValue().tableId()),
 +            Collectors.toMap(Entry::getKey, Entry::getValue)))
 +        .forEach((level, compactionsByLevel) -> compactionFailedForLevel(compactionsByLevel));
 +  }
 +
 +  void compactionFailedForLevel(Map<ExternalCompactionId,KeyExtent> compactions) {
 +
 +    try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) {
 +      compactions.forEach((ecid, extent) -> {
 +        try {
 +          ctx.requireNotDeleted(extent.tableId());
 +          tabletsMutator.mutateTablet(extent).requireAbsentOperation().requireCompaction(ecid)
 +              .deleteExternalCompaction(ecid).submit(new RejectionHandler() {
 +
 +                @Override
 +                public boolean callWhenTabletDoesNotExists() {
 +                  return true;
 +                }
 +
 +                @Override
 +                public boolean test(TabletMetadata tabletMetadata) {
 +                  return tabletMetadata == null
 +                      || !tabletMetadata.getExternalCompactions().containsKey(ecid);
 +                }
 +
 +              });
 +        } catch (TableDeletedException e) {
 +          LOG.warn("Table {} was deleted, unable to update metadata for compaction failure.",
 +              extent.tableId());
 +        }
 +      });
 +
 +      final List<ExternalCompactionId> ecidsForTablet = new ArrayList<>();
 +      tabletsMutator.process().forEach((extent, result) -> {
 +        if (result.getStatus() != Ample.ConditionalResult.Status.ACCEPTED) {
 +
 +          // this should try again later when the dead compaction detector runs, lets log it in case
 +          // its a persistent problem
 +          if (LOG.isDebugEnabled()) {
 +            var ecid =
 +                compactions.entrySet().stream().filter(entry -> entry.getValue().equals(extent))
 +                    .findFirst().map(Map.Entry::getKey).orElse(null);
 +            LOG.debug("Unable to remove failed compaction {} {}", extent, ecid);
 +          }
 +        } else {
 +          // compactionFailed is called from the Compactor when either a compaction fails or
 +          // is cancelled and it's called from the DeadCompactionDetector. This block is
 +          // entered when the conditional mutator above successfully deletes an ecid from
 +          // the tablet metadata. Remove compaction tmp files from the tablet directory
 +          // that have a corresponding ecid in the name.
 +
 +          ecidsForTablet.clear();
 +          compactions.entrySet().stream().filter(e -> e.getValue().compareTo(extent) == 0)
 +              .map(Entry::getKey).forEach(ecidsForTablet::add);
 +
 +          if (!ecidsForTablet.isEmpty()) {
 +            final TabletMetadata tm = ctx.getAmple().readTablet(extent, ColumnType.DIR);
 +            if (tm != null) {
 +              final Collection<Volume> vols = ctx.getVolumeManager().getVolumes();
 +              for (Volume vol : vols) {
 +                try {
 +                  final String volPath =
 +                      vol.getBasePath() + Constants.HDFS_TABLES_DIR + Path.SEPARATOR
 +                          + extent.tableId().canonical() + Path.SEPARATOR + tm.getDirName();
 +                  final FileSystem fs = vol.getFileSystem();
 +                  for (ExternalCompactionId ecid : ecidsForTablet) {
 +                    final String fileSuffix = "_tmp_" + ecid.canonical();
 +                    FileStatus[] files = fs.listStatus(new Path(volPath), (path) -> {
 +                      return path.getName().endsWith(fileSuffix);
 +                    });
 +                    if (files.length > 0) {
 +                      for (FileStatus file : files) {
 +                        if (!fs.delete(file.getPath(), false)) {
 +                          LOG.warn("Unable to delete ecid tmp file: {}: ", file.getPath());
 +                        } else {
 +                          LOG.debug("Deleted ecid tmp file: {}", file.getPath());
 +                        }
 +                      }
 +                    }
 +                  }
 +                } catch (IOException e) {
 +                  LOG.error("Exception deleting compaction tmp files for tablet: {}", extent, e);
 +                }
 +              }
 +            } else {
 +              // TabletMetadata does not exist for the extent. This could be due to a merge or
 +              // split operation. Use the utility to find tmp files at the table level
 +              deadCompactionDetector.addTableId(extent.tableId());
 +            }
 +          }
 +        }
 +      });
 +    }
 +
 +    compactions.forEach((k, v) -> recordCompletion(k));
 +  }
 +
 +  /**
 +   * Compactor calls to update the status of the assigned compaction
 +   *
 +   * @param tinfo trace info
 +   * @param credentials tcredentials object
 +   * @param externalCompactionId compaction id
 +   * @param update compaction status update
 +   * @param timestamp timestamp of the message
 +   * @throws ThriftSecurityException when permission error
 +   */
 +  @Override
 +  public void updateCompactionStatus(TInfo tinfo, TCredentials credentials,
 +      String externalCompactionId, TCompactionStatusUpdate update, long timestamp)
 +      throws ThriftSecurityException {
 +    // do not expect users to call this directly, expect other tservers to call this method
 +    if (!security.canPerformSystemActions(credentials)) {
 +      throw new AccumuloSecurityException(credentials.getPrincipal(),
 +          SecurityErrorCode.PERMISSION_DENIED).asThriftException();
 +    }
 +    LOG.debug("Compaction status update, id: {}, timestamp: {}, update: {}", externalCompactionId,
 +        timestamp, update);
 +    final RunningCompaction rc = RUNNING_CACHE.get(ExternalCompactionId.of(externalCompactionId));
 +    if (null != rc) {
 +      rc.addUpdate(timestamp, update);
 +    }
 +  }
 +
 +  public void recordCompletion(ExternalCompactionId ecid) {
 +    var rc = RUNNING_CACHE.remove(ecid);
 +    if (rc != null) {
 +      completed.put(ecid, rc);
 +    }
 +  }
 +
 +  protected Set<ExternalCompactionId> readExternalCompactionIds() {
 +    try (TabletsMetadata tabletsMetadata =
 +        this.ctx.getAmple().readTablets().forLevel(Ample.DataLevel.USER)
 +            .filter(new HasExternalCompactionsFilter()).fetch(ECOMP).build()) {
 +      return tabletsMetadata.stream().flatMap(tm -> tm.getExternalCompactions().keySet().stream())
 +          .collect(Collectors.toSet());
 +    }
 +  }
 +
 +  /**
 +   * The RUNNING_CACHE set may contain external compactions that are not actually running. This
 +   * method periodically cleans those up.
 +   */
 +  public void cleanUpRunning() {
 +
 +    // grab a snapshot of the ids in the set before reading the metadata table. This is done to
 +    // avoid removing things that are added while reading the metadata.
 +    Set<ExternalCompactionId> idsSnapshot = Set.copyOf(RUNNING_CACHE.keySet());
 +
 +    // grab the ids that are listed as running in the metadata table. It important that this is done
 +    // after getting the snapshot.
 +    Set<ExternalCompactionId> idsInMetadata = readExternalCompactionIds();
 +
 +    var idsToRemove = Sets.difference(idsSnapshot, idsInMetadata);
 +
 +    // remove ids that are in the running set but not in the metadata table
 +    idsToRemove.forEach(this::recordCompletion);
 +
 +    if (idsToRemove.size() > 0) {
 +      LOG.debug("Removed stale entries from RUNNING_CACHE : {}", idsToRemove);
 +    }
 +  }
 +
 +  /**
 +   * Return information about running compactions
 +   *
 +   * @param tinfo trace info
 +   * @param credentials tcredentials object
 +   * @return map of ECID to TExternalCompaction objects
 +   * @throws ThriftSecurityException permission error
 +   */
 +  @Override
 +  public TExternalCompactionList getRunningCompactions(TInfo tinfo, TCredentials credentials)
 +      throws ThriftSecurityException {
 +    // do not expect users to call this directly, expect other tservers to call this method
 +    if (!security.canPerformSystemActions(credentials)) {
 +      throw new AccumuloSecurityException(credentials.getPrincipal(),
 +          SecurityErrorCode.PERMISSION_DENIED).asThriftException();
 +    }
 +
 +    final TExternalCompactionList result = new TExternalCompactionList();
 +    RUNNING_CACHE.forEach((ecid, rc) -> {
 +      TExternalCompaction trc = new TExternalCompaction();
 +      trc.setGroupName(rc.getGroupName());
 +      trc.setCompactor(rc.getCompactorAddress());
 +      trc.setUpdates(rc.getUpdates());
 +      trc.setJob(rc.getJob());
 +      result.putToCompactions(ecid.canonical(), trc);
 +    });
 +    return result;
 +  }
 +
 +  /**
 +   * Return information about recently completed compactions
 +   *
 +   * @param tinfo trace info
 +   * @param credentials tcredentials object
 +   * @return map of ECID to TExternalCompaction objects
 +   * @throws ThriftSecurityException permission error
 +   */
 +  @Override
 +  public TExternalCompactionList getCompletedCompactions(TInfo tinfo, TCredentials credentials)
 +      throws ThriftSecurityException {
 +    // do not expect users to call this directly, expect other tservers to call this method
 +    if (!security.canPerformSystemActions(credentials)) {
 +      throw new AccumuloSecurityException(credentials.getPrincipal(),
 +          SecurityErrorCode.PERMISSION_DENIED).asThriftException();
 +    }
 +    final TExternalCompactionList result = new TExternalCompactionList();
 +    completed.asMap().forEach((ecid, rc) -> {
 +      TExternalCompaction trc = new TExternalCompaction();
 +      trc.setGroupName(rc.getGroupName());
 +      trc.setCompactor(rc.getCompactorAddress());
 +      trc.setJob(rc.getJob());
 +      trc.setUpdates(rc.getUpdates());
 +      result.putToCompactions(ecid.canonical(), trc);
 +    });
 +    return result;
 +  }
 +
 +  @Override
 +  public void cancel(TInfo tinfo, TCredentials credentials, String externalCompactionId)
 +      throws TException {
 +    var runningCompaction = RUNNING_CACHE.get(ExternalCompactionId.of(externalCompactionId));
 +    var extent = KeyExtent.fromThrift(runningCompaction.getJob().getExtent());
 +    try {
 +      NamespaceId nsId = this.ctx.getNamespaceId(extent.tableId());
 +      if (!security.canCompact(credentials, extent.tableId(), nsId)) {
 +        throw new AccumuloSecurityException(credentials.getPrincipal(),
 +            SecurityErrorCode.PERMISSION_DENIED).asThriftException();
 +      }
 +    } catch (TableNotFoundException e) {
 +      throw new ThriftTableOperationException(extent.tableId().canonical(), null,
 +          TableOperation.COMPACT_CANCEL, TableOperationExceptionType.NOTFOUND, e.getMessage());
 +    }
 +
 +    cancelCompactionOnCompactor(runningCompaction.getCompactorAddress(), externalCompactionId);
 +  }
 +
 +  /* Method exists to be called from test */
 +  public CompactionJobQueues getJobQueues() {
 +    return jobQueues;
 +  }
 +
 +  /* Method exists to be overridden in test to hide static method */
 +  protected List<RunningCompaction> getCompactionsRunningOnCompactors() {
 +    return ExternalCompactionUtil.getCompactionsRunningOnCompactors(this.ctx);
 +  }
 +
 +  /* Method exists to be overridden in test to hide static method */
 +  protected void cancelCompactionOnCompactor(String address, String externalCompactionId) {
 +    HostAndPort hostPort = HostAndPort.fromString(address);
 +    ExternalCompactionUtil.cancelCompaction(this.ctx, hostPort, externalCompactionId);
 +  }
 +
 +  private void deleteEmpty(ZooReaderWriter zoorw, String path)
 +      throws KeeperException, InterruptedException {
 +    try {
 +      LOG.debug("Deleting empty ZK node {}", path);
 +      zoorw.delete(path);
 +    } catch (KeeperException.NotEmptyException e) {
 +      LOG.debug("Failed to delete {} its not empty, likely an expected race condition.", path);
 +    }
 +  }
 +
 +  private void cleanUpCompactors() {
 +    final String compactorQueuesPath = this.ctx.getZooKeeperRoot() + Constants.ZCOMPACTORS;
 +
 +    var zoorw = this.ctx.getZooReaderWriter();
 +
 +    try {
 +      var groups = zoorw.getChildren(compactorQueuesPath);
 +
 +      for (String group : groups) {
 +        String qpath = compactorQueuesPath + "/" + group;
 +
 +        var compactors = zoorw.getChildren(qpath);
 +
 +        if (compactors.isEmpty()) {
 +          deleteEmpty(zoorw, qpath);
 +        }
 +
 +        for (String compactor : compactors) {
 +          String cpath = compactorQueuesPath + "/" + group + "/" + compactor;
 +          var lockNodes = zoorw.getChildren(compactorQueuesPath + "/" + group + "/" + compactor);
 +          if (lockNodes.isEmpty()) {
 +            deleteEmpty(zoorw, cpath);
 +          }
 +        }
 +      }
 +
 +    } catch (KeeperException | RuntimeException e) {
 +      LOG.warn("Failed to clean up compactors", e);
 +    } catch (InterruptedException e) {
 +      Thread.currentThread().interrupt();
 +      throw new IllegalStateException(e);
 +    }
 +  }
 +
 +}
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/QueueMetrics.java
index 425bdf4937,0000000000..da750e552b
mode 100644,000000..100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/QueueMetrics.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/QueueMetrics.java
@@@ -1,155 -1,0 +1,155 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   https://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.accumulo.manager.compaction.coordinator;
 +
 +import static org.apache.accumulo.core.metrics.MetricsUtil.formatString;
- import static org.apache.accumulo.core.metrics.MetricsUtil.getCommonTags;
 +
 +import java.util.HashMap;
 +import java.util.HashSet;
++import java.util.List;
 +import java.util.Map;
 +import java.util.Set;
 +import java.util.concurrent.ScheduledExecutorService;
 +import java.util.concurrent.TimeUnit;
 +
 +import org.apache.accumulo.core.metrics.MetricsProducer;
 +import org.apache.accumulo.core.spi.compaction.CompactorGroupId;
 +import org.apache.accumulo.core.util.threads.ThreadPools;
 +import org.apache.accumulo.manager.compaction.queue.CompactionJobPriorityQueue;
 +import org.apache.accumulo.manager.compaction.queue.CompactionJobQueues;
 +import org.slf4j.Logger;
 +import org.slf4j.LoggerFactory;
 +
 +import com.google.common.collect.Sets;
 +import com.google.common.collect.Sets.SetView;
 +
 +import io.micrometer.core.instrument.Gauge;
 +import io.micrometer.core.instrument.MeterRegistry;
- import io.micrometer.core.instrument.Tags;
++import io.micrometer.core.instrument.Tag;
 +
 +public class QueueMetrics implements MetricsProducer {
 +
 +  private static class QueueMeters {
 +    private final Gauge length;
 +    private final Gauge jobsQueued;
 +    private final Gauge jobsDequeued;
 +    private final Gauge jobsRejected;
 +    private final Gauge jobsLowestPriority;
 +
 +    public QueueMeters(MeterRegistry meterRegistry, CompactorGroupId cgid,
 +        CompactionJobPriorityQueue queue) {
 +      var queueId = formatString(cgid.canonical());
 +
 +      length =
 +          Gauge.builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH, queue, q -> q.getMaxSize())
-               .description("Length of priority queues")
-               .tags(Tags.concat(getCommonTags(), "queue.id", queueId)).register(meterRegistry);
++              .description("Length of priority queues").tags(List.of(Tag.of("queue.id", queueId)))
++              .register(meterRegistry);
 +
 +      jobsQueued = Gauge
 +          .builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED, queue, q -> q.getQueuedJobs())
-           .description("Count of queued jobs")
-           .tags(Tags.concat(getCommonTags(), "queue.id", queueId)).register(meterRegistry);
++          .description("Count of queued jobs").tags(List.of(Tag.of("queue.id", queueId)))
++          .register(meterRegistry);
 +
 +      jobsDequeued = Gauge
 +          .builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED, queue,
 +              q -> q.getDequeuedJobs())
-           .description("Count of jobs dequeued")
-           .tags(Tags.concat(getCommonTags(), "queue.id", queueId)).register(meterRegistry);
++          .description("Count of jobs dequeued").tags(List.of(Tag.of("queue.id", queueId)))
++          .register(meterRegistry);
 +
 +      jobsRejected = Gauge
 +          .builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED, queue,
 +              q -> q.getRejectedJobs())
-           .description("Count of rejected jobs")
-           .tags(Tags.concat(getCommonTags(), "queue.id", queueId)).register(meterRegistry);
++          .description("Count of rejected jobs").tags(List.of(Tag.of("queue.id", queueId)))
++          .register(meterRegistry);
 +
 +      jobsLowestPriority = Gauge
 +          .builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY, queue,
 +              q -> q.getLowestPriority())
-           .description("Lowest priority queued job")
-           .tags(Tags.concat(getCommonTags(), "queue.id", queueId)).register(meterRegistry);
++          .description("Lowest priority queued job").tags(List.of(Tag.of("queue.id", queueId)))
++          .register(meterRegistry);
 +    }
 +
 +    private void removeMeters(MeterRegistry registry) {
 +      registry.remove(length);
 +      registry.remove(jobsQueued);
 +      registry.remove(jobsDequeued);
 +      registry.remove(jobsRejected);
 +      registry.remove(jobsLowestPriority);
 +    }
 +  }
 +
 +  private static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.class);
 +  private static final long DEFAULT_MIN_REFRESH_DELAY = TimeUnit.SECONDS.toMillis(5);
 +  private volatile MeterRegistry meterRegistry = null;
 +  private final CompactionJobQueues compactionJobQueues;
 +  private final Map<CompactorGroupId,QueueMeters> perQueueMetrics = new HashMap<>();
 +  private Gauge queueCountMeter = null;
 +
 +  public QueueMetrics(CompactionJobQueues compactionJobQueues) {
 +    this.compactionJobQueues = compactionJobQueues;
 +    ScheduledExecutorService scheduler =
 +        ThreadPools.getServerThreadPools().createScheduledExecutorService(1, "queueMetricsPoller");
 +    Runtime.getRuntime().addShutdownHook(new Thread(scheduler::shutdownNow));
 +    ThreadPools.watchNonCriticalScheduledTask(scheduler.scheduleAtFixedRate(this::update,
 +        DEFAULT_MIN_REFRESH_DELAY, DEFAULT_MIN_REFRESH_DELAY, TimeUnit.MILLISECONDS));
 +  }
 +
 +  public void update() {
 +
 +    // read the volatile variable once so the rest of the method has consistent view
 +    var localRegistry = meterRegistry;
 +
 +    if (queueCountMeter == null) {
 +      queueCountMeter = Gauge
 +          .builder(METRICS_COMPACTOR_JOB_PRIORITY_QUEUES, compactionJobQueues,
 +              CompactionJobQueues::getQueueCount)
-           .description("Number of current Queues").tags(getCommonTags()).register(localRegistry);
++          .description("Number of current Queues").register(localRegistry);
 +    }
 +    LOG.debug("update - cjq queues: {}", compactionJobQueues.getQueueIds());
 +
 +    Set<CompactorGroupId> definedQueues = compactionJobQueues.getQueueIds();
 +    LOG.debug("update - defined queues: {}", definedQueues);
 +
 +    // Copy the keySet into a new Set so that changes to perQueueMetrics
 +    // don't affect the collection
 +    Set<CompactorGroupId> queuesWithMetrics = new HashSet<>(perQueueMetrics.keySet());
 +    LOG.debug("update - queues with metrics: {}", queuesWithMetrics);
 +
 +    SetView<CompactorGroupId> queuesWithoutMetrics =
 +        Sets.difference(definedQueues, queuesWithMetrics);
 +    queuesWithoutMetrics.forEach(q -> {
 +      LOG.debug("update - creating meters for queue: {}", q);
 +      perQueueMetrics.put(q, new QueueMeters(localRegistry, q, compactionJobQueues.getQueue(q)));
 +    });
 +
 +    SetView<CompactorGroupId> metricsWithoutQueues =
 +        Sets.difference(queuesWithMetrics, definedQueues);
 +    metricsWithoutQueues.forEach(q -> {
 +      LOG.debug("update - removing meters for queue: {}", q);
 +      perQueueMetrics.get(q).removeMeters(localRegistry);
 +      perQueueMetrics.remove(q);
 +    });
 +
 +  }
 +
 +  @Override
 +  public void registerMetrics(MeterRegistry registry) {
 +    this.meterRegistry = registry;
 +  }
 +}
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java
index 7f31b0d70c,02a30d4d22..8215e2fdb1
--- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/ManagerMetrics.java
@@@ -18,59 -18,21 +18,65 @@@
   */
  package org.apache.accumulo.manager.metrics;
  
 +import static java.util.Objects.requireNonNull;
 +
+ import java.util.ArrayList;
+ import java.util.List;
 +import java.util.concurrent.atomic.AtomicLong;
  
  import org.apache.accumulo.core.conf.AccumuloConfiguration;
  import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
  import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.core.metrics.MetricsUtil;
  import org.apache.accumulo.manager.Manager;
  import org.apache.accumulo.manager.metrics.fate.FateMetrics;
  
 -public class ManagerMetrics {
 +import io.micrometer.core.instrument.MeterRegistry;
 +
 +public class ManagerMetrics implements MetricsProducer {
 +
 +  private final FateMetrics fateMetrics;
 +
-   private AtomicLong rootTGWErrorsGauge;
-   private AtomicLong metadataTGWErrorsGauge;
-   private AtomicLong userTGWErrorsGauge;
++  private final AtomicLong rootTGWErrorsGauge = new AtomicLong(0);
++  private final AtomicLong metadataTGWErrorsGauge = new AtomicLong(0);
++  private final AtomicLong userTGWErrorsGauge = new AtomicLong(0);
 +
 +  public ManagerMetrics(final AccumuloConfiguration conf, final Manager manager) {
 +    requireNonNull(conf, "AccumuloConfiguration must not be null");
 +    requireNonNull(conf, "Manager must not be null");
 +    fateMetrics = new FateMetrics(manager.getContext(),
 +        conf.getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL));
 +  }
 +
 +  public void incrementTabletGroupWatcherError(DataLevel level) {
 +    switch (level) {
 +      case METADATA:
 +        metadataTGWErrorsGauge.incrementAndGet();
 +        break;
 +      case ROOT:
 +        rootTGWErrorsGauge.incrementAndGet();
 +        break;
 +      case USER:
 +        userTGWErrorsGauge.incrementAndGet();
 +        break;
 +      default:
 +        throw new IllegalStateException("Unhandled DataLevel: " + level);
 +    }
 +  }
 +
 +  @Override
 +  public void registerMetrics(MeterRegistry registry) {
 +    fateMetrics.registerMetrics(registry);
-     rootTGWErrorsGauge = registry.gauge(METRICS_MANAGER_ROOT_TGW_ERRORS,
-         MetricsUtil.getCommonTags(), new AtomicLong(0));
-     metadataTGWErrorsGauge = registry.gauge(METRICS_MANAGER_META_TGW_ERRORS,
-         MetricsUtil.getCommonTags(), new AtomicLong(0));
-     userTGWErrorsGauge = registry.gauge(METRICS_MANAGER_USER_TGW_ERRORS,
-         MetricsUtil.getCommonTags(), new AtomicLong(0));
++    registry.gauge(METRICS_MANAGER_ROOT_TGW_ERRORS, rootTGWErrorsGauge);
++    registry.gauge(METRICS_MANAGER_META_TGW_ERRORS, metadataTGWErrorsGauge);
++    registry.gauge(METRICS_MANAGER_USER_TGW_ERRORS, userTGWErrorsGauge);
++  }
+ 
 -  public static List<MetricsProducer> getProducers(AccumuloConfiguration conf, Manager manager) {
++  public List<MetricsProducer> getProducers(AccumuloConfiguration conf, Manager manager) {
+     ArrayList<MetricsProducer> producers = new ArrayList<>();
 -    producers.add(new FateMetrics(manager.getContext(),
 -        conf.getTimeInMillis(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL)));
++    producers.add(this);
++    producers.add(fateMetrics);
++    producers.add(manager.getCompactionCoordinator());
+     return producers;
    }
  }
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
index 15e8c4c44b,bed2e53411..1664dd8d93
--- a/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/metrics/fate/FateMetrics.java
@@@ -25,10 -26,9 +26,9 @@@ import java.util.concurrent.TimeUnit
  import java.util.concurrent.atomic.AtomicLong;
  
  import org.apache.accumulo.core.Constants;
 -import org.apache.accumulo.core.fate.ReadOnlyTStore;
 -import org.apache.accumulo.core.fate.ZooStore;
 +import org.apache.accumulo.core.fate.MetaFateStore;
 +import org.apache.accumulo.core.fate.ReadOnlyFateStore;
  import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.core.metrics.MetricsUtil;
  import org.apache.accumulo.core.util.threads.ThreadPools;
  import org.apache.accumulo.server.ServerContext;
  import org.apache.zookeeper.KeeperException;
@@@ -53,16 -54,16 +54,16 @@@ public class FateMetrics implements Met
    private final String fateRootPath;
    private final long refreshDelay;
  
--  private AtomicLong totalCurrentOpsGauge;
--  private AtomicLong totalOpsGauge;
--  private AtomicLong fateErrorsGauge;
--  private AtomicLong newTxGauge;
--  private AtomicLong submittedTxGauge;
--  private AtomicLong inProgressTxGauge;
--  private AtomicLong failedInProgressTxGauge;
--  private AtomicLong failedTxGauge;
--  private AtomicLong successfulTxGauge;
--  private AtomicLong unknownTxGauge;
++  private AtomicLong totalCurrentOpsGauge = new AtomicLong(0);
++  private AtomicLong totalOpsGauge = new AtomicLong(0);
++  private AtomicLong fateErrorsGauge = new AtomicLong(0);
++  private AtomicLong newTxGauge = new AtomicLong(0);
++  private AtomicLong submittedTxGauge = new AtomicLong(0);
++  private AtomicLong inProgressTxGauge = new AtomicLong(0);
++  private AtomicLong failedInProgressTxGauge = new AtomicLong(0);
++  private AtomicLong failedTxGauge = new AtomicLong(0);
++  private AtomicLong successfulTxGauge = new AtomicLong(0);
++  private AtomicLong unknownTxGauge = new AtomicLong(0);
  
    public FateMetrics(final ServerContext context, final long minimumRefreshDelay) {
  
@@@ -84,9 -85,9 +85,8 @@@
    }
  
    private void update() {
--
      FateMetricValues metricValues =
 -        FateMetricValues.getFromZooKeeper(context, fateRootPath, zooStore);
 +        FateMetricValues.getFromZooKeeper(context, fateRootPath, fateStore);
  
      totalCurrentOpsGauge.set(metricValues.getCurrentFateOps());
      totalOpsGauge.set(metricValues.getZkFateChildOpsTotal());
@@@ -119,7 -120,7 +119,6 @@@
            log.warn("Unhandled status type: {}", vals.getKey());
        }
      }
--
      metricValues.getOpTypeCounters().forEach((name, count) -> {
        Metrics.gauge(METRICS_FATE_TYPE_IN_PROGRESS, Tags.of(OP_TYPE_TAG, name), count);
      });
@@@ -127,29 -128,31 +126,29 @@@
  
    @Override
    public void registerMetrics(final MeterRegistry registry) {
-     totalCurrentOpsGauge = registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS,
-         MetricsUtil.getCommonTags(), new AtomicLong(0));
-     totalOpsGauge =
-         registry.gauge(METRICS_FATE_OPS_ACTIVITY, MetricsUtil.getCommonTags(), new AtomicLong(0));
-     fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS,
-         Tags.concat(MetricsUtil.getCommonTags(), "type", "zk.connection"), new AtomicLong(0));
-     newTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(), "state",
-         ReadOnlyFateStore.TStatus.NEW.name().toLowerCase()), new AtomicLong(0));
-     submittedTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
-         "state", ReadOnlyFateStore.TStatus.SUBMITTED.name().toLowerCase()), new AtomicLong(0));
-     inProgressTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
-         "state", ReadOnlyFateStore.TStatus.IN_PROGRESS.name().toLowerCase()), new AtomicLong(0));
-     failedInProgressTxGauge =
-         registry.gauge(METRICS_FATE_TX,
-             Tags.concat(MetricsUtil.getCommonTags(), "state",
-                 ReadOnlyFateStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase()),
-             new AtomicLong(0));
-     failedTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
-         "state", ReadOnlyFateStore.TStatus.FAILED.name().toLowerCase()), new AtomicLong(0));
-     successfulTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
-         "state", ReadOnlyFateStore.TStatus.SUCCESSFUL.name().toLowerCase()), new AtomicLong(0));
-     unknownTxGauge = registry.gauge(METRICS_FATE_TX, Tags.concat(MetricsUtil.getCommonTags(),
-         "state", ReadOnlyFateStore.TStatus.UNKNOWN.name().toLowerCase()), new AtomicLong(0));
 -    totalCurrentOpsGauge = registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS, new AtomicLong(0));
 -    totalOpsGauge = registry.gauge(METRICS_FATE_OPS_ACTIVITY, new AtomicLong(0));
 -    fateErrorsGauge = registry.gauge(METRICS_FATE_ERRORS, List.of(Tag.of("type", "zk.connection")),
 -        new AtomicLong(0));
 -    newTxGauge = registry.gauge(METRICS_FATE_TX,
 -        List.of(Tag.of("state", ReadOnlyTStore.TStatus.NEW.name().toLowerCase())),
 -        new AtomicLong(0));
 -    submittedTxGauge = registry.gauge(METRICS_FATE_TX,
 -        List.of(Tag.of("state", ReadOnlyTStore.TStatus.SUBMITTED.name().toLowerCase())),
 -        new AtomicLong(0));
 -    inProgressTxGauge = registry.gauge(METRICS_FATE_TX,
 -        List.of(Tag.of("state", ReadOnlyTStore.TStatus.IN_PROGRESS.name().toLowerCase())),
 -        new AtomicLong(0));
 -    failedInProgressTxGauge = registry.gauge(METRICS_FATE_TX,
 -        List.of(Tag.of("state", ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase())),
 -        new AtomicLong(0));
 -    failedTxGauge = registry.gauge(METRICS_FATE_TX,
 -        List.of(Tag.of("state", ReadOnlyTStore.TStatus.FAILED.name().toLowerCase())),
 -        new AtomicLong(0));
 -    successfulTxGauge = registry.gauge(METRICS_FATE_TX,
 -        List.of(Tag.of("state", ReadOnlyTStore.TStatus.SUCCESSFUL.name().toLowerCase())),
 -        new AtomicLong(0));
 -    unknownTxGauge = registry.gauge(METRICS_FATE_TX,
 -        List.of(Tag.of("state", ReadOnlyTStore.TStatus.UNKNOWN.name().toLowerCase())),
 -        new AtomicLong(0));
++    registry.gauge(METRICS_FATE_TOTAL_IN_PROGRESS, totalCurrentOpsGauge);
++    registry.gauge(METRICS_FATE_OPS_ACTIVITY, totalOpsGauge);
++    registry.gauge(METRICS_FATE_ERRORS, List.of(Tag.of("type", "zk.connection")), fateErrorsGauge);
++    registry.gauge(METRICS_FATE_TX,
++        List.of(Tag.of("state", ReadOnlyFateStore.TStatus.NEW.name().toLowerCase())), newTxGauge);
++    registry.gauge(METRICS_FATE_TX,
++        List.of(Tag.of("state", ReadOnlyFateStore.TStatus.SUBMITTED.name().toLowerCase())),
++        submittedTxGauge);
++    registry.gauge(METRICS_FATE_TX,
++        List.of(Tag.of("state", ReadOnlyFateStore.TStatus.IN_PROGRESS.name().toLowerCase())),
++        inProgressTxGauge);
++    registry.gauge(METRICS_FATE_TX,
++        List.of(Tag.of("state", ReadOnlyFateStore.TStatus.FAILED_IN_PROGRESS.name().toLowerCase())),
++        failedInProgressTxGauge);
++    registry.gauge(METRICS_FATE_TX,
++        List.of(Tag.of("state", ReadOnlyFateStore.TStatus.FAILED.name().toLowerCase())),
++        failedTxGauge);
++    registry.gauge(METRICS_FATE_TX,
++        List.of(Tag.of("state", ReadOnlyFateStore.TStatus.SUCCESSFUL.name().toLowerCase())),
++        successfulTxGauge);
++    registry.gauge(METRICS_FATE_TX,
++        List.of(Tag.of("state", ReadOnlyFateStore.TStatus.UNKNOWN.name().toLowerCase())),
++        unknownTxGauge);
  
      update();
  
diff --cc server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
index bb75b8c65c,0000000000..1c4a02a7a4
mode 100644,000000..100644
--- a/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
+++ b/server/manager/src/test/java/org/apache/accumulo/manager/compaction/CompactionCoordinatorTest.java
@@@ -1,530 -1,0 +1,555 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one
 + * or more contributor license agreements.  See the NOTICE file
 + * distributed with this work for additional information
 + * regarding copyright ownership.  The ASF licenses this file
 + * to you under the Apache License, Version 2.0 (the
 + * "License"); you may not use this file except in compliance
 + * with the License.  You may obtain a copy of the License at
 + *
 + *   https://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing,
 + * software distributed under the License is distributed on an
 + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 + * KIND, either express or implied.  See the License for the
 + * specific language governing permissions and limitations
 + * under the License.
 + */
 +package org.apache.accumulo.manager.compaction;
 +
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.ECOMP;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
 +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED;
 +import static org.apache.accumulo.manager.compaction.coordinator.CompactionCoordinator.canReserveCompaction;
++import static org.easymock.EasyMock.anyObject;
++import static org.easymock.EasyMock.createMock;
 +import static org.easymock.EasyMock.expect;
++import static org.easymock.EasyMock.expectLastCall;
++import static org.easymock.EasyMock.replay;
 +import static org.junit.jupiter.api.Assertions.assertEquals;
 +import static org.junit.jupiter.api.Assertions.assertFalse;
 +import static org.junit.jupiter.api.Assertions.assertNull;
 +import static org.junit.jupiter.api.Assertions.assertTrue;
 +
 +import java.net.URI;
 +import java.util.ArrayList;
 +import java.util.Collections;
 +import java.util.List;
 +import java.util.Map;
 +import java.util.Map.Entry;
 +import java.util.Optional;
 +import java.util.Set;
 +import java.util.UUID;
 +import java.util.concurrent.ScheduledThreadPoolExecutor;
 +import java.util.concurrent.atomic.AtomicReference;
 +import java.util.stream.Collectors;
 +
 +import org.apache.accumulo.core.client.admin.CompactionConfig;
 +import org.apache.accumulo.core.clientImpl.thrift.TInfo;
 +import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
 +import org.apache.accumulo.core.compaction.thrift.TExternalCompaction;
 +import org.apache.accumulo.core.conf.DefaultConfiguration;
 +import org.apache.accumulo.core.conf.Property;
 +import org.apache.accumulo.core.data.TableId;
 +import org.apache.accumulo.core.dataImpl.KeyExtent;
 +import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
 +import org.apache.accumulo.core.fate.Fate;
 +import org.apache.accumulo.core.fate.FateId;
 +import org.apache.accumulo.core.fate.FateInstanceType;
 +import org.apache.accumulo.core.iteratorsImpl.system.SystemIteratorUtil;
 +import org.apache.accumulo.core.manager.state.tables.TableState;
 +import org.apache.accumulo.core.metadata.CompactableFileImpl;
 +import org.apache.accumulo.core.metadata.ReferencedTabletFile;
 +import org.apache.accumulo.core.metadata.StoredTabletFile;
 +import org.apache.accumulo.core.metadata.schema.CompactionMetadata;
 +import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
 +import org.apache.accumulo.core.metadata.schema.SelectedFiles;
 +import org.apache.accumulo.core.metadata.schema.TabletMetadata;
 +import org.apache.accumulo.core.metadata.schema.TabletOperationId;
 +import org.apache.accumulo.core.metadata.schema.TabletOperationType;
++import org.apache.accumulo.core.metrics.MetricsInfo;
 +import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
 +import org.apache.accumulo.core.spi.compaction.CompactionJob;
 +import org.apache.accumulo.core.spi.compaction.CompactionKind;
 +import org.apache.accumulo.core.spi.compaction.CompactorGroupId;
 +import org.apache.accumulo.core.tabletserver.thrift.TCompactionKind;
 +import org.apache.accumulo.core.tabletserver.thrift.TCompactionStats;
 +import org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob;
 +import org.apache.accumulo.core.trace.TraceUtil;
 +import org.apache.accumulo.core.util.cache.Caches;
 +import org.apache.accumulo.core.util.compaction.CompactionJobImpl;
 +import org.apache.accumulo.core.util.compaction.RunningCompaction;
 +import org.apache.accumulo.manager.Manager;
 +import org.apache.accumulo.manager.compaction.coordinator.CompactionCoordinator;
 +import org.apache.accumulo.manager.compaction.queue.CompactionJobPriorityQueue;
 +import org.apache.accumulo.manager.compaction.queue.CompactionJobQueues.MetaJob;
 +import org.apache.accumulo.server.ServerContext;
 +import org.apache.accumulo.server.conf.TableConfiguration;
 +import org.apache.accumulo.server.security.AuditedSecurityOperation;
 +import org.apache.accumulo.server.security.SecurityOperation;
 +import org.apache.hadoop.fs.Path;
 +import org.apache.hadoop.io.Text;
 +import org.easymock.EasyMock;
 +import org.junit.jupiter.api.Test;
 +
 +import com.google.common.net.HostAndPort;
 +
 +public class CompactionCoordinatorTest {
 +
 +  // Need a non-null fateInstances reference for CompactionCoordinator.compactionCompleted
 +  private static final AtomicReference<Map<FateInstanceType,Fate<Manager>>> fateInstances =
 +      new AtomicReference<>(Map.of());
 +
 +  private static final CompactorGroupId GROUP_ID = CompactorGroupId.of("R2DQ");
 +
++  private final HostAndPort tserverAddr = HostAndPort.fromParts("192.168.1.1", 9090);
++
++  public MetricsInfo getMockMetrics() {
++    MetricsInfo metricsInfo = createMock(MetricsInfo.class);
++    metricsInfo.addServiceTags(anyObject(), anyObject(), anyObject());
++    expectLastCall().anyTimes();
++    metricsInfo.addMetricsProducers(anyObject());
++    expectLastCall().anyTimes();
++    metricsInfo.init();
++    expectLastCall().anyTimes();
++    replay(metricsInfo);
++    return metricsInfo;
++  }
++
 +  public class TestCoordinator extends CompactionCoordinator {
 +
 +    private final List<RunningCompaction> runningCompactions;
 +
 +    private Set<ExternalCompactionId> metadataCompactionIds = null;
 +
 +    public TestCoordinator(ServerContext ctx, SecurityOperation security,
 +        List<RunningCompaction> runningCompactions) {
-       super(ctx, security, fateInstances);
++      super(ctx, security, fateInstances, "TEST_GROUP");
 +      this.runningCompactions = runningCompactions;
 +    }
 +
 +    @Override
 +    protected void startDeadCompactionDetector() {}
 +
 +    @Override
 +    protected long getTServerCheckInterval() {
 +      return 5000L;
 +    }
 +
 +    @Override
 +    protected void startCompactionCleaner(ScheduledThreadPoolExecutor schedExecutor) {}
 +
 +    @Override
 +    protected void startRunningCleaner(ScheduledThreadPoolExecutor schedExecutor) {}
 +
 +    @Override
 +    protected void startIdleCompactionWatcher() {
 +      // This is called from CompactionCoordinator.run(). Counting down
 +      // the latch will exit the run method
 +      this.shutdown.countDown();
 +    }
 +
 +    @Override
 +    public void compactionCompleted(TInfo tinfo, TCredentials credentials,
 +        String externalCompactionId, TKeyExtent textent, TCompactionStats stats)
 +        throws ThriftSecurityException {}
 +
 +    @Override
 +    public void compactionFailed(TInfo tinfo, TCredentials credentials, String externalCompactionId,
 +        TKeyExtent extent) throws ThriftSecurityException {}
 +
 +    void setMetadataCompactionIds(Set<ExternalCompactionId> mci) {
 +      metadataCompactionIds = mci;
 +    }
 +
 +    @Override
 +    protected Set<ExternalCompactionId> readExternalCompactionIds() {
 +      if (metadataCompactionIds == null) {
 +        return RUNNING_CACHE.keySet();
 +      } else {
 +        return metadataCompactionIds;
 +      }
 +    }
 +
 +    public Map<ExternalCompactionId,RunningCompaction> getRunning() {
 +      return RUNNING_CACHE;
 +    }
 +
 +    public void resetInternals() {
 +      getRunning().clear();
 +      metadataCompactionIds = null;
 +    }
 +
 +    @Override
 +    protected List<RunningCompaction> getCompactionsRunningOnCompactors() {
 +      return runningCompactions;
 +    }
 +
 +    @Override
 +    protected CompactionMetadata reserveCompaction(MetaJob metaJob, String compactorAddress,
 +        ExternalCompactionId externalCompactionId) {
 +      return createExternalCompactionMetadata(metaJob.getJob(),
 +          metaJob.getJob().getFiles().stream().map(CompactableFileImpl::toStoredTabletFile)
 +              .collect(Collectors.toSet()),
 +          metaJob.getTabletMetadata(), compactorAddress, externalCompactionId);
 +    }
 +
 +    @Override
 +    protected CompactionMetadata createExternalCompactionMetadata(CompactionJob job,
 +        Set<StoredTabletFile> jobFiles, TabletMetadata tablet, String compactorAddress,
 +        ExternalCompactionId externalCompactionId) {
 +      FateInstanceType type = FateInstanceType.fromTableId(tablet.getExtent().tableId());
 +      FateId fateId = FateId.from(type, UUID.randomUUID());
 +      return new CompactionMetadata(jobFiles,
 +          new ReferencedTabletFile(new Path("file:///accumulo/tables/1/default_tablet/F00001.rf")),
 +          compactorAddress, job.getKind(), job.getPriority(), job.getGroup(), true, fateId);
 +    }
 +
 +    @Override
 +    protected TExternalCompactionJob createThriftJob(String externalCompactionId,
 +        CompactionMetadata ecm, MetaJob metaJob, Optional<CompactionConfig> compactionConfig) {
 +      return new TExternalCompactionJob(externalCompactionId,
 +          metaJob.getTabletMetadata().getExtent().toThrift(), List.of(),
 +          SystemIteratorUtil.toIteratorConfig(List.of()),
 +          ecm.getCompactTmpName().getNormalizedPathStr(), ecm.getPropagateDeletes(),
 +          TCompactionKind.valueOf(ecm.getKind().name()),
 +          FateId
 +              .from(FateInstanceType.fromTableId(metaJob.getTabletMetadata().getExtent().tableId()),
 +                  UUID.randomUUID())
 +              .toThrift(),
 +          Map.of());
 +    }
 +
 +    @Override
 +    protected void cancelCompactionOnCompactor(String address, String externalCompactionId) {}
 +
 +  }
 +
 +  @Test
 +  public void testCoordinatorColdStart() throws Exception {
 +
 +    ServerContext context = EasyMock.createNiceMock(ServerContext.class);
 +    expect(context.getCaches()).andReturn(Caches.getInstance()).anyTimes();
 +    expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
 +
++    MetricsInfo metricsInfo = getMockMetrics();
++    expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
++
 +    AuditedSecurityOperation security = EasyMock.createNiceMock(AuditedSecurityOperation.class);
 +
 +    EasyMock.replay(context, security);
 +
 +    var coordinator = new TestCoordinator(context, security, new ArrayList<>());
 +    assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
 +    assertEquals(0, coordinator.getRunning().size());
 +    coordinator.run();
 +    coordinator.shutdown();
 +
 +    assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
 +    assertEquals(0, coordinator.getRunning().size());
-     EasyMock.verify(context, security);
++    EasyMock.verify(context, security, metricsInfo);
 +  }
 +
 +  @Test
 +  public void testCoordinatorRestartOneRunningCompaction() throws Exception {
 +
 +    ServerContext context = EasyMock.createNiceMock(ServerContext.class);
 +    expect(context.getCaches()).andReturn(Caches.getInstance()).anyTimes();
 +    expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
 +
-     HostAndPort tserverAddress = HostAndPort.fromString("localhost:9997");
++    MetricsInfo metricsInfo = getMockMetrics();
++    expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
 +
 +    List<RunningCompaction> runningCompactions = new ArrayList<>();
 +    ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID());
 +    TExternalCompactionJob job = EasyMock.createNiceMock(TExternalCompactionJob.class);
 +    expect(job.getExternalCompactionId()).andReturn(eci.toString()).anyTimes();
 +    TKeyExtent extent = new TKeyExtent();
 +    extent.setTable("1".getBytes());
-     runningCompactions
-         .add(new RunningCompaction(job, tserverAddress.toString(), GROUP_ID.toString()));
++    runningCompactions.add(new RunningCompaction(job, tserverAddr.toString(), GROUP_ID.toString()));
 +
 +    AuditedSecurityOperation security = EasyMock.createNiceMock(AuditedSecurityOperation.class);
 +
 +    EasyMock.replay(context, job, security);
 +
 +    var coordinator = new TestCoordinator(context, security, runningCompactions);
 +    coordinator.resetInternals();
 +    assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
 +    assertEquals(0, coordinator.getRunning().size());
 +    coordinator.run();
 +    coordinator.shutdown();
 +    assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
 +    assertEquals(1, coordinator.getRunning().size());
 +
 +    Map<ExternalCompactionId,RunningCompaction> running = coordinator.getRunning();
 +    Entry<ExternalCompactionId,RunningCompaction> ecomp = running.entrySet().iterator().next();
 +    assertEquals(eci, ecomp.getKey());
 +    RunningCompaction rc = ecomp.getValue();
 +    assertEquals(GROUP_ID.toString(), rc.getGroupName());
-     assertEquals(tserverAddress.toString(), rc.getCompactorAddress());
++    assertEquals(tserverAddr.toString(), rc.getCompactorAddress());
 +
 +    EasyMock.verify(context, job, security);
 +  }
 +
 +  @Test
 +  public void testGetCompactionJob() throws Exception {
 +
 +    TableConfiguration tconf = EasyMock.createNiceMock(TableConfiguration.class);
 +    expect(tconf.get(Property.TABLE_COMPACTION_CONFIGURER))
 +        .andReturn(Property.TABLE_COMPACTION_CONFIGURER.getDefaultValue()).anyTimes();
 +
 +    ServerContext context = EasyMock.createNiceMock(ServerContext.class);
 +    expect(context.getCaches()).andReturn(Caches.getInstance()).anyTimes();
 +    expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
 +    expect(context.getTableConfiguration(TableId.of("2a"))).andReturn(tconf).anyTimes();
 +
++    MetricsInfo metricsInfo = getMockMetrics();
++    expect(context.getMetricsInfo()).andReturn(metricsInfo).anyTimes();
++
 +    TCredentials creds = EasyMock.createNiceMock(TCredentials.class);
 +    expect(context.rpcCreds()).andReturn(creds).anyTimes();
 +
 +    AuditedSecurityOperation security = EasyMock.createNiceMock(AuditedSecurityOperation.class);
 +    expect(security.canPerformSystemActions(creds)).andReturn(true).anyTimes();
 +
 +    KeyExtent ke = new KeyExtent(TableId.of("2a"), new Text("z"), new Text("b"));
 +    TabletMetadata tm = EasyMock.createNiceMock(TabletMetadata.class);
 +    expect(tm.getExtent()).andReturn(ke).anyTimes();
 +    expect(tm.getFiles()).andReturn(Collections.emptySet()).anyTimes();
 +    expect(tm.getTableId()).andReturn(ke.tableId()).anyTimes();
 +
 +    EasyMock.replay(tconf, context, creds, tm, security);
 +
 +    var coordinator = new TestCoordinator(context, security, new ArrayList<>());
 +    assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
 +    assertEquals(0, coordinator.getRunning().size());
 +    // Use coordinator.run() to populate the internal data structures. This is tested in a different
 +    // test.
 +    coordinator.run();
 +    coordinator.shutdown();
 +
 +    assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
 +    assertEquals(0, coordinator.getRunning().size());
 +
 +    // Add a job to the job queue
 +    CompactionJob job = new CompactionJobImpl((short) 1, GROUP_ID, Collections.emptyList(),
 +        CompactionKind.SYSTEM, Optional.of(true));
 +    coordinator.addJobs(tm, Collections.singleton(job));
 +    CompactionJobPriorityQueue queue = coordinator.getJobQueues().getQueue(GROUP_ID);
 +    assertEquals(1, queue.getQueuedJobs());
 +
 +    // Get the next job
 +    ExternalCompactionId eci = ExternalCompactionId.generate(UUID.randomUUID());
 +    TExternalCompactionJob createdJob = coordinator.getCompactionJob(new TInfo(), creds,
 +        GROUP_ID.toString(), "localhost:10241", eci.toString());
 +    assertEquals(eci.toString(), createdJob.getExternalCompactionId());
 +    assertEquals(ke, KeyExtent.fromThrift(createdJob.getExtent()));
 +
 +    assertEquals(0, coordinator.getJobQueues().getQueuedJobCount());
 +    assertEquals(1, coordinator.getRunning().size());
 +    Entry<ExternalCompactionId,RunningCompaction> entry =
 +        coordinator.getRunning().entrySet().iterator().next();
 +    assertEquals(eci.toString(), entry.getKey().toString());
 +    assertEquals("localhost:10241", entry.getValue().getCompactorAddress());
 +    assertEquals(eci.toString(), entry.getValue().getJob().getExternalCompactionId());
 +
 +    EasyMock.verify(tconf, context, creds, tm, security);
 +  }
 +
 +  @Test
 +  public void testGetCompactionJobNoJobs() throws Exception {
 +
 +    ServerContext context = EasyMock.createNiceMock(ServerContext.class);
 +    expect(context.getCaches()).andReturn(Caches.getInstance()).anyTimes();
 +    expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
 +
 +    TCredentials creds = EasyMock.createNiceMock(TCredentials.class);
 +
 +    AuditedSecurityOperation security = EasyMock.createNiceMock(AuditedSecurityOperation.class);
 +    expect(security.canPerformSystemActions(creds)).andReturn(true);
 +
 +    EasyMock.replay(context, creds, security);
 +
 +    var coordinator = new TestCoordinator(context, security, new ArrayList<>());
 +    TExternalCompactionJob job = coordinator.getCompactionJob(TraceUtil.traceInfo(), creds,
 +        GROUP_ID.toString(), "localhost:10240", UUID.randomUUID().toString());
 +    assertNull(job.getExternalCompactionId());
 +
 +    EasyMock.verify(context, creds, security);
 +  }
 +
 +  @Test
 +  public void testCleanUpRunning() throws Exception {
 +
 +    ServerContext context = EasyMock.createNiceMock(ServerContext.class);
 +    expect(context.getCaches()).andReturn(Caches.getInstance()).anyTimes();
 +    expect(context.getConfiguration()).andReturn(DefaultConfiguration.getInstance()).anyTimes();
 +
 +    TCredentials creds = EasyMock.createNiceMock(TCredentials.class);
 +
 +    AuditedSecurityOperation security = EasyMock.createNiceMock(AuditedSecurityOperation.class);
 +
 +    EasyMock.replay(context, creds, security);
 +
 +    TestCoordinator coordinator = new TestCoordinator(context, security, new ArrayList<>());
 +
 +    var ecid1 = ExternalCompactionId.generate(UUID.randomUUID());
 +    var ecid2 = ExternalCompactionId.generate(UUID.randomUUID());
 +    var ecid3 = ExternalCompactionId.generate(UUID.randomUUID());
 +
 +    coordinator.getRunning().put(ecid1, new RunningCompaction(new TExternalCompaction()));
 +    coordinator.getRunning().put(ecid2, new RunningCompaction(new TExternalCompaction()));
 +    coordinator.getRunning().put(ecid3, new RunningCompaction(new TExternalCompaction()));
 +
 +    coordinator.cleanUpRunning();
 +
 +    assertEquals(Set.of(ecid1, ecid2, ecid3), coordinator.getRunning().keySet());
 +
 +    coordinator.setMetadataCompactionIds(Set.of(ecid1, ecid2));
 +
 +    coordinator.cleanUpRunning();
 +
 +    assertEquals(Set.of(ecid1, ecid2), coordinator.getRunning().keySet());
 +
 +    EasyMock.verify(context, creds, security);
 +
 +  }
 +
 +  @Test
 +  public void testCanReserve() throws Exception {
 +    TableId tableId1 = TableId.of("5");
 +    TableId tableId2 = TableId.of("6");
 +
 +    var file1 = StoredTabletFile.of(new URI("file:///accumulo/tables/1/default_tablet/F00001.rf"));
 +    var file2 = StoredTabletFile.of(new URI("file:///accumulo/tables/1/default_tablet/F00002.rf"));
 +    var file3 = StoredTabletFile.of(new URI("file:///accumulo/tables/1/default_tablet/F00003.rf"));
 +    var file4 = StoredTabletFile.of(new URI("file:///accumulo/tables/1/default_tablet/F00004.rf"));
 +
 +    ServerContext context = EasyMock.mock(ServerContext.class);
 +    EasyMock.expect(context.getTableState(tableId1)).andReturn(TableState.ONLINE).atLeastOnce();
 +    EasyMock.expect(context.getTableState(tableId2)).andReturn(TableState.OFFLINE).atLeastOnce();
 +
 +    FateId fateId1 = FateId.from(FateInstanceType.USER, UUID.randomUUID());
 +
 +    CompactorGroupId cgid = CompactorGroupId.of("G1");
 +    ReferencedTabletFile tmp1 =
 +        ReferencedTabletFile.of(new Path("file:///accumulo/tables/1/default_tablet/C00005.rf_tmp"));
 +    CompactionMetadata cm1 = new CompactionMetadata(Set.of(file1, file2), tmp1, "localhost:4444",
 +        CompactionKind.SYSTEM, (short) 5, cgid, false, null);
 +
 +    ReferencedTabletFile tmp2 =
 +        ReferencedTabletFile.of(new Path("file:///accumulo/tables/1/default_tablet/C00006.rf_tmp"));
 +    CompactionMetadata cm2 = new CompactionMetadata(Set.of(file3), tmp2, "localhost:5555",
 +        CompactionKind.USER, (short) 5, cgid, false, fateId1);
 +
 +    EasyMock.replay(context);
 +
 +    KeyExtent extent1 = new KeyExtent(tableId1, null, null);
 +
 +    var dfv = new DataFileValue(1000, 100);
 +
 +    var cid1 = ExternalCompactionId.generate(UUID.randomUUID());
 +    var cid2 = ExternalCompactionId.generate(UUID.randomUUID());
 +
 +    var selected = new SelectedFiles(Set.of(file1, file2, file3), false, fateId1);
 +
 +    // should not be able to compact an offline table
 +    var tabletOffline = TabletMetadata.builder(new KeyExtent(tableId2, null, null))
 +        .putFile(file1, dfv).putFile(file2, dfv).putFile(file3, dfv).putFile(file4, dfv)
 +        .build(OPID, ECOMP, USER_COMPACTION_REQUESTED, SELECTED);
 +    assertFalse(
 +        canReserveCompaction(tabletOffline, CompactionKind.SYSTEM, Set.of(file1, file2), context));
 +
 +    // nothing should prevent this compaction
 +    var tablet1 =
 +        TabletMetadata.builder(extent1).putFile(file1, dfv).putFile(file2, dfv).putFile(file3, dfv)
 +            .putFile(file4, dfv).build(OPID, ECOMP, USER_COMPACTION_REQUESTED, SELECTED);
 +    assertTrue(canReserveCompaction(tablet1, CompactionKind.SYSTEM, Set.of(file1, file2), context));
 +
 +    // should not be able to do a user compaction unless selected files are present
 +    assertFalse(canReserveCompaction(tablet1, CompactionKind.USER, Set.of(file1, file2), context));
 +
 +    // should not be able to compact a tablet with user compaction request in place
 +    var tablet3 =
 +        TabletMetadata.builder(extent1).putFile(file1, dfv).putFile(file2, dfv).putFile(file3, dfv)
 +            .putFile(file4, dfv).putUserCompactionRequested(fateId1).build(OPID, ECOMP, SELECTED);
 +    assertFalse(
 +        canReserveCompaction(tablet3, CompactionKind.SYSTEM, Set.of(file1, file2), context));
 +
 +    // should not be able to compact a tablet when the job has files not present in the tablet
 +    var tablet4 = TabletMetadata.builder(extent1).putFile(file1, dfv).putFile(file2, dfv)
 +        .putFile(file3, dfv).build(OPID, ECOMP, USER_COMPACTION_REQUESTED, SELECTED);
 +    assertFalse(
 +        canReserveCompaction(tablet4, CompactionKind.SYSTEM, Set.of(file1, file2, file4), context));
 +
 +    // should not be able to compact a tablet with an operation id present
 +    TabletOperationId opid = TabletOperationId.from(TabletOperationType.SPLITTING, fateId1);
 +    var tablet5 = TabletMetadata.builder(extent1).putFile(file1, dfv).putFile(file2, dfv)
 +        .putFile(file3, dfv).putFile(file4, dfv).putOperation(opid)
 +        .build(ECOMP, USER_COMPACTION_REQUESTED, SELECTED);
 +    assertFalse(
 +        canReserveCompaction(tablet5, CompactionKind.SYSTEM, Set.of(file1, file2), context));
 +
 +    // should not be able to compact a tablet if the job files overlaps with running compactions
 +    var tablet6 = TabletMetadata.builder(extent1).putFile(file1, dfv).putFile(file2, dfv)
 +        .putFile(file3, dfv).putFile(file4, dfv).putExternalCompaction(cid1, cm1)
 +        .putExternalCompaction(cid2, cm2).build(OPID, USER_COMPACTION_REQUESTED, SELECTED);
 +    assertFalse(
 +        canReserveCompaction(tablet6, CompactionKind.SYSTEM, Set.of(file1, file2), context));
 +    // should be able to compact the file that is outside of the set of files currently compacting
 +    assertTrue(canReserveCompaction(tablet6, CompactionKind.SYSTEM, Set.of(file4), context));
 +
 +    // create a tablet with a selected set of files
 +    var selTablet = TabletMetadata.builder(extent1).putFile(file1, dfv).putFile(file2, dfv)
 +        .putFile(file3, dfv).putFile(file4, dfv).putSelectedFiles(selected)
 +        .build(OPID, USER_COMPACTION_REQUESTED, ECOMP);
 +    // should not be able to start a system compaction if the set of files overlaps with the
 +    // selected files
 +    assertFalse(
 +        canReserveCompaction(selTablet, CompactionKind.SYSTEM, Set.of(file1, file2), context));
 +    assertFalse(
 +        canReserveCompaction(selTablet, CompactionKind.SYSTEM, Set.of(file3, file4), context));
 +    // should be able to start a system compaction on the set of files not in the selected set
 +    assertTrue(canReserveCompaction(selTablet, CompactionKind.SYSTEM, Set.of(file4), context));
 +    // should be able to start user compactions on files that are selected
 +    assertTrue(canReserveCompaction(selTablet, CompactionKind.USER, Set.of(file1, file2), context));
 +    assertTrue(canReserveCompaction(selTablet, CompactionKind.USER, Set.of(file2, file3), context));
 +    assertTrue(
 +        canReserveCompaction(selTablet, CompactionKind.USER, Set.of(file1, file2, file3), context));
 +    // should not be able to start user compactions on files that fall outside of the selected set
 +    assertFalse(
 +        canReserveCompaction(selTablet, CompactionKind.USER, Set.of(file1, file4), context));
 +    assertFalse(canReserveCompaction(selTablet, CompactionKind.USER, Set.of(file4), context));
 +    assertFalse(canReserveCompaction(selTablet, CompactionKind.USER,
 +        Set.of(file1, file2, file3, file4), context));
 +
 +    // test selected files and running compaction
 +    var selRunningTablet = TabletMetadata.builder(extent1).putFile(file1, dfv).putFile(file2, dfv)
 +        .putFile(file3, dfv).putFile(file4, dfv).putSelectedFiles(selected)
 +        .putExternalCompaction(cid2, cm2).build(OPID, USER_COMPACTION_REQUESTED);
 +    // should be able to compact files that are in the selected set and not in the running set
 +    assertTrue(
 +        canReserveCompaction(selRunningTablet, CompactionKind.USER, Set.of(file1, file2), context));
 +    // should not be able to compact because files overlap the running set
 +    assertFalse(
 +        canReserveCompaction(selRunningTablet, CompactionKind.USER, Set.of(file2, file3), context));
 +    // should not be able to start a system compaction if the set of files overlaps with the
 +    // selected files and/or the running set
 +    assertFalse(canReserveCompaction(selRunningTablet, CompactionKind.SYSTEM, Set.of(file1, file2),
 +        context));
 +    assertFalse(canReserveCompaction(selRunningTablet, CompactionKind.SYSTEM, Set.of(file3, file4),
 +        context));
 +    // should be able to start a system compaction on the set of files not in the selected set
 +    assertTrue(
 +        canReserveCompaction(selRunningTablet, CompactionKind.SYSTEM, Set.of(file4), context));
 +
 +    // should not be able to compact a tablet that does not exists
 +    assertFalse(canReserveCompaction(null, CompactionKind.SYSTEM, Set.of(file1, file2), context));
 +    assertFalse(canReserveCompaction(null, CompactionKind.USER, Set.of(file1, file2), context));
 +
 +    EasyMock.verify(context);
 +  }
 +}
diff --cc server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 1ecf3301f7,fde338f422..0d99ea3f0f
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@@ -486,6 -491,11 +487,11 @@@ public class Monitor extends AbstractSe
      }
      log.debug("Using {} to advertise monitor location in ZooKeeper", advertiseHost);
  
+     MetricsInfo metricsInfo = getContext().getMetricsInfo();
 -    metricsInfo.addServiceTags(getApplicationName(),
 -        HostAndPort.fromParts(advertiseHost, livePort));
++    metricsInfo.addServiceTags(getApplicationName(), HostAndPort.fromParts(advertiseHost, livePort),
++        getResourceGroup());
+     metricsInfo.init();
+ 
      try {
        URL url = new URL(server.isSecure() ? "https" : "http", advertiseHost, server.getPort(), "/");
        final String path = context.getZooKeeperRoot() + Constants.ZMONITOR_HTTP_ADDR;
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 50c3bc190d,ca4b28d06e..77db9e440f
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@@ -376,17 -373,13 +375,12 @@@ public class ScanServer extends Abstrac
        throw new RuntimeException("Failed to start the compactor client service", e1);
      }
  
-     try {
-       MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName,
-           clientAddress, getContext().getInstanceName(), this.getResourceGroup());
-       scanMetrics = new TabletServerScanMetrics();
-       MetricsUtil.initializeProducers(this, scanMetrics);
-     } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
-         | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
-         | SecurityException e1) {
-       LOG.error("Error initializing metrics, metrics will not be emitted.", e1);
-     }
+     MetricsInfo metricsInfo = getContext().getMetricsInfo();
 -    metricsInfo.addServiceTags(getApplicationName(), clientAddress);
 -
++    metricsInfo.addServiceTags(getApplicationName(), clientAddress, getResourceGroup());
+     scanMetrics = new TabletServerScanMetrics();
  
+     metricsInfo.addMetricsProducers(this, scanMetrics);
+     metricsInfo.init();
      // We need to set the compaction manager so that we don't get an NPE in CompactableImpl.close
  
      ServiceLock lock = announceExistence();
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 026c904ca2,89d503ef58..9947b5e48b
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -87,9 -90,9 +87,9 @@@ import org.apache.accumulo.core.manager
  import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
  import org.apache.accumulo.core.metadata.AccumuloTable;
  import org.apache.accumulo.core.metadata.TServerInstance;
 +import org.apache.accumulo.core.metadata.schema.Ample.TabletsMutator;
  import org.apache.accumulo.core.metadata.schema.TabletMetadata;
- import org.apache.accumulo.core.metrics.MetricsUtil;
 -import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
+ import org.apache.accumulo.core.metrics.MetricsInfo;
  import org.apache.accumulo.core.rpc.ThriftUtil;
  import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
  import org.apache.accumulo.core.spi.fs.VolumeChooserEnvironment;
@@@ -576,24 -704,25 +576,18 @@@ public class TabletServer extends Abstr
        throw new RuntimeException("Failed to start the tablet client service", e1);
      }
  
-     try {
-       MetricsUtil.initializeMetrics(context.getConfiguration(), this.applicationName, clientAddress,
-           getContext().getInstanceName(), this.getResourceGroup());
- 
-       metrics = new TabletServerMetrics(this);
-       updateMetrics = new TabletServerUpdateMetrics();
-       scanMetrics = new TabletServerScanMetrics();
-       mincMetrics = new TabletServerMinCMetrics();
-       pausedMetrics = new PausedCompactionMetrics();
-       MetricsUtil.initializeProducers(this, metrics, updateMetrics, scanMetrics, mincMetrics,
-           pausedMetrics);
- 
-     } catch (ClassNotFoundException | InstantiationException | IllegalAccessException
-         | IllegalArgumentException | InvocationTargetException | NoSuchMethodException
-         | SecurityException e1) {
-       log.error("Error initializing metrics, metrics will not be emitted.", e1);
-     }
+     MetricsInfo metricsInfo = getContext().getMetricsInfo();
 -    metricsInfo.addServiceTags(getApplicationName(), clientAddress);
 -
++    metricsInfo.addServiceTags(getApplicationName(), clientAddress, getResourceGroup());
+     metrics = new TabletServerMetrics(this);
+     updateMetrics = new TabletServerUpdateMetrics();
+     scanMetrics = new TabletServerScanMetrics();
+     mincMetrics = new TabletServerMinCMetrics();
 -    ceMetrics = new CompactionExecutorsMetrics();
+     pausedMetrics = new PausedCompactionMetrics();
+ 
 -    metricsInfo.addMetricsProducers(metrics, updateMetrics, scanMetrics, mincMetrics, ceMetrics,
++    metricsInfo.addMetricsProducers(metrics, updateMetrics, scanMetrics, mincMetrics,
+         pausedMetrics);
+     metricsInfo.init();
  
 -    this.compactionManager = new CompactionManager(() -> Iterators
 -        .transform(onlineTablets.snapshot().values().iterator(), Tablet::asCompactable),
 -        getContext(), ceMetrics);
 -    compactionManager.start();
 -
      announceExistence();
  
      try {
diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index 0a86efdf7a,2bb80309a1..cfa95cd269
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@@ -300,17 -303,25 +301,17 @@@ public class TabletServerResourceManage
      }
  
      minorCompactionThreadPool = ThreadPools.getServerThreadPools().createExecutorService(acuConf,
--        Property.TSERV_MINC_MAXCONCURRENT, true);
++        Property.TSERV_MINC_MAXCONCURRENT, enableMetrics);
      modifyThreadPoolSizesAtRuntime(
          () -> context.getConfiguration().getCount(Property.TSERV_MINC_MAXCONCURRENT),
          "minor compactor", minorCompactionThreadPool);
  
 -    splitThreadPool = ThreadPools.getServerThreadPools().getPoolBuilder("splitter")
 -        .numCoreThreads(0).numMaxThreads(1).withTimeOut(1, SECONDS)
 -        .enableThreadPoolMetrics(enableMetrics).build();
 -
 -    defaultSplitThreadPool = ThreadPools.getServerThreadPools().getPoolBuilder("md splitter")
 -        .numCoreThreads(0).numMaxThreads(1).withTimeOut(60, SECONDS)
 -        .enableThreadPoolMetrics(enableMetrics).build();
 -
 -    defaultMigrationPool = ThreadPools.getServerThreadPools()
 -        .getPoolBuilder("metadata tablet migration").numCoreThreads(0).numMaxThreads(1)
 -        .withTimeOut(60, SECONDS).enableThreadPoolMetrics(enableMetrics).build();
 +    defaultMigrationPool =
 +        ThreadPools.getServerThreadPools().getPoolBuilder("metadata tablet migration")
 +            .numCoreThreads(0).numMaxThreads(1).withTimeOut(60L, SECONDS).build();
  
      migrationPool = ThreadPools.getServerThreadPools().createExecutorService(acuConf,
-         Property.TSERV_MIGRATE_MAXCONCURRENT, true);
+         Property.TSERV_MIGRATE_MAXCONCURRENT, enableMetrics);
      modifyThreadPoolSizesAtRuntime(
          () -> context.getConfiguration().getCount(Property.TSERV_MIGRATE_MAXCONCURRENT),
          "tablet migration", migrationPool);
diff --cc test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
index 0bacb4fd95,4b92480f19..5ed78da2ca
--- a/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/metrics/MetricsIT.java
@@@ -48,9 -43,9 +48,10 @@@ import org.apache.accumulo.core.conf.Pr
  import org.apache.accumulo.core.data.Mutation;
  import org.apache.accumulo.core.data.Value;
  import org.apache.accumulo.core.metrics.MetricsProducer;
+ import org.apache.accumulo.core.spi.metrics.LoggingMeterRegistryFactory;
  import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
  import org.apache.accumulo.test.functional.ConfigurableMacBase;
 +import org.apache.accumulo.test.functional.SlowIterator;
  import org.apache.accumulo.test.metrics.TestStatsDSink.Metric;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.io.Text;
@@@ -84,13 -79,15 +85,16 @@@ public class MetricsIT extends Configur
      cfg.setProperty(Property.GC_CYCLE_START, "1s");
      cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
      cfg.setProperty(Property.MANAGER_FATE_METRICS_MIN_UPDATE_INTERVAL, "1s");
-     cfg.setProperty(Property.GENERAL_MICROMETER_CACHE_METRICS_ENABLED, "true");
-     cfg.setProperty(Property.GENERAL_MICROMETER_USER_TAGS, "tag1=value1,tag2=value2");
- 
-     // Tell the server processes to use a StatsDMeterRegistry that will be configured
-     // to push all metrics to the sink we started.
+     // Tell the server processes to use a StatsDMeterRegistry and the simple logging registry
+     // that will be configured to push all metrics to the sink we started.
      cfg.setProperty(Property.GENERAL_MICROMETER_ENABLED, "true");
-     cfg.setProperty(Property.GENERAL_MICROMETER_FACTORY, TestStatsDRegistryFactory.class.getName());
++    cfg.setProperty(Property.GENERAL_MICROMETER_USER_TAGS, "tag1=value1,tag2=value2");
++    cfg.setProperty(Property.GENERAL_MICROMETER_CACHE_METRICS_ENABLED, "true");
+     cfg.setProperty(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED, "true");
 -    cfg.setProperty("general.custom.metrics.opts.logging.step", "1s");
++    cfg.setProperty("general.custom.metrics.opts.logging.step", "10s");
+     String clazzList = LoggingMeterRegistryFactory.class.getName() + ","
+         + TestStatsDRegistryFactory.class.getName();
+     cfg.setProperty(Property.GENERAL_MICROMETER_FACTORY, clazzList);
      Map<String,String> sysProps = Map.of(TestStatsDRegistryFactory.SERVER_HOST, "127.0.0.1",
          TestStatsDRegistryFactory.SERVER_PORT, Integer.toString(sink.getPort()));
      cfg.setSystemProperties(sysProps);
@@@ -99,15 -96,13 +103,13 @@@
    @Test
    public void confirmMetricsPublished() throws Exception {
  
 -    doWorkToGenerateMetrics();
 -    cluster.stop();
 -
      Set<String> unexpectedMetrics = Set.of(METRICS_SCAN_YIELDS, METRICS_UPDATE_ERRORS,
 -        METRICS_COMPACTOR_MAJC_STUCK, METRICS_SCAN_BUSY_TIMEOUT, METRICS_SCAN_PAUSED_FOR_MEM,
 -        METRICS_SCAN_RETURN_FOR_MEM, METRICS_MINC_PAUSED, METRICS_MAJC_PAUSED);
 -    Set<String> flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, METRICS_FATE_TYPE_IN_PROGRESS);
 +        METRICS_SCAN_BUSY_TIMEOUT, METRICS_SCAN_PAUSED_FOR_MEM, METRICS_SCAN_RETURN_FOR_MEM,
 +        METRICS_MINC_PAUSED, METRICS_MAJC_PAUSED, METRICS_SERVER_IDLE);
 +    Set<String> flakyMetrics = Set.of(METRICS_GC_WAL_ERRORS, METRICS_FATE_TYPE_IN_PROGRESS,
-         METRICS_PROPSTORE_EVICTION_COUNT, METRICS_PROPSTORE_REFRESH_COUNT,
-         METRICS_PROPSTORE_REFRESH_LOAD_COUNT, METRICS_PROPSTORE_ZK_ERROR_COUNT,
 +        METRICS_TSERVER_TABLETS_ONLINE_ONDEMAND, METRICS_TSERVER_TABLETS_ONDEMAND_UNLOADED_FOR_MEM,
 +        METRICS_COMPACTOR_MAJC_STUCK, METRICS_MANAGER_ROOT_TGW_ERRORS,
 +        METRICS_MANAGER_META_TGW_ERRORS, METRICS_MANAGER_USER_TGW_ERRORS);
  
      Map<String,String> expectedMetricNames = this.getMetricFields();
      flakyMetrics.forEach(expectedMetricNames::remove); // might not see these
@@@ -152,29 -125,11 +154,36 @@@
              } else if (flakyMetrics.contains(name)) {
                // ignore any flaky metric names seen
                // these aren't always expected, but we shouldn't be surprised if we see them
 +            } else if (name.startsWith(METRICS_COMPACTOR_PREFIX)) {
 +              // Compactor queue metrics are not guaranteed to be emitted
 +              // during the call to doWorkToGenerateMetrics above. This will
 +              // flip a bit in the BitSet when each metric is seen. The top-level
 +              // loop will continue to iterate until all the metrics are seen.
 +              seenMetricNames.put(name, expectedMetricNames.remove(name));
-               if (METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH.equals(name)) {
-                 queueMetricsSeen.set(compactionPriorityQueueLengthBit, true);
-               } else if (METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED.equals(name)) {
-                 queueMetricsSeen.set(compactionPriorityQueueQueuedBit, true);
-               } else if (METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED.equals(name)) {
-                 queueMetricsSeen.set(compactionPriorityQueueDequeuedBit, true);
-               } else if (METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED.equals(name)) {
-                 queueMetricsSeen.set(compactionPriorityQueueRejectedBit, true);
-               } else if (METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY.equals(name)) {
-                 queueMetricsSeen.set(compactionPriorityQueuePriorityBit, true);
++              switch (name) {
++                case METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH:
++                  queueMetricsSeen.set(compactionPriorityQueueLengthBit, true);
++                  break;
++                case METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED:
++                  queueMetricsSeen.set(compactionPriorityQueueQueuedBit, true);
++                  break;
++                case METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED:
++                  queueMetricsSeen.set(compactionPriorityQueueDequeuedBit, true);
++                  break;
++                case METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED:
++                  queueMetricsSeen.set(compactionPriorityQueueRejectedBit, true);
++                  break;
++                case METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY:
++                  queueMetricsSeen.set(compactionPriorityQueuePriorityBit, true);
++                  break;
 +              }
              } else {
                // completely unexpected metric
                fail("Found accumulo metric not in expectedMetricNames or flakyMetricNames: " + name);
              }
            });
++      log.debug("METRICS: metrics expected, but not seen so far: {}", expectedMetricNames);
 +      Thread.sleep(4_000);
      }
      assertTrue(expectedMetricNames.isEmpty(),
          "Did not see all expected metric names, missing: " + expectedMetricNames.values());
@@@ -249,7 -191,7 +258,8 @@@
        statsDMetrics.stream().filter(line -> line.startsWith("accumulo"))
            .map(TestStatsDSink::parseStatsDMetric).forEach(a -> {
              var t = a.getTags();
--            log.trace("METRICS, name: '{}' num tags: {}, tags: {}", a.getName(), t.size(), t);
++            log.trace("METRICS, received from statsd - name: '{}' num tags: {}, tags: {} = {}",
++                a.getName(), t.size(), t, a.getValue());
              // check hostname is always set and is valid
              assertNotEquals("0.0.0.0", a.getTags().get("host"));
              assertNotNull(a.getTags().get("instance.name"));
diff --cc test/src/main/resources/log4j2-test.properties
index 7a92825296,0c77a3871b..1df47e471b
--- a/test/src/main/resources/log4j2-test.properties
+++ b/test/src/main/resources/log4j2-test.properties
@@@ -142,31 -142,5 +142,8 @@@ logger.38.level = debu
  logger.39.name = org.apache.accumulo.manager.Manager
  logger.39.level = trace
  
 +logger.40.name = org.apache.accumulo.tablet
 +logger.40.level = trace
 +
- property.metricsFilename = ./target/test-metrics
- 
- # appender.metrics.type = Console
- appender.metrics.type = RollingFile
- appender.metrics.name = LoggingMetricsOutput
- appender.metrics.fileName = ${metricsFilename}.metrics
- appender.metrics.filePattern = ${metricsFilename}-%d{MM-dd-yy-HH}-%i.metrics.gz
- appender.metrics.layout.type = PatternLayout
- appender.metrics.layout.pattern = METRICS: %d{ISO8601}, %m%n
- appender.metrics.policies.type = Policies
- appender.metrics.policies.time.type = TimeBasedTriggeringPolicy
- appender.metrics.policies.time.interval = 1
- appender.metrics.policies.time.modulate = true
- appender.metrics.policies.size.type = SizeBasedTriggeringPolicy
- appender.metrics.policies.size.size=100MB
- appender.metrics.strategy.type = DefaultRolloverStrategy
- appender.metrics.strategy.max = 10
- 
- logger.metrics.name = org.apache.accumulo.metrics
- logger.metrics.level = info
- logger.metrics.additivity = false
- logger.metrics.appenderRef.metrics.ref = LoggingMetricsOutput
- 
  rootLogger.level = debug
  rootLogger.appenderRef.console.ref = STDOUT