You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by em...@apache.org on 2024/11/15 22:01:34 UTC

(polaris) branch main updated: Create a new singleton metric with an API_NAME tag (#431)

This is an automated email from the ASF dual-hosted git repository.

emaynard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git


The following commit(s) were added to refs/heads/main by this push:
     new 877959fb Create a new singleton metric with an API_NAME tag (#431)
877959fb is described below

commit 877959fb4993bdb2fe638bff3c80c123ddd73f11
Author: Andrew Guterman <an...@gmail.com>
AuthorDate: Fri Nov 15 14:01:31 2024 -0800

    Create a new singleton metric with an API_NAME tag (#431)
    
    * Emit metrics with the API name as a tag
    
    * Add tests
    
    * slight cleanup
    
    * Improve comments/readability
    
    * Switch to snake_case, start deprecation
    
    * Remove debug line
    
    * Spotless toggle for deprecated tags
    
    * Slightly scope down spotless toggle
    
    * Fix test assumption on snowman principal name
    
    * Fix RateLimiterFilterTest to use TestMetricsUtil
    
    ---------
    
    Co-authored-by: Eric Maynard <er...@snowflake.com>
---
 .../src/main/kotlin/polaris-java.gradle.kts        |   1 +
 .../core/monitor/PolarisMetricRegistry.java        | 101 ++++++++-
 .../service/TimedApplicationEventListener.java     |  29 +++
 .../service/TimedApplicationEventListenerTest.java | 234 +++++++++++++++++++++
 .../service/ratelimiter/RateLimiterFilterTest.java |  33 ++-
 .../polaris/service/test/TestMetricsUtil.java      |  70 ++++++
 6 files changed, 442 insertions(+), 26 deletions(-)

diff --git a/build-logic/src/main/kotlin/polaris-java.gradle.kts b/build-logic/src/main/kotlin/polaris-java.gradle.kts
index d6992225..a5f8e35e 100644
--- a/build-logic/src/main/kotlin/polaris-java.gradle.kts
+++ b/build-logic/src/main/kotlin/polaris-java.gradle.kts
@@ -80,6 +80,7 @@ spotless {
     licenseHeaderFile(rootProject.file("codestyle/copyright-header-java.txt"))
     endWithNewline()
     custom("disallowWildcardImports", disallowWildcardImports)
+    toggleOffOn()
   }
   kotlinGradle {
     ktfmt().googleStyle()
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java b/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java
index 20fae4fd..7f09353c 100644
--- a/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java
+++ b/polaris-core/src/main/java/org/apache/polaris/core/monitor/PolarisMetricRegistry.java
@@ -20,6 +20,7 @@ package org.apache.polaris.core.monitor;
 
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.MeterRegistry;
+import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Timer;
 import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics;
 import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics;
@@ -27,26 +28,45 @@ import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics;
 import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics;
 import io.micrometer.core.instrument.binder.system.ProcessorMetrics;
 import java.lang.reflect.Method;
+import java.util.Collections;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import org.apache.polaris.core.resource.TimedApi;
+import org.jetbrains.annotations.VisibleForTesting;
 
 /**
  * Wrapper around the Micrometer {@link MeterRegistry} providing additional metric management
  * functions for the Polaris application. Implements in-memory caching of timers and counters.
  * Records two metrics for each instrument with one tagged by the realm ID (realm-specific metric)
  * and one without. The realm-specific metric is suffixed with ".realm".
+ *
+ * <p>Uppercase tag names are now deprecated. Prefer snake_casing instead. Old metrics are emitted
+ * with both variations but the uppercase versions may eventually be removed. New methods for tag
+ * emission (those that allow you to pass in an Iterable<Tag>) only emit the snake_case version.
  */
 public class PolarisMetricRegistry {
   private final MeterRegistry meterRegistry;
   private final ConcurrentMap<String, Timer> timers = new ConcurrentHashMap<>();
   private final ConcurrentMap<String, Counter> counters = new ConcurrentHashMap<>();
-  private static final String TAG_REALM = "REALM_ID";
-  private static final String TAG_RESP_CODE = "HTTP_RESPONSE_CODE";
-  private static final String SUFFIX_COUNTER = ".count";
-  private static final String SUFFIX_ERROR = ".error";
-  private static final String SUFFIX_REALM = ".realm";
+
+  /**
+   * @deprecated See class Javadoc.
+   */
+  public static final String TAG_REALM_DEPRECATED = "REALM_ID";
+
+  public static final String TAG_REALM = "realm_id";
+
+  /**
+   * @deprecated See class Javadoc.
+   */
+  public static final String TAG_RESP_CODE_DEPRECATED = "HTTP_RESPONSE_CODE";
+
+  public static final String TAG_RESP_CODE = "http_response_code";
+
+  public static final String SUFFIX_COUNTER = ".count";
+  public static final String SUFFIX_ERROR = ".error";
+  public static final String SUFFIX_REALM = ".realm";
 
   public PolarisMetricRegistry(MeterRegistry meterRegistry) {
     this.meterRegistry = meterRegistry;
@@ -61,6 +81,12 @@ public class PolarisMetricRegistry {
     return meterRegistry;
   }
 
+  @VisibleForTesting
+  public void clear() {
+    meterRegistry.clear();
+    counters.clear();
+  }
+
   public void init(Class<?>... classes) {
     for (Class<?> clazz : classes) {
       Method[] methods = clazz.getDeclaredMethods();
@@ -75,8 +101,12 @@ public class PolarisMetricRegistry {
 
           // Error counters contain the HTTP response code in a tag, thus caching them would not be
           // meaningful.
-          Counter.builder(metric + SUFFIX_ERROR).tags(TAG_RESP_CODE, "400").register(meterRegistry);
-          Counter.builder(metric + SUFFIX_ERROR).tags(TAG_RESP_CODE, "500").register(meterRegistry);
+          Counter.builder(metric + SUFFIX_ERROR)
+              .tags(TAG_RESP_CODE, "400", TAG_RESP_CODE_DEPRECATED, "400")
+              .register(meterRegistry);
+          Counter.builder(metric + SUFFIX_ERROR)
+              .tags(TAG_RESP_CODE, "500", TAG_RESP_CODE_DEPRECATED, "500")
+              .register(meterRegistry);
         }
       }
     }
@@ -93,10 +123,15 @@ public class PolarisMetricRegistry {
             m ->
                 Timer.builder(metric + SUFFIX_REALM)
                     .tag(TAG_REALM, realmId)
+                    .tag(TAG_REALM_DEPRECATED, realmId)
                     .register(meterRegistry));
     timerRealm.record(elapsedTimeMs, TimeUnit.MILLISECONDS);
   }
 
+  /**
+   * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric.
+   * Counters are cached.
+   */
   public void incrementCounter(String metric, String realmId) {
     String counterMetric = metric + SUFFIX_COUNTER;
     Counter counter =
@@ -110,21 +145,73 @@ public class PolarisMetricRegistry {
             m ->
                 Counter.builder(counterMetric + SUFFIX_REALM)
                     .tag(TAG_REALM, realmId)
+                    .tag(TAG_REALM_DEPRECATED, realmId)
                     .register(meterRegistry));
     counterRealm.increment();
   }
 
+  /**
+   * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric.
+   * Both metrics are tagged with the statusCode and counters are not cached.
+   */
   public void incrementErrorCounter(String metric, int statusCode, String realmId) {
     String errorMetric = metric + SUFFIX_ERROR;
     Counter.builder(errorMetric)
         .tag(TAG_RESP_CODE, String.valueOf(statusCode))
+        .tag(TAG_RESP_CODE_DEPRECATED, String.valueOf(statusCode))
         .register(meterRegistry)
         .increment();
 
     Counter.builder(errorMetric + SUFFIX_REALM)
         .tag(TAG_RESP_CODE, String.valueOf(statusCode))
+        .tag(TAG_RESP_CODE_DEPRECATED, String.valueOf(statusCode))
+        .tag(TAG_REALM, realmId)
+        .tag(TAG_REALM_DEPRECATED, realmId)
+        .register(meterRegistry)
+        .increment();
+  }
+
+  /**
+   * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric.
+   * Arbitrary tags can be specified and counters are not cached.
+   */
+  public void incrementCounter(String metric, String realmId, Iterable<Tag> tags) {
+    Counter.builder(metric + SUFFIX_COUNTER).tags(tags).register(meterRegistry).increment();
+
+    Counter.builder(metric + SUFFIX_COUNTER + SUFFIX_REALM)
+        .tags(tags)
+        .tag(TAG_REALM, realmId)
+        .register(meterRegistry)
+        .increment();
+  }
+
+  /**
+   * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric.
+   * Arbitrary tags can be specified and counters are not cached.
+   */
+  public void incrementCounter(String metric, String realmId, Tag tag) {
+    incrementCounter(metric, realmId, Collections.singleton(tag));
+  }
+
+  /**
+   * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric.
+   * Arbitrary tags can be specified and counters are not cached.
+   */
+  public void incrementErrorCounter(String metric, String realmId, Iterable<Tag> tags) {
+    Counter.builder(metric + SUFFIX_ERROR).tags(tags).register(meterRegistry).increment();
+
+    Counter.builder(metric + SUFFIX_ERROR + SUFFIX_REALM)
+        .tags(tags)
         .tag(TAG_REALM, realmId)
         .register(meterRegistry)
         .increment();
   }
+
+  /**
+   * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric.
+   * Arbitrary tags can be specified and counters are not cached.
+   */
+  public void incrementErrorCounter(String metric, String realmId, Tag tag) {
+    incrementErrorCounter(metric, realmId, Collections.singleton(tag));
+  }
 }
diff --git a/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java b/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java
index e4ab8afd..deaa0bba 100644
--- a/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java
+++ b/polaris-service/src/main/java/org/apache/polaris/service/TimedApplicationEventListener.java
@@ -18,8 +18,12 @@
  */
 package org.apache.polaris.service;
 
+import static org.apache.polaris.core.monitor.PolarisMetricRegistry.TAG_RESP_CODE;
+
 import com.google.common.base.Stopwatch;
+import io.micrometer.core.instrument.Tag;
 import java.lang.reflect.Method;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import javax.ws.rs.ext.Provider;
 import org.apache.polaris.core.context.CallContext;
@@ -29,6 +33,7 @@ import org.glassfish.jersey.server.monitoring.ApplicationEvent;
 import org.glassfish.jersey.server.monitoring.ApplicationEventListener;
 import org.glassfish.jersey.server.monitoring.RequestEvent;
 import org.glassfish.jersey.server.monitoring.RequestEventListener;
+import org.jetbrains.annotations.VisibleForTesting;
 
 /**
  * An ApplicationEventListener that supports timing and error counting of Jersey resource methods
@@ -38,6 +43,14 @@ import org.glassfish.jersey.server.monitoring.RequestEventListener;
 @Provider
 public class TimedApplicationEventListener implements ApplicationEventListener {
 
+  /**
+   * Each API will increment a common counter (SINGLETON_METRIC_NAME) but have its API name tagged
+   * (TAG_API_NAME).
+   */
+  public static final String SINGLETON_METRIC_NAME = "polaris.api";
+
+  public static final String TAG_API_NAME = "api_name";
+
   // The PolarisMetricRegistry instance used for recording metrics and error counters.
   private final PolarisMetricRegistry polarisMetricRegistry;
 
@@ -45,6 +58,11 @@ public class TimedApplicationEventListener implements ApplicationEventListener {
     this.polarisMetricRegistry = polarisMetricRegistry;
   }
 
+  @VisibleForTesting
+  public PolarisMetricRegistry getMetricRegistry() {
+    return polarisMetricRegistry;
+  }
+
   @Override
   public void onEvent(ApplicationEvent event) {}
 
@@ -72,7 +90,11 @@ public class TimedApplicationEventListener implements ApplicationEventListener {
         if (method.isAnnotationPresent(TimedApi.class)) {
           TimedApi timedApi = method.getAnnotation(TimedApi.class);
           metric = timedApi.value();
+
+          // Increment both the counter with the API name in the metric name and a common metric
           polarisMetricRegistry.incrementCounter(metric, realmId);
+          polarisMetricRegistry.incrementCounter(
+              SINGLETON_METRIC_NAME, realmId, Tag.of(TAG_API_NAME, metric));
         }
       } else if (event.getType() == RequestEvent.Type.RESOURCE_METHOD_START && metric != null) {
         sw = Stopwatch.createStarted();
@@ -82,7 +104,14 @@ public class TimedApplicationEventListener implements ApplicationEventListener {
           polarisMetricRegistry.recordTimer(metric, sw.elapsed(TimeUnit.MILLISECONDS), realmId);
         } else {
           int statusCode = event.getContainerResponse().getStatus();
+
+          // Increment both the counter with the API name in the metric name and a common metric
           polarisMetricRegistry.incrementErrorCounter(metric, statusCode, realmId);
+          polarisMetricRegistry.incrementErrorCounter(
+              SINGLETON_METRIC_NAME,
+              realmId,
+              List.of(
+                  Tag.of(TAG_API_NAME, metric), Tag.of(TAG_RESP_CODE, String.valueOf(statusCode))));
         }
       }
     }
diff --git a/polaris-service/src/test/java/org/apache/polaris/service/TimedApplicationEventListenerTest.java b/polaris-service/src/test/java/org/apache/polaris/service/TimedApplicationEventListenerTest.java
new file mode 100644
index 00000000..d5369a7a
--- /dev/null
+++ b/polaris-service/src/test/java/org/apache/polaris/service/TimedApplicationEventListenerTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service;
+
+import static org.apache.polaris.core.monitor.PolarisMetricRegistry.*;
+import static org.apache.polaris.service.TimedApplicationEventListener.SINGLETON_METRIC_NAME;
+import static org.apache.polaris.service.TimedApplicationEventListener.TAG_API_NAME;
+import static org.apache.polaris.service.context.DefaultContextResolver.REALM_PROPERTY_KEY;
+
+import io.dropwizard.testing.ConfigOverride;
+import io.dropwizard.testing.ResourceHelpers;
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import io.micrometer.core.instrument.Tag;
+import jakarta.ws.rs.core.Response;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.polaris.core.monitor.PolarisMetricRegistry;
+import org.apache.polaris.core.resource.TimedApi;
+import org.apache.polaris.service.admin.api.PolarisPrincipalsApi;
+import org.apache.polaris.service.config.PolarisApplicationConfig;
+import org.apache.polaris.service.test.PolarisConnectionExtension;
+import org.apache.polaris.service.test.PolarisRealm;
+import org.apache.polaris.service.test.SnowmanCredentialsExtension;
+import org.apache.polaris.service.test.TestEnvironmentExtension;
+import org.apache.polaris.service.test.TestMetricsUtil;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith({
+  DropwizardExtensionsSupport.class,
+  TestEnvironmentExtension.class,
+  PolarisConnectionExtension.class,
+  SnowmanCredentialsExtension.class
+})
+public class TimedApplicationEventListenerTest {
+  private static final DropwizardAppExtension<PolarisApplicationConfig> EXT =
+      new DropwizardAppExtension<>(
+          PolarisApplication.class,
+          ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"),
+          ConfigOverride.config(
+              "server.applicationConnectors[0].port",
+              "0"), // Bind to random port to support parallelism
+          ConfigOverride.config(
+              "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism
+
+  private static final int ERROR_CODE = Response.Status.NOT_FOUND.getStatusCode();
+  private static final String ENDPOINT = "api/management/v1/principals";
+  private static final String API_ANNOTATION =
+      Arrays.stream(PolarisPrincipalsApi.class.getMethods())
+          .filter(m -> m.getName().contains("getPrincipal"))
+          .findFirst()
+          .orElseThrow()
+          .getAnnotation(TimedApi.class)
+          .value();
+
+  private static PolarisConnectionExtension.PolarisToken userToken;
+  private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials;
+  private static String realm;
+
+  @BeforeAll
+  public static void setup(
+      PolarisConnectionExtension.PolarisToken userToken,
+      SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials,
+      @PolarisRealm String realm)
+      throws IOException {
+    TimedApplicationEventListenerTest.userToken = userToken;
+    TimedApplicationEventListenerTest.snowmanCredentials = snowmanCredentials;
+    TimedApplicationEventListenerTest.realm = realm;
+  }
+
+  @BeforeEach
+  public void clearMetrics() {
+    getPolarisMetricRegistry().clear();
+  }
+
+  @Test
+  public void testMetricsEmittedOnSuccessfulRequest() {
+    sendSuccessfulRequest();
+    Assertions.assertTrue(getPerApiMetricCount() > 0);
+    Assertions.assertTrue(getPerApiRealmMetricCount() > 0);
+    Assertions.assertTrue(getCommonMetricCount() > 0);
+    Assertions.assertTrue(getCommonRealmMetricCount() > 0);
+    Assertions.assertEquals(0, getPerApiMetricErrorCount());
+    Assertions.assertEquals(0, getPerApiRealmMetricErrorCount());
+    Assertions.assertEquals(0, getCommonMetricErrorCount());
+    Assertions.assertEquals(0, getCommonRealmMetricErrorCount());
+  }
+
+  @Test
+  public void testMetricsEmittedOnFailedRequest() {
+    sendFailingRequest();
+    Assertions.assertTrue(getPerApiMetricCount() > 0);
+    Assertions.assertTrue(getPerApiRealmMetricCount() > 0);
+    Assertions.assertTrue(getCommonMetricCount() > 0);
+    Assertions.assertTrue(getCommonRealmMetricCount() > 0);
+    Assertions.assertTrue(getPerApiMetricErrorCount() > 0);
+    Assertions.assertTrue(getPerApiRealmMetricErrorCount() > 0);
+    Assertions.assertTrue(getCommonMetricErrorCount() > 0);
+    Assertions.assertTrue(getCommonRealmMetricErrorCount() > 0);
+  }
+
+  private PolarisMetricRegistry getPolarisMetricRegistry() {
+    TimedApplicationEventListener listener =
+        (TimedApplicationEventListener)
+            EXT.getEnvironment().jersey().getResourceConfig().getSingletons().stream()
+                .filter(
+                    s ->
+                        TimedApplicationEventListener.class
+                            .getName()
+                            .equals(s.getClass().getName()))
+                .findAny()
+                .orElseThrow();
+    return listener.getMetricRegistry();
+  }
+
+  private double getPerApiMetricCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT, API_ANNOTATION + SUFFIX_COUNTER, Collections.emptyList());
+  }
+
+  private double getPerApiRealmMetricCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT,
+        API_ANNOTATION + SUFFIX_COUNTER + SUFFIX_REALM,
+        List.of(
+            Tag.of(TAG_REALM, realm),
+            // spotless:off
+            Tag.of(TAG_REALM_DEPRECATED, realm)));
+            // spotless:on
+  }
+
+  private double getPerApiMetricErrorCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT,
+        API_ANNOTATION + SUFFIX_ERROR,
+        List.of(
+            Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)),
+            // spotless:off
+            Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE))));
+            // spotless:on
+  }
+
+  private double getPerApiRealmMetricErrorCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT,
+        API_ANNOTATION + SUFFIX_ERROR + SUFFIX_REALM,
+        List.of(
+            Tag.of(TAG_REALM, realm),
+            Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)),
+            // spotless:off
+            Tag.of(TAG_REALM_DEPRECATED, realm),
+            Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE))));
+            // spotless:on
+  }
+
+  private double getCommonMetricCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT,
+        SINGLETON_METRIC_NAME + SUFFIX_COUNTER,
+        Collections.singleton(Tag.of(TAG_API_NAME, API_ANNOTATION)));
+  }
+
+  private double getCommonRealmMetricCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT,
+        SINGLETON_METRIC_NAME + SUFFIX_COUNTER + SUFFIX_REALM,
+        List.of(Tag.of(TAG_API_NAME, API_ANNOTATION), Tag.of(TAG_REALM, realm)));
+  }
+
+  private double getCommonMetricErrorCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT,
+        SINGLETON_METRIC_NAME + SUFFIX_ERROR,
+        List.of(
+            Tag.of(TAG_API_NAME, API_ANNOTATION),
+            Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE))));
+  }
+
+  private double getCommonRealmMetricErrorCount() {
+    return TestMetricsUtil.getTotalCounter(
+        EXT,
+        SINGLETON_METRIC_NAME + SUFFIX_ERROR + SUFFIX_REALM,
+        List.of(
+            Tag.of(TAG_API_NAME, API_ANNOTATION),
+            Tag.of(TAG_REALM, realm),
+            Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE))));
+  }
+
+  private int sendRequest(String principalName) {
+    try (Response response =
+        EXT.client()
+            .target(
+                String.format(
+                    "http://localhost:%d/%s/%s", EXT.getLocalPort(), ENDPOINT, principalName))
+            .request("application/json")
+            .header("Authorization", "Bearer " + userToken.token())
+            .header(REALM_PROPERTY_KEY, realm)
+            .get()) {
+      return response.getStatus();
+    }
+  }
+
+  private void sendSuccessfulRequest() {
+    Assertions.assertEquals(
+        Response.Status.OK.getStatusCode(),
+        sendRequest(snowmanCredentials.identifier().principalName()));
+  }
+
+  private void sendFailingRequest() {
+    Assertions.assertEquals(ERROR_CODE, sendRequest("notarealprincipal"));
+  }
+}
diff --git a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java
index 90c72a7d..772fc6aa 100644
--- a/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java
+++ b/polaris-service/src/test/java/org/apache/polaris/service/ratelimiter/RateLimiterFilterTest.java
@@ -18,15 +18,19 @@
  */
 package org.apache.polaris.service.ratelimiter;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.apache.polaris.core.monitor.PolarisMetricRegistry.*;
+import static org.apache.polaris.service.TimedApplicationEventListener.SINGLETON_METRIC_NAME;
+import static org.apache.polaris.service.TimedApplicationEventListener.TAG_API_NAME;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import io.dropwizard.testing.ConfigOverride;
 import io.dropwizard.testing.ResourceHelpers;
 import io.dropwizard.testing.junit5.DropwizardAppExtension;
 import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import io.micrometer.core.instrument.Tag;
 import jakarta.ws.rs.core.Response;
 import java.time.Duration;
+import java.util.List;
 import java.util.function.Consumer;
 import org.apache.polaris.service.PolarisApplication;
 import org.apache.polaris.service.config.PolarisApplicationConfig;
@@ -34,6 +38,7 @@ import org.apache.polaris.service.test.PolarisConnectionExtension;
 import org.apache.polaris.service.test.PolarisRealm;
 import org.apache.polaris.service.test.SnowmanCredentialsExtension;
 import org.apache.polaris.service.test.TestEnvironmentExtension;
+import org.apache.polaris.service.test.TestMetricsUtil;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -109,24 +114,14 @@ public class RateLimiterFilterTest {
     requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS);
 
     assertTrue(
-        getCounter(
-                "polaris_principal_roles_listPrincipalRoles_error_total{HTTP_RESPONSE_CODE=\"429\"}")
+        TestMetricsUtil.getTotalCounter(
+                EXT,
+                SINGLETON_METRIC_NAME + SUFFIX_ERROR,
+                List.of(
+                    Tag.of(TAG_API_NAME, "polaris.principal-roles.listPrincipalRoles"),
+                    Tag.of(
+                        TAG_RESP_CODE,
+                        String.valueOf(Response.Status.TOO_MANY_REQUESTS.getStatusCode()))))
             > 0);
   }
-
-  private double getCounter(String metricName) {
-    Response response =
-        EXT.client()
-            .target(String.format("http://localhost:%d/metrics", EXT.getAdminPort()))
-            .request()
-            .get();
-    assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
-    String[] responseLines = response.readEntity(String.class).split("\n");
-    for (String line : responseLines) {
-      if (line.startsWith(metricName)) {
-        return Double.parseDouble(line.split(" ")[1]);
-      }
-    }
-    return 0;
-  }
 }
diff --git a/polaris-service/src/test/java/org/apache/polaris/service/test/TestMetricsUtil.java b/polaris-service/src/test/java/org/apache/polaris/service/test/TestMetricsUtil.java
new file mode 100644
index 00000000..f76de83f
--- /dev/null
+++ b/polaris-service/src/test/java/org/apache/polaris/service/test/TestMetricsUtil.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.test;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.micrometer.core.instrument.Tag;
+import jakarta.ws.rs.core.Response;
+import java.util.Collection;
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.polaris.service.config.PolarisApplicationConfig;
+
+/** Utils for working with metrics in tests */
+public class TestMetricsUtil {
+  private static final String SUFFIX_TOTAL = ".total";
+
+  /** Gets a total counter by calling the Prometheus metrics endpoint */
+  public static double getTotalCounter(
+      DropwizardAppExtension<PolarisApplicationConfig> dropwizardAppExtension,
+      String metricName,
+      Collection<Tag> tags) {
+
+    metricName += SUFFIX_TOTAL;
+    metricName = metricName.replace('.', '_').replace('-', '_');
+
+    // Example of a line from the metrics endpoint:
+    // polaris_TimedApi_count_realm_total{API_NAME="polaris.principals.getPrincipal",REALM_ID="org_apache_polaris_service_TimedApplicationEventListenerTest"} 1.0
+    // This method assumes that tag ordering isn't guaranteed
+    List<String> tagFilters =
+        tags.stream().map(tag -> String.format("%s=\"%s\"", tag.getKey(), tag.getValue())).toList();
+
+    Response response =
+        dropwizardAppExtension
+            .client()
+            .target(
+                String.format("http://localhost:%d/metrics", dropwizardAppExtension.getAdminPort()))
+            .request()
+            .get();
+    assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+    String[] responseLines = response.readEntity(String.class).split("\n");
+    for (String line : responseLines) {
+      int numTags =
+          StringUtils.countMatches(line, '='); // Assumes the tag values don't contain an '='
+      if (line.startsWith(metricName)
+          && tagFilters.stream().allMatch(line::contains)
+          && numTags == tagFilters.size()) {
+        return Double.parseDouble(line.split(" ")[1]);
+      }
+    }
+    return 0;
+  }
+}