You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2020/10/09 10:42:19 UTC
[james-project] 01/05: JAMES-3405 Expose metrics over HTTP
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit bdfc7acf4073047fe823f867c242f2329ee06f4f
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Tue Oct 6 14:21:50 2020 +0700
JAMES-3405 Expose metrics over HTTP
---
CHANGELOG.md | 1 +
.../dropwizard/DropWizardMetricFactory.java | 1 +
pom.xml | 5 ++
.../modules/server/DropWizardMetricsModule.java | 2 +-
server/container/guice/protocols/webadmin/pom.xml | 5 ++
.../james/modules/server/MetricsRoutesModule.java | 38 +++++++++
.../james/modules/server/WebAdminServerModule.java | 1 +
.../integration/AuthorizedEndpointsTest.java | 8 ++
.../integration/WebAdminServerIntegrationTest.java | 13 ++++
server/protocols/webadmin/pom.xml | 1 +
.../webadmin/webadmin-dropwizard-metrics}/pom.xml | 54 ++++++++++---
.../james/webadmin/dropwizard/MetricsRoutes.java | 89 ++++++++++++++++++++++
.../webadmin/dropwizard/MetricsRoutesTest.java} | 54 ++++++++-----
13 files changed, 238 insertions(+), 34 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d3e97c7..72e2314 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
- JAMES-3296 Webadmin endpoint to rebuild RabbitMQMailQueue in the Distributed Server
- JAMES-3266 Offer an option to disable ElasticSearch in Distributed James product
- JAMES-3202 Reindex only outdated documents with the Mode option set to CORRECT in reindexing tasks
+- JAMES-3405 Expose metrics of Guice servers over HTTP - enables easy Prometheus metrics collection
### Changed
- Switch to Java 11 for build and run
diff --git a/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetricFactory.java b/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetricFactory.java
index 8a87b88..263dcee 100644
--- a/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetricFactory.java
+++ b/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetricFactory.java
@@ -81,5 +81,6 @@ public class DropWizardMetricFactory implements MetricFactory, Startable {
@PreDestroy
public void stop() {
jmxReporter.stop();
+ metricRegistry.removeMatching((name, metric) -> true);
}
}
diff --git a/pom.xml b/pom.xml
index 3ffa5b4..236b695 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1847,6 +1847,11 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>james-server-webadmin-dropwizard-metrics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>james-server-webadmin-integration-test-common</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java b/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java
index 612dd97..f585ebf 100644
--- a/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java
+++ b/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java
@@ -37,12 +37,12 @@ public class DropWizardMetricsModule extends AbstractModule {
@Override
protected void configure() {
install(new LoggingMetricsModule());
- bind(MetricRegistry.class).in(Scopes.SINGLETON);
bind(DropWizardMetricFactory.class).in(Scopes.SINGLETON);
bind(DropWizardGaugeRegistry.class).in(Scopes.SINGLETON);
bind(DropWizardJVMMetrics.class).in(Scopes.SINGLETON);
bind(MetricFactory.class).to(DropWizardMetricFactory.class);
+ bind(MetricRegistry.class).toInstance(new MetricRegistry());
bind(GaugeRegistry.class).to(DropWizardGaugeRegistry.class);
}
diff --git a/server/container/guice/protocols/webadmin/pom.xml b/server/container/guice/protocols/webadmin/pom.xml
index d383898..cedd56d 100644
--- a/server/container/guice/protocols/webadmin/pom.xml
+++ b/server/container/guice/protocols/webadmin/pom.xml
@@ -46,6 +46,11 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>james-server-webadmin-dropwizard-metrics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>testing-base</artifactId>
<scope>test</scope>
</dependency>
diff --git a/server/container/guice/protocols/webadmin/src/main/java/org/apache/james/modules/server/MetricsRoutesModule.java b/server/container/guice/protocols/webadmin/src/main/java/org/apache/james/modules/server/MetricsRoutesModule.java
new file mode 100644
index 0000000..1429b23
--- /dev/null
+++ b/server/container/guice/protocols/webadmin/src/main/java/org/apache/james/modules/server/MetricsRoutesModule.java
@@ -0,0 +1,38 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.modules.server;
+
+import org.apache.james.webadmin.Routes;
+import org.apache.james.webadmin.dropwizard.MetricsRoutes;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Scopes;
+import com.google.inject.multibindings.Multibinder;
+
+public class MetricsRoutesModule extends AbstractModule {
+ @Override
+ protected void configure() {
+ bind(MetricsRoutes.class).in(Scopes.SINGLETON);
+
+ Multibinder.newSetBinder(binder(), Routes.class)
+ .addBinding()
+ .to(MetricsRoutes.class);
+ }
+}
diff --git a/server/container/guice/protocols/webadmin/src/main/java/org/apache/james/modules/server/WebAdminServerModule.java b/server/container/guice/protocols/webadmin/src/main/java/org/apache/james/modules/server/WebAdminServerModule.java
index ff2f263..42fff3e 100644
--- a/server/container/guice/protocols/webadmin/src/main/java/org/apache/james/modules/server/WebAdminServerModule.java
+++ b/server/container/guice/protocols/webadmin/src/main/java/org/apache/james/modules/server/WebAdminServerModule.java
@@ -78,6 +78,7 @@ public class WebAdminServerModule extends AbstractModule {
protected void configure() {
install(new TaskRoutesModule());
install(new HealthCheckRoutesModule());
+ install(new MetricsRoutesModule());
bind(JsonTransformer.class).in(Scopes.SINGLETON);
bind(WebAdminServer.class).in(Scopes.SINGLETON);
diff --git a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/AuthorizedEndpointsTest.java b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/AuthorizedEndpointsTest.java
index 0ddbf24..8ec6ee7 100644
--- a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/AuthorizedEndpointsTest.java
+++ b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/AuthorizedEndpointsTest.java
@@ -52,6 +52,14 @@ public abstract class AuthorizedEndpointsTest {
}
@Test
+ void getMetricsShouldNotNeedAuthentication() {
+ when()
+ .get("/metrics")
+ .then()
+ .statusCode(not(HttpStatus.UNAUTHORIZED_401));
+ }
+
+ @Test
void getSwaggerShouldNotNeedAuthentication() {
when()
.get(SwaggerRoutes.SWAGGER_ENDPOINT)
diff --git a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/WebAdminServerIntegrationTest.java b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/WebAdminServerIntegrationTest.java
index b02ee24..4d303a4 100644
--- a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/WebAdminServerIntegrationTest.java
+++ b/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/WebAdminServerIntegrationTest.java
@@ -99,6 +99,19 @@ public abstract class WebAdminServerIntegrationTest {
}
@Test
+ void metricsRoutesShouldBeExposed() {
+ String body = when()
+ .get("/metrics").prettyPeek()
+ .then()
+ .statusCode(HttpStatus.OK_200)
+ .extract()
+ .body()
+ .asString();
+
+ assertThat(body).contains("outgoingMails_total 0.0");
+ }
+
+ @Test
void healthCheckShouldReturn200WhenCalledRepeatedly() {
given().get(HealthCheckRoutes.HEALTHCHECK);
given().get(HealthCheckRoutes.HEALTHCHECK);
diff --git a/server/protocols/webadmin/pom.xml b/server/protocols/webadmin/pom.xml
index db7f387..54a0609 100644
--- a/server/protocols/webadmin/pom.xml
+++ b/server/protocols/webadmin/pom.xml
@@ -36,6 +36,7 @@
<module>webadmin-cassandra</module>
<module>webadmin-cassandra-data</module>
<module>webadmin-core</module>
+ <module>webadmin-dropwizard-metrics</module>
<module>webadmin-data</module>
<module>webadmin-jmap</module>
<module>webadmin-mailbox</module>
diff --git a/server/container/guice/protocols/webadmin/pom.xml b/server/protocols/webadmin/webadmin-dropwizard-metrics/pom.xml
similarity index 51%
copy from server/container/guice/protocols/webadmin/pom.xml
copy to server/protocols/webadmin/webadmin-dropwizard-metrics/pom.xml
index d383898..6094907 100644
--- a/server/container/guice/protocols/webadmin/pom.xml
+++ b/server/protocols/webadmin/webadmin-dropwizard-metrics/pom.xml
@@ -17,32 +17,37 @@
specific language governing permissions and limitations
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.james</groupId>
- <artifactId>james-server-guice</artifactId>
+ <artifactId>james-server</artifactId>
<version>3.6.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
+ <relativePath>../../../pom.xml</relativePath>
</parent>
- <artifactId>james-server-guice-webadmin</artifactId>
+ <artifactId>james-server-webadmin-dropwizard-metrics</artifactId>
+ <packaging>jar</packaging>
- <name>Apache James :: Server :: Guice :: Webadmin</name>
- <description>Webadmin modules for Guice implementation of James server</description>
+ <name>Apache James :: Server :: Web Admin :: Dropwizard metrics</name>
+ <description>HTTP endpoint to expose dropwizard collected metrics. This endpoint is intended to be called by
+ Prometheus.</description>
<dependencies>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>james-server-guice-configuration</artifactId>
+ <artifactId>james-server-webadmin-core</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>james-server-guice-utils</artifactId>
+ <artifactId>james-server-webadmin-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>james-server-webadmin-core</artifactId>
+ <artifactId>metrics-tests</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
@@ -50,8 +55,33 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient_dropwizard</artifactId>
+ <version>0.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient_servlet</artifactId>
+ <version>0.9.0</version>
+ </dependency>
+ <dependency>
+ <groupId>io.rest-assured</groupId>
+ <artifactId>rest-assured</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>test</scope>
</dependency>
</dependencies>
-</project>
+
+</project>
\ No newline at end of file
diff --git a/server/protocols/webadmin/webadmin-dropwizard-metrics/src/main/java/org/apache/james/webadmin/dropwizard/MetricsRoutes.java b/server/protocols/webadmin/webadmin-dropwizard-metrics/src/main/java/org/apache/james/webadmin/dropwizard/MetricsRoutes.java
new file mode 100644
index 0000000..8cfe191
--- /dev/null
+++ b/server/protocols/webadmin/webadmin-dropwizard-metrics/src/main/java/org/apache/james/webadmin/dropwizard/MetricsRoutes.java
@@ -0,0 +1,89 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.webadmin.dropwizard;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.Optional;
+import java.util.Set;
+
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.james.webadmin.PublicRoutes;
+
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.collect.ImmutableSet;
+
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.dropwizard.DropwizardExports;
+import io.prometheus.client.exporter.common.TextFormat;
+import spark.Request;
+import spark.Response;
+import spark.Service;
+
+/**
+ * Content adapted from https://github.com/prometheus/client_java/blob/master/simpleclient_servlet/src/main/java/io/prometheus/client/exporter/MetricsServlet.java
+ */
+public class MetricsRoutes implements PublicRoutes {
+
+ public static final String BASE = "/metrics";
+ private final CollectorRegistry collectorRegistry;
+
+ @Inject
+ public MetricsRoutes(MetricRegistry registry) {
+ collectorRegistry = CollectorRegistry.defaultRegistry;
+ new DropwizardExports(registry).register(collectorRegistry);
+ }
+
+ @Override
+ public String getBasePath() {
+ return BASE;
+ }
+
+ @Override
+ public void define(Service service) {
+ service.get(BASE, this::getMetrics);
+ }
+
+ public Response getMetrics(Request request, Response response) throws IOException {
+ Set<String> params = parse(request.raw());
+ HttpServletResponse rawResponse = response.raw();
+ rawResponse.setStatus(HttpServletResponse.SC_OK);
+ rawResponse.setContentType(TextFormat.CONTENT_TYPE_004);
+
+ try (Writer writer = new BufferedWriter(rawResponse.getWriter())) {
+ TextFormat.write004(writer, collectorRegistry.filteredMetricFamilySamples(params));
+ writer.flush();
+ }
+ return response;
+ }
+
+
+ private Set<String> parse(HttpServletRequest req) {
+ String[] includedParam = req.getParameterValues("name[]");
+
+ return Optional.ofNullable(includedParam)
+ .map(ImmutableSet::copyOf)
+ .orElse(ImmutableSet.of());
+ }
+}
diff --git a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/AuthorizedEndpointsTest.java b/server/protocols/webadmin/webadmin-dropwizard-metrics/src/test/java/org/apache/james/webadmin/dropwizard/MetricsRoutesTest.java
similarity index 53%
copy from server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/AuthorizedEndpointsTest.java
copy to server/protocols/webadmin/webadmin-dropwizard-metrics/src/test/java/org/apache/james/webadmin/dropwizard/MetricsRoutesTest.java
index 0ddbf24..90f9ec7 100644
--- a/server/protocols/webadmin-integration-test/webadmin-integration-test-common/src/main/java/org/apache/james/webadmin/integration/AuthorizedEndpointsTest.java
+++ b/server/protocols/webadmin/webadmin-dropwizard-metrics/src/test/java/org/apache/james/webadmin/dropwizard/MetricsRoutesTest.java
@@ -17,45 +17,57 @@
* under the License. *
****************************************************************/
-package org.apache.james.webadmin.integration;
+package org.apache.james.webadmin.dropwizard;
import static io.restassured.RestAssured.when;
-import static org.hamcrest.core.IsNot.not;
+import static org.assertj.core.api.Assertions.assertThat;
-import org.apache.james.GuiceJamesServer;
-import org.apache.james.utils.WebAdminGuiceProbe;
+import org.apache.james.webadmin.WebAdminServer;
import org.apache.james.webadmin.WebAdminUtils;
-import org.apache.james.webadmin.routes.HealthCheckRoutes;
-import org.apache.james.webadmin.swagger.routes.SwaggerRoutes;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import com.codahale.metrics.MetricRegistry;
+
import io.restassured.RestAssured;
-public abstract class AuthorizedEndpointsTest {
+class MetricsRoutesTest {
+ WebAdminServer webAdminServer;
+ MetricRegistry registry;
@BeforeEach
- void setUp(GuiceJamesServer guiceJamesServer) {
- WebAdminGuiceProbe webAdminGuiceProbe = guiceJamesServer.getProbe(WebAdminGuiceProbe.class);
+ void setUp() {
+ registry = new MetricRegistry();
+ webAdminServer = WebAdminUtils.createWebAdminServer(new MetricsRoutes(registry))
+ .start();
- RestAssured.requestSpecification = WebAdminUtils.buildRequestSpecification(webAdminGuiceProbe.getWebAdminPort())
+ RestAssured.requestSpecification = WebAdminUtils.buildRequestSpecification(webAdminServer)
.build();
+ RestAssured.enableLoggingOfRequestAndResponseIfValidationFails();
}
@Test
- void getHealthchecksShouldNotNeedAuthentication() {
- when()
- .get(HealthCheckRoutes.HEALTHCHECK)
- .then()
- .statusCode(not(HttpStatus.UNAUTHORIZED_401));
- }
+ void getShouldReturnSeveralMetric() {
+ registry.counter("easy").inc();
+ registry.counter("hard").inc();
+ registry.counter("hard").inc();
- @Test
- void getSwaggerShouldNotNeedAuthentication() {
- when()
- .get(SwaggerRoutes.SWAGGER_ENDPOINT)
+ String body = when()
+ .get("/metrics")
.then()
- .statusCode(not(HttpStatus.UNAUTHORIZED_401));
+ .statusCode(HttpStatus.OK_200)
+ .extract()
+ .body()
+ .asString();
+
+ assertThat(body)
+ .contains(
+ "# HELP hard Generated from Dropwizard metric import (metric=hard, type=com.codahale.metrics.Counter)\n" +
+ "# TYPE hard gauge\n" +
+ "hard 2.0\n" +
+ "# HELP easy Generated from Dropwizard metric import (metric=easy, type=com.codahale.metrics.Counter)\n" +
+ "# TYPE easy gauge\n" +
+ "easy 1.0");
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org