You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by fr...@apache.org on 2024/03/23 00:46:11 UTC
(cassandra-sidecar) branch trunk updated: CASSANDRASC-111 Improve observability in Sidecar with Dropwizard metrics (#102)
This is an automated email from the ASF dual-hosted git repository.
frankgh pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-sidecar.git
The following commit(s) were added to refs/heads/trunk by this push:
new 056faad8 CASSANDRASC-111 Improve observability in Sidecar with Dropwizard metrics (#102)
056faad8 is described below
commit 056faad8278365a77a36dd336e002136c269165a
Author: Saranya Krishnakumar <sa...@apple.com>
AuthorDate: Fri Mar 22 17:46:06 2024 -0700
CASSANDRASC-111 Improve observability in Sidecar with Dropwizard metrics (#102)
Patch by Saranya Krishnakumar; Reviewed by Yifan Cai, Francisco Guerrero for CASSANDRASC-111
---
CHANGES.txt | 1 +
src/main/dist/conf/sidecar.yaml | 9 ++
.../sidecar/cluster/instance/InstanceMetadata.java | 8 +-
.../cluster/instance/InstanceMetadataImpl.java | 42 ++++-
.../MetricsConfiguration.java} | 40 +----
.../sidecar/config/SidecarConfiguration.java | 5 +
.../VertxMetricsConfiguration.java} | 36 ++---
.../config/yaml/MetricsConfigurationImpl.java | 67 ++++++++
.../config/yaml/SidecarConfigurationImpl.java | 27 ++++
.../config/yaml/VertxMetricsConfigurationImpl.java | 100 ++++++++++++
.../cassandra/sidecar/metrics/NamedMetric.java | 169 +++++++++++++++++++++
.../cassandra/sidecar/metrics/ServerMetrics.java | 55 +++++++
.../instance/InstanceMetrics.java} | 39 ++---
.../metrics/instance/InstanceMetricsImpl.java | 61 ++++++++
.../metrics/instance/InstanceResourceMetrics.java | 49 ++++++
.../metrics/instance/StreamSSTableMetrics.java | 88 +++++++++++
.../metrics/instance/UploadSSTableMetrics.java | 88 +++++++++++
.../sidecar/routes/DiskSpaceProtectionHandler.java | 1 +
.../sidecar/routes/FileStreamHandler.java | 7 +-
.../sstableuploads/SSTableUploadHandler.java | 15 +-
.../cassandra/sidecar/server/MainModule.java | 49 ++++--
.../apache/cassandra/sidecar/server/Server.java | 9 +-
.../sidecar/tasks/HealthCheckPeriodicTask.java | 14 +-
.../cassandra/sidecar/utils/FileStreamer.java | 52 +++++--
.../cassandra/sidecar/utils/MetricUtils.java | 42 +++++
.../testing/CassandraSidecarTestContext.java | 1 +
.../org/apache/cassandra/sidecar/TestModule.java | 3 +
.../org/apache/cassandra/sidecar/ThrottleTest.java | 17 ++-
.../sidecar/config/SidecarConfigurationTest.java | 49 ++++++
.../cassandra/sidecar/metrics/NamedMetricTest.java | 103 +++++++++++++
.../sidecar/restore/RestoreJobManagerTest.java | 4 +-
.../sstableuploads/BaseUploadsHandlerTest.java | 3 +
.../sstableuploads/SSTableUploadHandlerTest.java | 13 ++
.../cassandra/sidecar/server/ServerSSLTest.java | 23 ++-
.../cassandra/sidecar/snapshots/SnapshotUtils.java | 2 +
.../sidecar/tasks/HealthCheckPeriodicTaskTest.java | 50 +++++-
.../cassandra/sidecar/utils/TestMetricUtils.java | 39 +++++
src/test/resources/config/sidecar_metrics.yaml | 69 +++++++++
38 files changed, 1310 insertions(+), 139 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 638ee316..3e79c80f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,5 +1,6 @@
1.0.0
-----
+ * Improve observability in Sidecar (CASSANDRASC-111)
* Improve logging for slice restore task (CASSANDRASC-107)
* Add restore task watcher to report long running tasks (CASSANDRASC-106)
* RestoreSliceTask could be stuck due to missing exception handling (CASSANDRASC-105)
diff --git a/src/main/dist/conf/sidecar.yaml b/src/main/dist/conf/sidecar.yaml
index 82e4b1b2..2369f252 100644
--- a/src/main/dist/conf/sidecar.yaml
+++ b/src/main/dist/conf/sidecar.yaml
@@ -142,6 +142,15 @@ healthcheck:
initial_delay_millis: 0
poll_freq_millis: 30000
+metrics:
+ registry_name: cassandra_sidecar
+ vertx:
+ enabled: true
+ expose_via_jmx: false
+ jmx_domain_name: sidecar.vertx.jmx_domain
+ monitored_server_route_regexes: # regex list to match server routes
+ - /api/v1/.*
+
cassandra_input_validation:
forbidden_keyspaces:
- system_schema
diff --git a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java b/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
index 26da25e7..8862c8a1 100644
--- a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
+++ b/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
@@ -21,9 +21,10 @@ package org.apache.cassandra.sidecar.cluster.instance;
import java.util.List;
import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics;
+import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-
/**
* Metadata of an instance
*/
@@ -58,4 +59,9 @@ public interface InstanceMetadata
* @return a {@link CassandraAdapterDelegate} specific for the instance
*/
@Nullable CassandraAdapterDelegate delegate();
+
+ /**
+ * @return {@link InstanceMetrics} metrics specific for the Cassandra instance
+ */
+ @NotNull InstanceMetrics metrics();
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadataImpl.java b/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadataImpl.java
index ca39b6c8..d0e4fd97 100644
--- a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadataImpl.java
+++ b/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadataImpl.java
@@ -21,9 +21,14 @@ package org.apache.cassandra.sidecar.cluster.instance;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate;
import org.apache.cassandra.sidecar.common.DataObjectBuilder;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetricsImpl;
import org.jetbrains.annotations.Nullable;
/**
@@ -38,6 +43,7 @@ public class InstanceMetadataImpl implements InstanceMetadata
private final String stagingDir;
@Nullable
private final CassandraAdapterDelegate delegate;
+ private final InstanceMetrics metrics;
protected InstanceMetadataImpl(Builder builder)
{
@@ -47,6 +53,7 @@ public class InstanceMetadataImpl implements InstanceMetadata
dataDirs = Collections.unmodifiableList(builder.dataDirs);
stagingDir = builder.stagingDir;
delegate = builder.delegate;
+ metrics = builder.metrics;
}
@Override
@@ -85,6 +92,12 @@ public class InstanceMetadataImpl implements InstanceMetadata
return delegate;
}
+ @Override
+ public InstanceMetrics metrics()
+ {
+ return metrics;
+ }
+
public static Builder builder()
{
return new Builder();
@@ -95,12 +108,14 @@ public class InstanceMetadataImpl implements InstanceMetadata
*/
public static class Builder implements DataObjectBuilder<Builder, InstanceMetadataImpl>
{
- protected int id;
+ protected Integer id;
protected String host;
protected int port;
protected List<String> dataDirs;
protected String stagingDir;
protected CassandraAdapterDelegate delegate;
+ protected String globalRegistryName;
+ protected InstanceMetrics metrics;
protected Builder()
{
@@ -114,6 +129,7 @@ public class InstanceMetadataImpl implements InstanceMetadata
dataDirs = new ArrayList<>(instanceMetadata.dataDirs);
stagingDir = instanceMetadata.stagingDir;
delegate = instanceMetadata.delegate;
+ metrics = instanceMetadata.metrics;
}
@Override
@@ -188,6 +204,18 @@ public class InstanceMetadataImpl implements InstanceMetadata
return update(b -> b.delegate = delegate);
}
+ /**
+ * Sets the {@code globalRegistryName} and returns a reference to this Builder enabling method chaining.
+ *
+ * @param registryName global {@link com.codahale.metrics.MetricRegistry} name
+ * @return a reference to this Builder
+ */
+ public Builder globalMetricRegistryName(String registryName)
+
+ {
+ return update(b -> b.globalRegistryName = registryName);
+ }
+
/**
* Returns a {@code InstanceMetadataImpl} built from the parameters previously set.
*
@@ -196,7 +224,19 @@ public class InstanceMetadataImpl implements InstanceMetadata
@Override
public InstanceMetadataImpl build()
{
+ Objects.requireNonNull(id);
+ Objects.requireNonNull(globalRegistryName);
+
+ String instanceRegistryName = instanceRegistryName(globalRegistryName);
+ MetricRegistry instanceMetricRegistry = SharedMetricRegistries.getOrCreate(instanceRegistryName);
+ metrics = new InstanceMetricsImpl(instanceMetricRegistry);
+
return new InstanceMetadataImpl(this);
}
+
+ private String instanceRegistryName(String globalRegistryName)
+ {
+ return globalRegistryName + "_" + id;
+ }
}
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java b/src/main/java/org/apache/cassandra/sidecar/config/MetricsConfiguration.java
similarity index 50%
copy from src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
copy to src/main/java/org/apache/cassandra/sidecar/config/MetricsConfiguration.java
index 26da25e7..2fc622a0 100644
--- a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
+++ b/src/main/java/org/apache/cassandra/sidecar/config/MetricsConfiguration.java
@@ -16,46 +16,20 @@
* limitations under the License.
*/
-package org.apache.cassandra.sidecar.cluster.instance;
-
-import java.util.List;
-
-import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate;
-import org.jetbrains.annotations.Nullable;
-
+package org.apache.cassandra.sidecar.config;
/**
- * Metadata of an instance
+ * Configuration needed for capturing metrics.
*/
-public interface InstanceMetadata
+public interface MetricsConfiguration
{
/**
- * @return an identifier for the Cassandra instance
- */
- int id();
-
- /**
- * @return the host address of the Cassandra instance
- */
- String host();
-
- /**
- * @return the port number of the Cassandra instance
- */
- int port();
-
- /**
- * @return a list of data directories of cassandra instance
- */
- List<String> dataDirs();
-
- /**
- * @return a staging directory of the cassandra instance
+ * @return global registry name to be used for registering sidecar metrics
*/
- String stagingDir();
+ String registryName();
/**
- * @return a {@link CassandraAdapterDelegate} specific for the instance
+ * @return configuration needed for capturing metrics released by Vert.x framework.
*/
- @Nullable CassandraAdapterDelegate delegate();
+ VertxMetricsConfiguration vertxConfiguration();
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/config/SidecarConfiguration.java b/src/main/java/org/apache/cassandra/sidecar/config/SidecarConfiguration.java
index f5b15211..e9a84370 100644
--- a/src/main/java/org/apache/cassandra/sidecar/config/SidecarConfiguration.java
+++ b/src/main/java/org/apache/cassandra/sidecar/config/SidecarConfiguration.java
@@ -51,6 +51,11 @@ public interface SidecarConfiguration
*/
HealthCheckConfiguration healthCheckConfiguration();
+ /**
+ * @return configuration needed for metrics capture
+ */
+ MetricsConfiguration metricsConfiguration();
+
/**
* @return the configuration for Cassandra input validation
*/
diff --git a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java b/src/main/java/org/apache/cassandra/sidecar/config/VertxMetricsConfiguration.java
similarity index 53%
copy from src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
copy to src/main/java/org/apache/cassandra/sidecar/config/VertxMetricsConfiguration.java
index 26da25e7..824a8c57 100644
--- a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
+++ b/src/main/java/org/apache/cassandra/sidecar/config/VertxMetricsConfiguration.java
@@ -16,46 +16,32 @@
* limitations under the License.
*/
-package org.apache.cassandra.sidecar.cluster.instance;
+package org.apache.cassandra.sidecar.config;
import java.util.List;
-import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate;
-import org.jetbrains.annotations.Nullable;
-
-
/**
- * Metadata of an instance
+ * Holds configuration needed for enabling metrics capture in Vert.x framework, for analyzing from Sidecar.
*/
-public interface InstanceMetadata
+public interface VertxMetricsConfiguration
{
/**
- * @return an identifier for the Cassandra instance
- */
- int id();
-
- /**
- * @return the host address of the Cassandra instance
- */
- String host();
-
- /**
- * @return the port number of the Cassandra instance
+ * @return boolean indicating whether metrics captured is enabled for Vert.x
*/
- int port();
+ boolean enabled();
/**
- * @return a list of data directories of cassandra instance
+ * @return boolean indicating whether Vert.x metrics will be exposed via JMX
*/
- List<String> dataDirs();
+ boolean exposeViaJMX();
/**
- * @return a staging directory of the cassandra instance
+ * @return JMX domain to be used when metrics are exposed via JMX
*/
- String stagingDir();
+ String jmxDomainName();
/**
- * @return a {@link CassandraAdapterDelegate} specific for the instance
+ * @return List of regexes used for identifying what server routes will be monitored
*/
- @Nullable CassandraAdapterDelegate delegate();
+ List<String> monitoredServerRouteRegexes();
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/config/yaml/MetricsConfigurationImpl.java b/src/main/java/org/apache/cassandra/sidecar/config/yaml/MetricsConfigurationImpl.java
new file mode 100644
index 00000000..fc60a6d2
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/config/yaml/MetricsConfigurationImpl.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.config.yaml;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.cassandra.sidecar.config.MetricsConfiguration;
+import org.apache.cassandra.sidecar.config.VertxMetricsConfiguration;
+
+/**
+ * Configuration needed for capturing metrics.
+ */
+public class MetricsConfigurationImpl implements MetricsConfiguration
+{
+ public static final String DEFAULT_DROPWIZARD_REGISTRY_NAME = "cassandra_sidecar";
+ public static final VertxMetricsConfiguration DEFAULT_VERTX_METRICS_CONFIGURATION
+ = new VertxMetricsConfigurationImpl();
+
+ @JsonProperty(value = "registry_name")
+ protected final String registryName;
+ @JsonProperty(value = "vertx")
+ protected final VertxMetricsConfiguration vertxConfiguration;
+
+ public MetricsConfigurationImpl()
+ {
+ this(DEFAULT_DROPWIZARD_REGISTRY_NAME, DEFAULT_VERTX_METRICS_CONFIGURATION);
+ }
+
+ public MetricsConfigurationImpl(String registryName, VertxMetricsConfiguration vertxConfiguration)
+ {
+ this.registryName = registryName;
+ this.vertxConfiguration = vertxConfiguration;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String registryName()
+ {
+ return registryName;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public VertxMetricsConfiguration vertxConfiguration()
+ {
+ return vertxConfiguration;
+ }
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/config/yaml/SidecarConfigurationImpl.java b/src/main/java/org/apache/cassandra/sidecar/config/yaml/SidecarConfigurationImpl.java
index d2c2ef44..eb3160ed 100644
--- a/src/main/java/org/apache/cassandra/sidecar/config/yaml/SidecarConfigurationImpl.java
+++ b/src/main/java/org/apache/cassandra/sidecar/config/yaml/SidecarConfigurationImpl.java
@@ -42,6 +42,7 @@ import org.apache.cassandra.sidecar.config.CassandraInputValidationConfiguration
import org.apache.cassandra.sidecar.config.DriverConfiguration;
import org.apache.cassandra.sidecar.config.HealthCheckConfiguration;
import org.apache.cassandra.sidecar.config.InstanceConfiguration;
+import org.apache.cassandra.sidecar.config.MetricsConfiguration;
import org.apache.cassandra.sidecar.config.RestoreJobConfiguration;
import org.apache.cassandra.sidecar.config.S3ClientConfiguration;
import org.apache.cassandra.sidecar.config.ServiceConfiguration;
@@ -72,6 +73,9 @@ public class SidecarConfigurationImpl implements SidecarConfiguration
@JsonProperty("healthcheck")
protected final HealthCheckConfiguration healthCheckConfiguration;
+ @JsonProperty("metrics")
+ protected final MetricsConfiguration metricsConfiguration;
+
@JsonProperty("cassandra_input_validation")
protected final CassandraInputValidationConfiguration cassandraInputValidationConfiguration;
@@ -93,6 +97,7 @@ public class SidecarConfigurationImpl implements SidecarConfiguration
serviceConfiguration = builder.serviceConfiguration;
sslConfiguration = builder.sslConfiguration;
healthCheckConfiguration = builder.healthCheckConfiguration;
+ metricsConfiguration = builder.metricsConfiguration;
cassandraInputValidationConfiguration = builder.cassandraInputValidationConfiguration;
driverConfiguration = builder.driverConfiguration;
restoreJobConfiguration = builder.restoreJobConfiguration;
@@ -159,6 +164,16 @@ public class SidecarConfigurationImpl implements SidecarConfiguration
return healthCheckConfiguration;
}
+ /**
+ * @return the configuration needed for metrics capture
+ */
+ @Override
+ @JsonProperty("metrics")
+ public MetricsConfiguration metricsConfiguration()
+ {
+ return metricsConfiguration;
+ }
+
@Override
@JsonProperty("driver_parameters")
public DriverConfiguration driverConfiguration()
@@ -290,6 +305,7 @@ public class SidecarConfigurationImpl implements SidecarConfiguration
private ServiceConfiguration serviceConfiguration = new ServiceConfigurationImpl();
private SslConfiguration sslConfiguration = null;
private HealthCheckConfiguration healthCheckConfiguration = new HealthCheckConfigurationImpl();
+ private MetricsConfiguration metricsConfiguration = new MetricsConfigurationImpl();
private CassandraInputValidationConfiguration cassandraInputValidationConfiguration
= new CassandraInputValidationConfigurationImpl();
private DriverConfiguration driverConfiguration = new DriverConfigurationImpl();
@@ -361,6 +377,17 @@ public class SidecarConfigurationImpl implements SidecarConfiguration
return update(b -> b.healthCheckConfiguration = healthCheckConfiguration);
}
+ /**
+ * Sets the {@code metricsConfiguration} and returns a reference to this Builder enabling method chaining.
+ *
+ * @param metricsConfiguration the {@code metricsConfiguration} to set
+ * @return a reference to this Builder
+ */
+ public Builder metricsConfiguration(MetricsConfiguration metricsConfiguration)
+ {
+ return update(b -> b.metricsConfiguration = metricsConfiguration);
+ }
+
/**
* Sets the {@code driverConfiguration} and returns a reference to this Builder enabling
* method chaining.
diff --git a/src/main/java/org/apache/cassandra/sidecar/config/yaml/VertxMetricsConfigurationImpl.java b/src/main/java/org/apache/cassandra/sidecar/config/yaml/VertxMetricsConfigurationImpl.java
new file mode 100644
index 00000000..c32e4060
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/config/yaml/VertxMetricsConfigurationImpl.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.config.yaml;
+
+import java.util.Collections;
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.cassandra.sidecar.config.VertxMetricsConfiguration;
+
+/**
+ * Configuration needed for capturing metrics release by Vert.x framework.
+ */
+public class VertxMetricsConfigurationImpl implements VertxMetricsConfiguration
+{
+ public static final boolean DEFAULT_ENABLED = true;
+ public static final boolean DEFAULT_EXPOSE_VIA_JMX = false;
+ public static final String DEFAULT_JMX_DOMAIN_NAME = "sidecar.vertx.jmx_domain";
+ public static final List<String> DEFAULT_MONITORED_SERVER_ROUTE_REGEXES = Collections.singletonList("/api/v1/.*");
+
+ @JsonProperty(value = "enabled")
+ protected final boolean enabled;
+ @JsonProperty(value = "expose_via_jmx")
+ protected final boolean exposeViaJMX;
+ @JsonProperty(value = "jmx_domain_name")
+ protected final String jmxDomainName;
+ @JsonProperty(value = "monitored_server_route_regexes")
+ protected final List<String> monitoredServerRouteRegexes;
+
+ public VertxMetricsConfigurationImpl()
+ {
+ this(DEFAULT_ENABLED,
+ DEFAULT_EXPOSE_VIA_JMX,
+ DEFAULT_JMX_DOMAIN_NAME,
+ DEFAULT_MONITORED_SERVER_ROUTE_REGEXES);
+ }
+
+ public VertxMetricsConfigurationImpl(boolean enabled,
+ boolean exposeViaJMX,
+ String jmxDomainName,
+ List<String> monitoredServerRouteRegexes)
+ {
+ this.enabled = enabled;
+ this.exposeViaJMX = exposeViaJMX;
+ this.jmxDomainName = jmxDomainName;
+ this.monitoredServerRouteRegexes = monitoredServerRouteRegexes;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean enabled()
+ {
+ return enabled;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean exposeViaJMX()
+ {
+ return exposeViaJMX;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String jmxDomainName()
+ {
+ return jmxDomainName;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public List<String> monitoredServerRouteRegexes()
+ {
+ return monitoredServerRouteRegexes;
+ }
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/metrics/NamedMetric.java b/src/main/java/org/apache/cassandra/sidecar/metrics/NamedMetric.java
new file mode 100644
index 00000000..e5966253
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/metrics/NamedMetric.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import com.codahale.metrics.Metric;
+import org.apache.cassandra.sidecar.common.DataObjectBuilder;
+
+/**
+ * {@link NamedMetric} is for creating {@link Metric} with a structured name. Metric name should contain domain it
+ * is captured for and optional tags for additional context.
+ *
+ * @param <T> Metric type
+ */
+public class NamedMetric<T extends Metric>
+{
+ public final String canonicalName;
+ public final T metric;
+
+ private NamedMetric(Builder<T> builder)
+ {
+ this.canonicalName = builder.makeFullName();
+ this.metric = builder.register(canonicalName);
+ }
+
+ public static <T extends Metric> Builder<T> builder(Function<String, T> metricCreator)
+ {
+ return new Builder<>(metricCreator);
+ }
+
+ /**
+ * Builder for {@link NamedMetric}
+ *
+ * @param <T> Metric type
+ */
+ public static class Builder<T extends Metric> implements DataObjectBuilder<Builder<T>, NamedMetric<T>>
+ {
+ private final Function<String, T> metricCreator;
+ private final List<Tag> tags = new ArrayList<>();
+ private String domain;
+ private String name;
+
+ public Builder(Function<String, T> metricCreator)
+ {
+ this.metricCreator = metricCreator;
+ }
+
+ /**
+ * Sets {@code domain} of metric.
+ *
+ * @param domain domain metric is part of
+ * @return a reference to this Builder
+ */
+ public Builder<T> withDomain(String domain)
+ {
+ return update(b -> b.domain = domain);
+ }
+
+ /**
+ * Sets {@code name} of metric.
+ *
+ * @param name metric name
+ * @return a reference to this Builder
+ */
+ public Builder<T> withName(String name)
+ {
+ return update(b -> b.name = name);
+ }
+
+ /**
+ * Additional name tag added to metric name for more clarity. Tags are usually added like,
+ * component=data, route=/stream/component, etc.
+ *
+ * @param tag tag added to {@code tags}
+ * @return a reference to this Builder
+ */
+ public Builder<T> addTag(Tag tag)
+ {
+ return update(b -> b.tags.add(tag));
+ }
+
+ /**
+ * Additional name tag added to metric name for more clarity. Tags are usually added like,
+ * component=data, route=/stream/component, etc.
+ *
+ * @param key key of tag to be added
+ * @param value value of tag to be added
+ * @return a reference to this Builder
+ */
+ public Builder<T> addTag(String key, String value)
+ {
+ return addTag(Tag.of(key, value));
+ }
+
+ @Override
+ public Builder<T> self()
+ {
+ return this;
+ }
+
+ @Override
+ public NamedMetric<T> build()
+ {
+ Objects.requireNonNull(metricCreator);
+ Objects.requireNonNull(domain);
+ Objects.requireNonNull(name);
+
+ return new NamedMetric<>(this);
+ }
+
+ private String makeFullName()
+ {
+ String domainPart = domain + '.';
+ String combinedTags = !tags.isEmpty() ? combineTags() + '.' : "";
+ return domainPart + combinedTags + name;
+ }
+
+ private String combineTags()
+ {
+ return tags.stream().map(tag -> tag.key + '=' + tag.value).collect(Collectors.joining("."));
+ }
+
+ private T register(String metricName)
+ {
+ return metricCreator.apply(metricName);
+ }
+ }
+
+ /**
+ * Used for tagging {@link NamedMetric} for additional context.
+ */
+ public static class Tag
+ {
+ public final String key;
+ public final String value;
+
+ public static Tag of(String key, String value)
+ {
+ return new Tag(key, value);
+ }
+
+ private Tag(String key, String value)
+ {
+ this.key = Objects.requireNonNull(key, "Key can not be null");
+ this.value = Objects.requireNonNull(value, "Value can not be null");
+ }
+ }
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/metrics/ServerMetrics.java b/src/main/java/org/apache/cassandra/sidecar/metrics/ServerMetrics.java
new file mode 100644
index 00000000..af555c2e
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/metrics/ServerMetrics.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.metrics;
+
+import java.util.Objects;
+
+import com.codahale.metrics.DefaultSettableGauge;
+import com.codahale.metrics.MetricRegistry;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * Tracks metrics related to server initialization, periodic tasks related to server.
+ */
+@Singleton
+public class ServerMetrics
+{
+ public static final String DOMAIN = "sidecar.server";
+ protected final MetricRegistry metricRegistry;
+ public final NamedMetric<DefaultSettableGauge<Integer>> cassandraInstancesUp;
+ public final NamedMetric<DefaultSettableGauge<Integer>> cassandraInstancesDown;
+
+ @Inject
+ public ServerMetrics(MetricRegistry metricRegistry)
+ {
+ this.metricRegistry = Objects.requireNonNull(metricRegistry, "MetricRegistry can not be null");
+
+ cassandraInstancesUp =
+ NamedMetric.builder(name -> metricRegistry.gauge(name, () -> new DefaultSettableGauge<>(0)))
+ .withDomain(DOMAIN)
+ .withName("instances_up")
+ .build();
+ cassandraInstancesDown =
+ NamedMetric.builder(name -> metricRegistry.gauge(name, () -> new DefaultSettableGauge<>(0)))
+ .withDomain(DOMAIN)
+ .withName("instances_down")
+ .build();
+ }
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceMetrics.java
similarity index 51%
copy from src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
copy to src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceMetrics.java
index 26da25e7..43a5528d 100644
--- a/src/main/java/org/apache/cassandra/sidecar/cluster/instance/InstanceMetadata.java
+++ b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceMetrics.java
@@ -16,46 +16,27 @@
* limitations under the License.
*/
-package org.apache.cassandra.sidecar.cluster.instance;
-
-import java.util.List;
-
-import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate;
-import org.jetbrains.annotations.Nullable;
-
+package org.apache.cassandra.sidecar.metrics.instance;
/**
- * Metadata of an instance
+ * {@link InstanceMetrics} tracks metrics related to a Cassandra instance that Sidecar maintains.
*/
-public interface InstanceMetadata
+public interface InstanceMetrics
{
- /**
- * @return an identifier for the Cassandra instance
- */
- int id();
-
- /**
- * @return the host address of the Cassandra instance
- */
- String host();
-
- /**
- * @return the port number of the Cassandra instance
- */
- int port();
+ String INSTANCE_PREFIX = "sidecar.cas_instance";
/**
- * @return a list of data directories of cassandra instance
+ * @return resource metrics tracked for cassandra instance
*/
- List<String> dataDirs();
+ InstanceResourceMetrics resource();
/**
- * @return a staging directory of the cassandra instance
+ * @return metrics that are tracked during streaming of SSTable components from a cassandra instance
*/
- String stagingDir();
+ StreamSSTableMetrics streamSSTable();
/**
- * @return a {@link CassandraAdapterDelegate} specific for the instance
+ * @return metrics that are tracked during upload of SSTable components to a cassandra instance
*/
- @Nullable CassandraAdapterDelegate delegate();
+ UploadSSTableMetrics uploadSSTable();
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceMetricsImpl.java b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceMetricsImpl.java
new file mode 100644
index 00000000..f30ddf99
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceMetricsImpl.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.metrics.instance;
+
+import java.util.Objects;
+
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * Tracks metrics related to a Cassandra instance that Sidecar maintains.
+ */
+public class InstanceMetricsImpl implements InstanceMetrics
+{
+ protected final MetricRegistry metricRegistry;
+ protected final StreamSSTableMetrics streamSSTableMetrics;
+ protected final UploadSSTableMetrics uploadSSTableMetrics;
+ protected final InstanceResourceMetrics resourceMetrics;
+
+ public InstanceMetricsImpl(MetricRegistry metricRegistry)
+ {
+ this.metricRegistry = Objects.requireNonNull(metricRegistry, "Metrics registry can not be null");
+
+ this.resourceMetrics = new InstanceResourceMetrics(metricRegistry);
+ this.streamSSTableMetrics = new StreamSSTableMetrics(metricRegistry);
+ this.uploadSSTableMetrics = new UploadSSTableMetrics(metricRegistry);
+ }
+
+ @Override
+ public InstanceResourceMetrics resource()
+ {
+ return resourceMetrics;
+ }
+
+ @Override
+ public StreamSSTableMetrics streamSSTable()
+ {
+ return new StreamSSTableMetrics(metricRegistry);
+ }
+
+ @Override
+ public UploadSSTableMetrics uploadSSTable()
+ {
+ return new UploadSSTableMetrics(metricRegistry);
+ }
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceResourceMetrics.java b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceResourceMetrics.java
new file mode 100644
index 00000000..faa442f4
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/InstanceResourceMetrics.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.metrics.instance;
+
+import java.util.Objects;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import org.apache.cassandra.sidecar.metrics.NamedMetric;
+
+import static org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics.INSTANCE_PREFIX;
+
+/**
+ * {@link InstanceResourceMetrics} contains metrics to track resource usage of a Cassandra instance maintained
+ * by Sidecar.
+ */
+public class InstanceResourceMetrics
+{
+ public static final String DOMAIN = INSTANCE_PREFIX + ".resource";
+ protected final MetricRegistry metricRegistry;
+ public final NamedMetric<Meter> insufficientStagingSpace;
+
+ public InstanceResourceMetrics(MetricRegistry metricRegistry)
+ {
+ this.metricRegistry = Objects.requireNonNull(metricRegistry, "Metric registry can not be null");
+
+ insufficientStagingSpace
+ = NamedMetric.builder(metricRegistry::meter)
+ .withDomain(DOMAIN)
+ .withName("insufficient_staging_space")
+ .build();
+ }
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/metrics/instance/StreamSSTableMetrics.java b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/StreamSSTableMetrics.java
new file mode 100644
index 00000000..67545b48
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/StreamSSTableMetrics.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.metrics.instance;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import org.apache.cassandra.sidecar.metrics.NamedMetric;
+
+import static org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics.INSTANCE_PREFIX;
+
+/**
+ * {@link StreamSSTableMetrics} tracks metrics captured during streaming of SSTable components from Sidecar.
+ */
+public class StreamSSTableMetrics
+{
+ public static final String DOMAIN = INSTANCE_PREFIX + ".stream_sstable";
+ protected final MetricRegistry metricRegistry;
+ protected final Map<String, StreamSSTableComponentMetrics> streamComponentMetrics = new ConcurrentHashMap<>();
+
+ public StreamSSTableMetrics(MetricRegistry metricRegistry)
+ {
+ this.metricRegistry = metricRegistry;
+ }
+
+ public StreamSSTableComponentMetrics forComponent(String component)
+ {
+ return streamComponentMetrics
+ .computeIfAbsent(component, sstableComponent -> new StreamSSTableComponentMetrics(metricRegistry, sstableComponent));
+ }
+
+ /**
+ * Metrics tracked during streaming of a specific SSTable component
+ */
+ public static class StreamSSTableComponentMetrics
+ {
+ protected final MetricRegistry metricRegistry;
+ public final String sstableComponent;
+ public final NamedMetric<Meter> rateLimitedCalls;
+ public final NamedMetric<Timer> sendFileLatency;
+
+ public StreamSSTableComponentMetrics(MetricRegistry metricRegistry, String sstableComponent)
+ {
+ this.metricRegistry = Objects.requireNonNull(metricRegistry, "Metric registry can not be null");
+ this.sstableComponent
+ = Objects.requireNonNull(sstableComponent, "SSTable component required for component specific metrics capture");
+ if (sstableComponent.isEmpty())
+ {
+ throw new IllegalArgumentException("SSTableComponent required for component specific metrics capture");
+ }
+
+ NamedMetric.Tag componentTag = NamedMetric.Tag.of("component", sstableComponent);
+
+ rateLimitedCalls
+ = NamedMetric.builder(metricRegistry::meter)
+ .withDomain(DOMAIN)
+ .withName("throttled_429")
+ .addTag(componentTag)
+ .build();
+ sendFileLatency
+ = NamedMetric.builder(metricRegistry::timer)
+ .withDomain(DOMAIN)
+ .withName("sendfile_latency")
+ .addTag(componentTag)
+ .build();
+ }
+ }
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/metrics/instance/UploadSSTableMetrics.java b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/UploadSSTableMetrics.java
new file mode 100644
index 00000000..010bbc07
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/metrics/instance/UploadSSTableMetrics.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.metrics.instance;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import org.apache.cassandra.sidecar.metrics.NamedMetric;
+
+import static org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics.INSTANCE_PREFIX;
+
+/**
+ * {@link UploadSSTableMetrics} tracks metrics captured during upload of SSTable components into Sidecar.
+ */
+public class UploadSSTableMetrics
+{
+ public static final String DOMAIN = INSTANCE_PREFIX + ".upload_sstable";
+ protected final MetricRegistry metricRegistry;
+ protected final Map<String, UploadSSTableComponentMetrics> uploadComponentMetrics = new ConcurrentHashMap<>();
+
+ public UploadSSTableMetrics(MetricRegistry metricRegistry)
+ {
+ this.metricRegistry = metricRegistry;
+ }
+
+ public UploadSSTableComponentMetrics forComponent(String component)
+ {
+ return uploadComponentMetrics
+ .computeIfAbsent(component, sstableComponent -> new UploadSSTableComponentMetrics(metricRegistry, sstableComponent));
+ }
+
+ /**
+ * Metrics tracking during upload of a specific SSTable component
+ */
+ public static class UploadSSTableComponentMetrics
+ {
+ protected final MetricRegistry metricRegistry;
+ public final String sstableComponent;
+ public final NamedMetric<Meter> rateLimitedCalls;
+ public final NamedMetric<Meter> diskUsageHigh;
+
+ public UploadSSTableComponentMetrics(MetricRegistry metricRegistry, String sstableComponent)
+ {
+ this.metricRegistry = Objects.requireNonNull(metricRegistry, "Metric registry can not be null");
+ this.sstableComponent
+ = Objects.requireNonNull(sstableComponent, "SSTable component required for component specific metrics capture");
+ if (sstableComponent.isEmpty())
+ {
+ throw new IllegalArgumentException("SSTableComponent required for component specific metrics capture");
+ }
+
+ NamedMetric.Tag componentTag = NamedMetric.Tag.of("component", sstableComponent);
+
+ rateLimitedCalls
+ = NamedMetric.builder(metricRegistry::meter)
+ .withDomain(DOMAIN)
+ .withName("throttled_429")
+ .addTag(componentTag)
+ .build();
+ diskUsageHigh
+ = NamedMetric.builder(metricRegistry::meter)
+ .withDomain(DOMAIN)
+ .withName("disk_usage_high")
+ .addTag(componentTag)
+ .build();
+ }
+ }
+
+}
diff --git a/src/main/java/org/apache/cassandra/sidecar/routes/DiskSpaceProtectionHandler.java b/src/main/java/org/apache/cassandra/sidecar/routes/DiskSpaceProtectionHandler.java
index e3a35853..c4657928 100644
--- a/src/main/java/org/apache/cassandra/sidecar/routes/DiskSpaceProtectionHandler.java
+++ b/src/main/java/org/apache/cassandra/sidecar/routes/DiskSpaceProtectionHandler.java
@@ -99,6 +99,7 @@ public class DiskSpaceProtectionHandler extends AbstractHandler<Void>
.onFailure(throwable -> {
if (throwable instanceof InsufficientStorageException)
{
+ instance.metrics().resource().insufficientStagingSpace.metric.mark();
InsufficientStorageException exception = (InsufficientStorageException) throwable;
throwable = wrapHttpException(HttpResponseStatus.INSUFFICIENT_STORAGE,
exception.getMessage(),
diff --git a/src/main/java/org/apache/cassandra/sidecar/routes/FileStreamHandler.java b/src/main/java/org/apache/cassandra/sidecar/routes/FileStreamHandler.java
index ec8d8ecb..2f06d72f 100644
--- a/src/main/java/org/apache/cassandra/sidecar/routes/FileStreamHandler.java
+++ b/src/main/java/org/apache/cassandra/sidecar/routes/FileStreamHandler.java
@@ -26,6 +26,7 @@ import io.vertx.core.file.FileSystem;
import io.vertx.core.http.HttpServerRequest;
import io.vertx.core.net.SocketAddress;
import io.vertx.ext.web.RoutingContext;
+import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata;
import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.models.HttpResponse;
import org.apache.cassandra.sidecar.utils.FileStreamer;
@@ -59,11 +60,13 @@ public class FileStreamHandler extends AbstractHandler<String>
SocketAddress remoteAddress,
String localFile)
{
+ InstanceMetadata instanceMetadata = metadataFetcher.instance(host);
FileSystem fs = context.vertx().fileSystem();
fs.exists(localFile)
.compose(exists -> ensureValidFile(fs, localFile, exists))
- .compose(fileProps -> fileStreamer.stream(new HttpResponse(httpRequest, context.response()), localFile,
- fileProps.size(), httpRequest.getHeader(HttpHeaderNames.RANGE)))
+ .compose(fileProps -> fileStreamer.stream(new HttpResponse(httpRequest, context.response()),
+ instanceMetadata.id(), localFile, fileProps.size(),
+ httpRequest.getHeader(HttpHeaderNames.RANGE)))
.onSuccess(v -> logger.debug("Completed streaming file '{}'", localFile))
.onFailure(context::fail);
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandler.java b/src/main/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandler.java
index 9347f7ff..ee2dfd7b 100644
--- a/src/main/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandler.java
+++ b/src/main/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandler.java
@@ -38,6 +38,8 @@ import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.config.SSTableUploadConfiguration;
import org.apache.cassandra.sidecar.config.ServiceConfiguration;
import org.apache.cassandra.sidecar.data.SSTableUploadRequest;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics;
+import org.apache.cassandra.sidecar.metrics.instance.UploadSSTableMetrics;
import org.apache.cassandra.sidecar.routes.AbstractHandler;
import org.apache.cassandra.sidecar.stats.SSTableStats;
import org.apache.cassandra.sidecar.stats.SidecarStats;
@@ -51,6 +53,7 @@ import org.apache.cassandra.sidecar.utils.SSTableUploadsPathBuilder;
import static org.apache.cassandra.sidecar.utils.HttpExceptions.cassandraServiceUnavailable;
import static org.apache.cassandra.sidecar.utils.HttpExceptions.wrapHttpException;
+import static org.apache.cassandra.sidecar.utils.MetricUtils.sstableExtension;
/**
* Handler for managing uploaded SSTable components
@@ -116,10 +119,15 @@ public class SSTableUploadHandler extends AbstractHandler<SSTableUploadRequest>
// accept the upload.
httpRequest.pause();
+ InstanceMetrics instanceMetrics = metadataFetcher.instance(host).metrics();
+ UploadSSTableMetrics.UploadSSTableComponentMetrics componentMetrics
+ = instanceMetrics.uploadSSTable().forComponent(sstableExtension(request.component()));
+
long startTimeInNanos = System.nanoTime();
if (!limiter.tryAcquire())
{
String message = String.format("Concurrent upload limit (%d) exceeded", limiter.limit());
+ componentMetrics.rateLimitedCalls.metric.mark();
context.fail(wrapHttpException(HttpResponseStatus.TOO_MANY_REQUESTS, message));
return;
}
@@ -128,7 +136,8 @@ public class SSTableUploadHandler extends AbstractHandler<SSTableUploadRequest>
validateKeyspaceAndTable(host, request)
.compose(validRequest -> uploadPathBuilder.resolveStagingDirectory(host))
- .compose(this::ensureSufficientSpaceAvailable)
+ .compose(uploadDir -> ensureSufficientSpaceAvailable(uploadDir,
+ componentMetrics))
.compose(v -> uploadPathBuilder.build(host, request))
.compose(uploadDirectory -> {
DigestVerifier digestVerifier = digestVerifierFactory.verifier(httpRequest.headers());
@@ -223,7 +232,8 @@ public class SSTableUploadHandler extends AbstractHandler<SSTableUploadRequest>
* @param uploadDirectory the directory where the SSTables are uploaded
* @return a succeeded future if there is sufficient space available, or failed future otherwise
*/
- private Future<String> ensureSufficientSpaceAvailable(String uploadDirectory)
+ private Future<String> ensureSufficientSpaceAvailable(String uploadDirectory,
+ UploadSSTableMetrics.UploadSSTableComponentMetrics metrics)
{
float minimumPercentageRequired = configuration.minimumSpacePercentageRequired();
if (minimumPercentageRequired == 0)
@@ -248,6 +258,7 @@ public class SSTableUploadHandler extends AbstractHandler<SSTableUploadRequest>
logger.warn("Insufficient space available for upload in stagingDir={}, available={}%, " +
"required={}%", uploadDirectory,
availableDiskSpacePercentage, minimumPercentageRequired);
+ metrics.diskUsageHigh.metric.mark();
return Future.failedFuture(wrapHttpException(HttpResponseStatus.INSUFFICIENT_STORAGE,
"Insufficient space available for upload"));
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java b/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java
index a85dbb6c..9db7d95a 100644
--- a/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java
+++ b/src/main/java/org/apache/cassandra/sidecar/server/MainModule.java
@@ -29,6 +29,8 @@ import com.google.common.util.concurrent.SidecarRateLimiter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
import com.datastax.driver.core.NettyOptions;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
@@ -39,6 +41,8 @@ import io.vertx.core.Vertx;
import io.vertx.core.VertxOptions;
import io.vertx.core.http.HttpMethod;
import io.vertx.ext.dropwizard.DropwizardMetricsOptions;
+import io.vertx.ext.dropwizard.Match;
+import io.vertx.ext.dropwizard.MatchType;
import io.vertx.ext.web.Router;
import io.vertx.ext.web.handler.BodyHandler;
import io.vertx.ext.web.handler.ErrorHandler;
@@ -65,6 +69,7 @@ import org.apache.cassandra.sidecar.config.InstanceConfiguration;
import org.apache.cassandra.sidecar.config.JmxConfiguration;
import org.apache.cassandra.sidecar.config.ServiceConfiguration;
import org.apache.cassandra.sidecar.config.SidecarConfiguration;
+import org.apache.cassandra.sidecar.config.VertxMetricsConfiguration;
import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl;
import org.apache.cassandra.sidecar.db.schema.RestoreJobsSchema;
import org.apache.cassandra.sidecar.db.schema.RestoreSlicesSchema;
@@ -137,12 +142,19 @@ public class MainModule extends AbstractModule
@Provides
@Singleton
- public Vertx vertx()
+ public Vertx vertx(SidecarConfiguration sidecarConfiguration)
{
- return Vertx.vertx(new VertxOptions().setMetricsOptions(new DropwizardMetricsOptions()
- .setEnabled(true)
- .setJmxEnabled(true)
- .setJmxDomain("cassandra-sidecar-metrics")));
+ VertxMetricsConfiguration metricsConfig = sidecarConfiguration.metricsConfiguration().vertxConfiguration();
+ DropwizardMetricsOptions dropwizardMetricsOptions
+ = new DropwizardMetricsOptions().setEnabled(metricsConfig.enabled())
+ .setJmxEnabled(metricsConfig.exposeViaJMX())
+ .setJmxDomain(metricsConfig.jmxDomainName())
+ .setRegistryName(sidecarConfiguration.metricsConfiguration().registryName());
+ for (String regex : metricsConfig.monitoredServerRouteRegexes())
+ {
+ dropwizardMetricsOptions.addMonitoredHttpServerRoute(new Match().setType(MatchType.REGEX).setValue(regex));
+ }
+ return Vertx.vertx(new VertxOptions().setMetricsOptions(dropwizardMetricsOptions));
}
@Provides
@@ -371,7 +383,8 @@ public class MainModule extends AbstractModule
sidecarVersionProvider.sidecarVersion(),
jmxConfiguration,
cqlSessionProvider,
- driverUtils);
+ driverUtils,
+ configuration.metricsConfiguration().registryName());
})
.collect(Collectors.toList());
@@ -510,6 +523,13 @@ public class MainModule extends AbstractModule
stats, sidecarInternalKeyspace, cqlSessionProvider);
}
+ @Provides
+ @Singleton
+ public MetricRegistry globalMetricRegistry(SidecarConfiguration sidecarConfiguration)
+ {
+ return SharedMetricRegistries.getOrCreate(sidecarConfiguration.metricsConfiguration().registryName());
+ }
+
/**
* The provided hasher is used in {@link org.apache.cassandra.sidecar.restore.RestoreJobUtil}
*/
@@ -533,12 +553,13 @@ public class MainModule extends AbstractModule
* Builds the {@link InstanceMetadata} from the {@link InstanceConfiguration},
* a provided {@code versionProvider}, and {@code healthCheckFrequencyMillis}.
*
- * @param vertx the vertx instance
- * @param cassandraInstance the cassandra instance configuration
- * @param versionProvider a Cassandra version provider
- * @param sidecarVersion the version of the Sidecar from the current binary
- * @param jmxConfiguration the configuration for the JMX Client
- * @param session the CQL Session provider
+ * @param vertx the vertx instance
+ * @param cassandraInstance the cassandra instance configuration
+ * @param versionProvider a Cassandra version provider
+ * @param sidecarVersion the version of the Sidecar from the current binary
+ * @param jmxConfiguration the configuration for the JMX Client
+ * @param session the CQL Session provider
+ * @param globalRegistryName global registry name used for tracking Sidecar metrics
* @return the build instance metadata object
*/
private static InstanceMetadata buildInstanceMetadata(Vertx vertx,
@@ -547,7 +568,8 @@ public class MainModule extends AbstractModule
String sidecarVersion,
JmxConfiguration jmxConfiguration,
CQLSessionProvider session,
- DriverUtils driverUtils)
+ DriverUtils driverUtils,
+ String globalRegistryName)
{
String host = cassandraInstance.host();
int port = cassandraInstance.port();
@@ -577,6 +599,7 @@ public class MainModule extends AbstractModule
.dataDirs(cassandraInstance.dataDirs())
.stagingDir(cassandraInstance.stagingDir())
.delegate(delegate)
+ .globalMetricRegistryName(globalRegistryName)
.build();
}
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/server/Server.java b/src/main/java/org/apache/cassandra/sidecar/server/Server.java
index 3c9b0a78..51b67b2d 100644
--- a/src/main/java/org/apache/cassandra/sidecar/server/Server.java
+++ b/src/main/java/org/apache/cassandra/sidecar/server/Server.java
@@ -52,6 +52,7 @@ import org.apache.cassandra.sidecar.common.utils.Preconditions;
import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.config.SidecarConfiguration;
import org.apache.cassandra.sidecar.config.SslConfiguration;
+import org.apache.cassandra.sidecar.metrics.ServerMetrics;
import org.apache.cassandra.sidecar.tasks.HealthCheckPeriodicTask;
import org.apache.cassandra.sidecar.tasks.KeyStoreCheckPeriodicTask;
import org.apache.cassandra.sidecar.tasks.PeriodicTaskExecutor;
@@ -75,6 +76,7 @@ public class Server
protected final Router router;
protected final PeriodicTaskExecutor periodicTaskExecutor;
protected final HttpServerOptionsProvider optionsProvider;
+ protected final ServerMetrics serverMetrics;
protected final List<ServerVerticle> deployedServerVerticles = new CopyOnWriteArrayList<>();
// Keeps track of all the Cassandra instance identifiers where CQL is ready
private final Set<Integer> cqlReadyInstanceIds = Collections.synchronizedSet(new HashSet<>());
@@ -86,7 +88,8 @@ public class Server
InstancesConfig instancesConfig,
ExecutorPools executorPools,
PeriodicTaskExecutor periodicTaskExecutor,
- HttpServerOptionsProvider optionsProvider)
+ HttpServerOptionsProvider optionsProvider,
+ ServerMetrics serverMetrics)
{
this.vertx = vertx;
this.executorPools = executorPools;
@@ -95,6 +98,7 @@ public class Server
this.router = router;
this.periodicTaskExecutor = periodicTaskExecutor;
this.optionsProvider = optionsProvider;
+ this.serverMetrics = serverMetrics;
}
/**
@@ -274,7 +278,8 @@ public class Server
periodicTaskExecutor.schedule(new HealthCheckPeriodicTask(vertx,
sidecarConfiguration,
instancesConfig,
- executorPools));
+ executorPools,
+ serverMetrics));
maybeScheduleKeyStoreCheckPeriodicTask();
MessageConsumer<JsonObject> cqlReadyConsumer = vertx.eventBus().localConsumer(ON_CASSANDRA_CQL_READY.address());
diff --git a/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java b/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
index 28eefaba..8eb8ece5 100644
--- a/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
+++ b/src/main/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTask.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.sidecar.tasks;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@@ -31,6 +32,7 @@ import io.vertx.core.eventbus.EventBus;
import org.apache.cassandra.sidecar.cluster.InstancesConfig;
import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.config.SidecarConfiguration;
+import org.apache.cassandra.sidecar.metrics.ServerMetrics;
import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_START;
import static org.apache.cassandra.sidecar.server.SidecarServerEvents.ON_SERVER_STOP;
@@ -45,16 +47,19 @@ public class HealthCheckPeriodicTask implements PeriodicTask
private final SidecarConfiguration configuration;
private final InstancesConfig instancesConfig;
private final ExecutorPools.TaskExecutorPool internalPool;
+ private final ServerMetrics serverMetrics;
public HealthCheckPeriodicTask(Vertx vertx,
SidecarConfiguration configuration,
InstancesConfig instancesConfig,
- ExecutorPools executorPools)
+ ExecutorPools executorPools,
+ ServerMetrics serverMetrics)
{
eventBus = vertx.eventBus();
this.configuration = configuration;
this.instancesConfig = instancesConfig;
internalPool = executorPools.internal();
+ this.serverMetrics = serverMetrics;
}
@Override
@@ -82,6 +87,7 @@ public class HealthCheckPeriodicTask implements PeriodicTask
@Override
public void execute(Promise<Void> promise)
{
+ AtomicInteger instanceDown = new AtomicInteger(0);
List<Future<?>> futures = instancesConfig.instances()
.stream()
.map(instanceMetadata -> internalPool.executeBlocking(p -> {
@@ -92,6 +98,7 @@ public class HealthCheckPeriodicTask implements PeriodicTask
}
catch (Throwable cause)
{
+ instanceDown.incrementAndGet();
p.fail(cause);
LOGGER.error("Unable to complete health check on instance={}",
instanceMetadata.id(), cause);
@@ -101,6 +108,11 @@ public class HealthCheckPeriodicTask implements PeriodicTask
// join always waits until all its futures are completed and will not fail as soon as one of the future fails
Future.join(futures)
+ .onComplete(v -> {
+ int instancesUp = instancesConfig.instances().size() - instanceDown.get();
+ serverMetrics.cassandraInstancesUp.metric.setValue(instancesUp);
+ serverMetrics.cassandraInstancesDown.metric.setValue(instanceDown.get());
+ })
.onSuccess(v -> promise.complete())
.onFailure(promise::fail);
}
diff --git a/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java b/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java
index 2f133af3..69192d1f 100644
--- a/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java
+++ b/src/main/java/org/apache/cassandra/sidecar/utils/FileStreamer.java
@@ -37,6 +37,8 @@ import org.apache.cassandra.sidecar.common.utils.HttpRange;
import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.config.ServiceConfiguration;
import org.apache.cassandra.sidecar.config.ThrottleConfiguration;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetrics;
+import org.apache.cassandra.sidecar.metrics.instance.StreamSSTableMetrics;
import org.apache.cassandra.sidecar.models.HttpResponse;
import org.apache.cassandra.sidecar.stats.SSTableStats;
import org.apache.cassandra.sidecar.stats.SidecarStats;
@@ -44,6 +46,7 @@ import org.apache.cassandra.sidecar.stats.SidecarStats;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
import static io.netty.handler.codec.http.HttpResponseStatus.REQUESTED_RANGE_NOT_SATISFIABLE;
import static io.netty.handler.codec.http.HttpResponseStatus.TOO_MANY_REQUESTS;
+import static org.apache.cassandra.sidecar.utils.MetricUtils.sstableExtension;
/**
* General handler for serving files
@@ -57,17 +60,20 @@ public class FileStreamer
private final ThrottleConfiguration config;
private final SidecarRateLimiter rateLimiter;
private final SSTableStats stats;
+ private final InstanceMetadataFetcher instanceMetadataFetcher;
@Inject
public FileStreamer(ExecutorPools executorPools,
ServiceConfiguration config,
@Named("StreamRequestRateLimiter") SidecarRateLimiter rateLimiter,
- SidecarStats stats)
+ SidecarStats stats,
+ InstanceMetadataFetcher instanceMetadataFetcher)
{
this.executorPools = executorPools;
this.config = config.throttleConfiguration();
this.rateLimiter = rateLimiter;
this.stats = stats.ssTableStats();
+ this.instanceMetadataFetcher = instanceMetadataFetcher;
}
/**
@@ -75,15 +81,17 @@ public class FileStreamer
* {@code rangeHeader} using the provided {@code response}.
*
* @param response the response to use
+ * @param instanceId Cassandra instance from which file is streamed
* @param filename the path to the file to serve
* @param fileLength the size of the file to serve
* @param rangeHeader (optional) a string representing the requested range for the file
* @return a future with the result of the streaming
*/
- public Future<Void> stream(HttpResponse response, String filename, long fileLength, String rangeHeader)
+ public Future<Void> stream(HttpResponse response, int instanceId, String filename,
+ long fileLength, String rangeHeader)
{
return parseRangeHeader(rangeHeader, fileLength)
- .compose(range -> stream(response, filename, fileLength, range));
+ .compose(range -> stream(response, instanceId, filename, fileLength, range));
}
/**
@@ -91,15 +99,16 @@ public class FileStreamer
* {@code range} using the provided {@code response}.
*
* @param response the response to use
+ * @param instanceId Cassandra instance from which file is streamed
* @param filename the path to the file to serve
* @param fileLength the size of the file to serve
* @param range the range to stream
* @return a future with the result of the streaming
*/
- public Future<Void> stream(HttpResponse response, String filename, long fileLength, HttpRange range)
+ public Future<Void> stream(HttpResponse response, int instanceId, String filename, long fileLength, HttpRange range)
{
Promise<Void> promise = Promise.promise();
- acquireAndSend(response, filename, fileLength, range, Instant.now(), promise);
+ acquireAndSend(response, instanceId, filename, fileLength, range, Instant.now(), promise);
return promise.future();
}
@@ -108,6 +117,7 @@ public class FileStreamer
* {@link SidecarRateLimiter}.
*
* @param response the response to use
+ * @param instanceId Cassandra instance from which file is streamed
* @param filename the path to the file to serve
* @param fileLength the size of the file to serve
* @param range the range to stream
@@ -115,20 +125,28 @@ public class FileStreamer
* @param promise a promise for the stream
*/
private void acquireAndSend(HttpResponse response,
+ int instanceId,
String filename,
long fileLength,
HttpRange range,
Instant startTime,
Promise<Void> promise)
{
- if (acquire(response, filename, fileLength, range, startTime, promise))
+ InstanceMetrics instanceMetrics = instanceMetadataFetcher.instance(instanceId).metrics();
+ String component = sstableExtension(filename);
+ StreamSSTableMetrics.StreamSSTableComponentMetrics componentMetrics
+ = instanceMetrics.streamSSTable().forComponent(component);
+ if (acquire(response, instanceId, filename, fileLength, range, startTime, componentMetrics, promise))
{
// Stream data if rate limiting is disabled or if we acquire
LOGGER.debug("Streaming range {} for file {} to client {}. Instance: {}", range, filename,
response.remoteAddress(), response.host());
+ long startTimeSendFile = System.nanoTime();
response.sendFile(filename, fileLength, range)
.onSuccess(v ->
{
+ long d = System.nanoTime() - startTimeSendFile;
+ componentMetrics.sendFileLatency.metric.update(d, TimeUnit.NANOSECONDS);
LOGGER.debug("Streamed file {} successfully to client {}. Instance: {}", filename,
response.remoteAddress(), response.host());
stats.onBytesStreamed(fileLength);
@@ -143,16 +161,18 @@ public class FileStreamer
* delay. Otherwise, it will retry acquiring the permit later in the future until it exhausts the
* retry timeout, in which case it will ask the client to retry later in the future.
*
- * @param response the response to use
- * @param filename the path to the file to serve
- * @param fileLength the size of the file to serve
- * @param range the range to stream
- * @param startTime the start time of this request
- * @param promise a promise for the stream
+ * @param response the response to use
+ * @param instanceId Cassandra instance from which file is streamed
+ * @param filename the path to the file to serve
+ * @param fileLength the size of the file to serve
+ * @param range the range to stream
+ * @param startTime the start time of this request
+ * @param componentMetrics metrics captured during streaming of specific SSTable component
+ * @param promise a promise for the stream
* @return {@code true} if the permit was acquired, {@code false} otherwise
*/
- private boolean acquire(HttpResponse response, String filename, long fileLength, HttpRange range, Instant startTime,
- Promise<Void> promise)
+ private boolean acquire(HttpResponse response, int instanceId, String filename, long fileLength, HttpRange range,
+ Instant startTime, StreamSSTableMetrics.StreamSSTableComponentMetrics componentMetrics, Promise<Void> promise)
{
if (rateLimiter.tryAcquire())
return true;
@@ -173,13 +193,15 @@ public class FileStreamer
response.remoteAddress(), response.host());
executorPools.service()
.setTimer(MICROSECONDS.toMillis(microsToWait),
- t -> acquireAndSend(response, filename, fileLength, range, startTime, promise));
+ t -> acquireAndSend(response, instanceId, filename, fileLength, range,
+ startTime, promise));
}
else
{
LOGGER.debug("Asking client {} to retry after {} micros. Instance: {}", response.remoteAddress(),
microsToWait, response.host());
response.setRetryAfterHeader(microsToWait);
+ componentMetrics.rateLimitedCalls.metric.mark();
promise.fail(new HttpException(TOO_MANY_REQUESTS.code(), "Ask client to retry later"));
}
return false;
diff --git a/src/main/java/org/apache/cassandra/sidecar/utils/MetricUtils.java b/src/main/java/org/apache/cassandra/sidecar/utils/MetricUtils.java
new file mode 100644
index 00000000..f11788ae
--- /dev/null
+++ b/src/main/java/org/apache/cassandra/sidecar/utils/MetricUtils.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.utils;
+
+/**
+ * Utilities for Metrics related operations
+ */
+public class MetricUtils
+{
+ /**
+ * Returns extension of SSTable components. For e.g. for nb-1-big-Data.db component returns db. This extension
+ * is used for marking metrics that are captured specific to SSTable components.
+ *
+ * @param filename name of SSTable component
+ * @return String extension of SSTable component, if none found "default" is returned
+ */
+ public static String sstableExtension(String filename)
+ {
+ int index = filename.lastIndexOf('.');
+ if (index != -1)
+ {
+ return filename.substring(index + 1);
+ }
+ return "default";
+ }
+}
diff --git a/src/test/integration/org/apache/cassandra/sidecar/testing/CassandraSidecarTestContext.java b/src/test/integration/org/apache/cassandra/sidecar/testing/CassandraSidecarTestContext.java
index 0b97589c..e101e391 100644
--- a/src/test/integration/org/apache/cassandra/sidecar/testing/CassandraSidecarTestContext.java
+++ b/src/test/integration/org/apache/cassandra/sidecar/testing/CassandraSidecarTestContext.java
@@ -257,6 +257,7 @@ public class CassandraSidecarTestContext implements AutoCloseable
.dataDirs(Arrays.asList(dataDirectories))
.stagingDir(stagingDir)
.delegate(delegate)
+ .globalMetricRegistryName("cassandra_sidecar")
.build());
}
return new InstancesConfigImpl(metadata, dnsResolver);
diff --git a/src/test/java/org/apache/cassandra/sidecar/TestModule.java b/src/test/java/org/apache/cassandra/sidecar/TestModule.java
index 68bb4d87..c69e06ad 100644
--- a/src/test/java/org/apache/cassandra/sidecar/TestModule.java
+++ b/src/test/java/org/apache/cassandra/sidecar/TestModule.java
@@ -55,10 +55,12 @@ import org.apache.cassandra.sidecar.config.yaml.SchemaKeyspaceConfigurationImpl;
import org.apache.cassandra.sidecar.config.yaml.ServiceConfigurationImpl;
import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl;
import org.apache.cassandra.sidecar.config.yaml.ThrottleConfigurationImpl;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetricsImpl;
import org.apache.cassandra.sidecar.stats.RestoreJobStats;
import org.apache.cassandra.sidecar.stats.TestRestoreJobStats;
import org.apache.cassandra.sidecar.utils.CassandraVersionProvider;
+import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -172,6 +174,7 @@ public class TestModule extends AbstractModule
when(instanceMeta.port()).thenReturn(6475);
when(instanceMeta.stagingDir()).thenReturn(stagingDir);
when(instanceMeta.dataDirs()).thenReturn(Collections.singletonList(dataDir));
+ when(instanceMeta.metrics()).thenReturn(new InstanceMetricsImpl(registry(id)));
CassandraAdapterDelegate delegate = mock(CassandraAdapterDelegate.class);
Metadata metadata = mock(Metadata.class);
diff --git a/src/test/java/org/apache/cassandra/sidecar/ThrottleTest.java b/src/test/java/org/apache/cassandra/sidecar/ThrottleTest.java
index 50d06814..7d22f709 100644
--- a/src/test/java/org/apache/cassandra/sidecar/ThrottleTest.java
+++ b/src/test/java/org/apache/cassandra/sidecar/ThrottleTest.java
@@ -39,11 +39,14 @@ import io.vertx.ext.web.client.WebClient;
import io.vertx.ext.web.codec.BodyCodec;
import io.vertx.junit5.VertxExtension;
import io.vertx.junit5.VertxTestContext;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetricsImpl;
+import org.apache.cassandra.sidecar.metrics.instance.StreamSSTableMetrics;
import org.apache.cassandra.sidecar.server.MainModule;
import org.apache.cassandra.sidecar.server.Server;
import static java.util.concurrent.TimeUnit.SECONDS;
-import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
+import static org.assertj.core.api.Assertions.assertThat;
/**
* Test rate limiting stream requests
@@ -73,6 +76,8 @@ public class ThrottleTest
void tearDown() throws InterruptedException
{
CountDownLatch closeLatch = new CountDownLatch(1);
+ registry().removeMatching((name, metric) -> true);
+ registry(1).removeMatching((name, metric) -> true);
server.close().onSuccess(res -> closeLatch.countDown());
if (closeLatch.await(60, SECONDS))
logger.info("Close event received before timeout.");
@@ -91,15 +96,19 @@ public class ThrottleTest
unblockingClientRequest(testRoute);
}
+ StreamSSTableMetrics.StreamSSTableComponentMetrics dbComponentMetrics
+ = new InstanceMetricsImpl(registry(1)).streamSSTable().forComponent("db");
+
HttpResponse response = blockingClientRequest(testRoute);
- assertEquals(HttpResponseStatus.TOO_MANY_REQUESTS.code(), response.statusCode());
+ assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.TOO_MANY_REQUESTS.code());
+ assertThat(dbComponentMetrics.rateLimitedCalls.metric.getCount()).isGreaterThanOrEqualTo(1);
long secsToWait = Long.parseLong(response.getHeader("Retry-After"));
Thread.sleep(SECONDS.toMillis(secsToWait));
HttpResponse finalResp = blockingClientRequest(testRoute);
- assertEquals(HttpResponseStatus.OK.code(), finalResp.statusCode());
- assertEquals("data", finalResp.bodyAsString());
+ assertThat(finalResp.statusCode()).isEqualTo(HttpResponseStatus.OK.code());
+ assertThat(finalResp.bodyAsString()).isEqualTo("data");
}
private void unblockingClientRequest(String route)
diff --git a/src/test/java/org/apache/cassandra/sidecar/config/SidecarConfigurationTest.java b/src/test/java/org/apache/cassandra/sidecar/config/SidecarConfigurationTest.java
index 8775c670..3498d970 100644
--- a/src/test/java/org/apache/cassandra/sidecar/config/SidecarConfigurationTest.java
+++ b/src/test/java/org/apache/cassandra/sidecar/config/SidecarConfigurationTest.java
@@ -23,15 +23,23 @@ import java.net.InetSocketAddress;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
+import java.util.regex.Pattern;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import com.fasterxml.jackson.databind.JsonMappingException;
import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl;
+import org.apache.cassandra.sidecar.config.yaml.VertxMetricsConfigurationImpl;
import org.assertj.core.api.Condition;
+import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.COMPONENTS_ROUTE;
+import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.KEYSPACE_SCHEMA_ROUTE;
+import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.RESTORE_JOB_SLICES_ROUTE;
+import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.TIME_SKEW_ROUTE;
import static org.apache.cassandra.sidecar.common.ResourceUtils.writeResourceToPath;
+import static org.apache.cassandra.sidecar.config.yaml.MetricsConfigurationImpl.DEFAULT_DROPWIZARD_REGISTRY_NAME;
+import static org.apache.cassandra.sidecar.config.yaml.VertxMetricsConfigurationImpl.DEFAULT_JMX_DOMAIN_NAME;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType;
@@ -206,6 +214,35 @@ class SidecarConfigurationTest
.isEqualTo("{'class':'SimpleStrategy', 'replication_factor':'3'}");
}
+ @Test
+ void testMetricOptionsParsedFromYaml() throws IOException
+ {
+ Path yamlPath = yaml("config/sidecar_metrics.yaml");
+ SidecarConfiguration config = SidecarConfigurationImpl.readYamlConfiguration(yamlPath);
+
+ MetricsConfiguration configuration = config.metricsConfiguration();
+ assertThat(configuration.registryName()).isEqualTo(DEFAULT_DROPWIZARD_REGISTRY_NAME);
+ VertxMetricsConfiguration vertxMetricsConfiguration = configuration.vertxConfiguration();
+ assertThat(vertxMetricsConfiguration.enabled()).isTrue();
+ assertThat(vertxMetricsConfiguration.exposeViaJMX()).isFalse();
+ assertThat(vertxMetricsConfiguration.jmxDomainName()).isEqualTo(DEFAULT_JMX_DOMAIN_NAME);
+ assertThat(vertxMetricsConfiguration.monitoredServerRouteRegexes().size()).isEqualTo(2);
+ assertThat(vertxMetricsConfiguration.monitoredServerRouteRegexes().get(0)).isEqualTo("/api/v1/keyspaces/.*");
+ assertThat(vertxMetricsConfiguration.monitoredServerRouteRegexes().get(1)).isEqualTo("/api/v1/cassandra/.*");
+ }
+
+ @Test
+ void testRoutesAllowedWithDefaultMonitoredRegex()
+ {
+ List<String> defaultMonitoredRegexes = VertxMetricsConfigurationImpl.DEFAULT_MONITORED_SERVER_ROUTE_REGEXES;
+ assertThat(defaultMonitoredRegexes.size()).isOne();
+ Pattern pattern = Pattern.compile(defaultMonitoredRegexes.get(0));
+ assertThat(pattern.matcher(COMPONENTS_ROUTE)).matches();
+ assertThat(pattern.matcher(KEYSPACE_SCHEMA_ROUTE)).matches();
+ assertThat(pattern.matcher(TIME_SKEW_ROUTE)).matches();
+ assertThat(pattern.matcher(RESTORE_JOB_SLICES_ROUTE)).matches();
+ }
+
void validateSingleInstanceSidecarConfiguration(SidecarConfiguration config)
{
assertThat(config.cassandraInstances()).isNotNull().hasSize(1);
@@ -235,6 +272,9 @@ class SidecarConfigurationTest
// health check configuration
validateHealthCheckConfigurationFromYaml(config.healthCheckConfiguration());
+ // metrics configuration
+ validateMetricsConfiguration(config.metricsConfiguration());
+
// cassandra input validation configuration
validateCassandraInputValidationConfigurationFromYaml(config.cassandraInputValidationConfiguration());
}
@@ -300,6 +340,9 @@ class SidecarConfigurationTest
// health check configuration
validateHealthCheckConfigurationFromYaml(config.healthCheckConfiguration());
+ // metrics configuration
+ validateMetricsConfiguration(config.metricsConfiguration());
+
// cassandra input validation configuration
validateCassandraInputValidationConfigurationFromYaml(config.cassandraInputValidationConfiguration());
}
@@ -386,6 +429,12 @@ class SidecarConfigurationTest
"TLS_RSA_WITH_AES_256_CBC_SHA");
}
+ void validateMetricsConfiguration(MetricsConfiguration config)
+ {
+ assertThat(config.vertxConfiguration()).isNotNull();
+ assertThat(config.vertxConfiguration().monitoredServerRouteRegexes()).isNotNull();
+ }
+
private Path yaml(String resourceName)
{
ClassLoader classLoader = this.getClass().getClassLoader();
diff --git a/src/test/java/org/apache/cassandra/sidecar/metrics/NamedMetricTest.java b/src/test/java/org/apache/cassandra/sidecar/metrics/NamedMetricTest.java
new file mode 100644
index 00000000..c053dd19
--- /dev/null
+++ b/src/test/java/org/apache/cassandra/sidecar/metrics/NamedMetricTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.metrics;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+
+import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+/**
+ * Tests for {@link NamedMetric}
+ */
+public class NamedMetricTest
+{
+ private Logger logger = LoggerFactory.getLogger(NamedMetricTest.class);
+ private MetricRegistry registry = registry();
+
+ @AfterEach
+ void clear()
+ {
+ registry.removeMatching((name, metric) -> true);
+ }
+
+ @Test
+ void printAllMetricNames()
+ {
+ registry.getMetrics().forEach((name, metric) -> logger.info("Metric name: {}, value: {}", name, metric));
+ }
+
+ @Test
+ void testGaugeRetrieved()
+ {
+ NamedMetric namedMetric = NamedMetric.builder(name -> registry.gauge(name))
+ .withName("testGauge")
+ .withDomain("testDomain")
+ .addTag("key", "value")
+ .build();
+ assertThat(namedMetric.metric).isInstanceOf(Gauge.class);
+ assertThat(namedMetric.canonicalName).isEqualTo("testDomain.key=value.testGauge");
+ }
+
+ @Test
+ void testCounterRetrieved()
+ {
+ NamedMetric namedMetric = NamedMetric.builder(name -> registry.counter(name))
+ .withName("testCounter")
+ .withDomain("testDomain")
+ .addTag("key", "value")
+ .build();
+ assertThat(namedMetric.metric).isInstanceOf(Counter.class);
+ assertThat(namedMetric.canonicalName).isEqualTo("testDomain.key=value.testCounter");
+ }
+
+ @Test
+ void testHistogramRetrieved()
+ {
+ NamedMetric namedMetric = NamedMetric.builder(name -> registry.histogram(name))
+ .withName("testHistogram")
+ .withDomain("testDomain")
+ .addTag("key", "value")
+ .build();
+ assertThat(namedMetric.metric).isInstanceOf(Histogram.class);
+ assertThat(namedMetric.canonicalName).isEqualTo("testDomain.key=value.testHistogram");
+ }
+
+ @Test
+ void testMeterRetrieved()
+ {
+ NamedMetric namedMetric = NamedMetric.builder(name -> registry.meter(name))
+ .withName("testMeter")
+ .withDomain("testDomain")
+ .addTag("key", "value")
+ .build();
+ assertThat(namedMetric.metric).isInstanceOf(Meter.class);
+ assertThat(namedMetric.canonicalName).isEqualTo("testDomain.key=value.testMeter");
+ }
+}
diff --git a/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobManagerTest.java b/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobManagerTest.java
index 3d20810a..60f76479 100644
--- a/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobManagerTest.java
+++ b/src/test/java/org/apache/cassandra/sidecar/restore/RestoreJobManagerTest.java
@@ -36,8 +36,10 @@ import org.junit.jupiter.api.io.TempDir;
import com.datastax.driver.core.utils.UUIDs;
import com.google.inject.Guice;
import com.google.inject.Injector;
+import com.google.inject.util.Modules;
import io.vertx.core.Vertx;
import org.apache.cassandra.sidecar.ExecutorPoolsHelper;
+import org.apache.cassandra.sidecar.TestModule;
import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata;
import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.config.RestoreJobConfiguration;
@@ -66,7 +68,7 @@ class RestoreJobManagerTest
@BeforeEach
void setup()
{
- Injector injector = Guice.createInjector(new MainModule());
+ Injector injector = Guice.createInjector(Modules.override(new MainModule()).with(new TestModule()));
vertx = injector.getInstance(Vertx.class);
executorPools = ExecutorPoolsHelper.createdSharedTestPool(vertx);
RestoreProcessor processor = mock(RestoreProcessor.class);
diff --git a/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/BaseUploadsHandlerTest.java b/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/BaseUploadsHandlerTest.java
index 6cf72c14..d45949f3 100644
--- a/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/BaseUploadsHandlerTest.java
+++ b/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/BaseUploadsHandlerTest.java
@@ -70,6 +70,7 @@ import static org.apache.cassandra.sidecar.config.yaml.TrafficShapingConfigurati
import static org.apache.cassandra.sidecar.config.yaml.TrafficShapingConfigurationImpl.DEFAULT_OUTBOUND_GLOBAL_BANDWIDTH_LIMIT;
import static org.apache.cassandra.sidecar.config.yaml.TrafficShapingConfigurationImpl.DEFAULT_PEAK_OUTBOUND_GLOBAL_BANDWIDTH_LIMIT;
import static org.apache.cassandra.sidecar.snapshots.SnapshotUtils.mockInstancesConfig;
+import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -157,6 +158,8 @@ class BaseUploadsHandlerTest
void tearDown() throws InterruptedException
{
final CountDownLatch closeLatch = new CountDownLatch(1);
+ registry().removeMatching((name, metric) -> true);
+ registry(1).removeMatching((name, metric) -> true);
server.close().onSuccess(res -> closeLatch.countDown());
if (closeLatch.await(60, TimeUnit.SECONDS))
logger.debug("Close event received before timeout.");
diff --git a/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandlerTest.java b/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandlerTest.java
index caf76e67..f3dc85de 100644
--- a/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandlerTest.java
+++ b/src/test/java/org/apache/cassandra/sidecar/routes/sstableuploads/SSTableUploadHandlerTest.java
@@ -47,6 +47,8 @@ import org.apache.cassandra.sidecar.common.data.Digest;
import org.apache.cassandra.sidecar.common.data.MD5Digest;
import org.apache.cassandra.sidecar.common.data.XXHash32Digest;
import org.apache.cassandra.sidecar.common.http.SidecarHttpResponseStatus;
+import org.apache.cassandra.sidecar.metrics.instance.InstanceMetricsImpl;
+import org.apache.cassandra.sidecar.metrics.instance.UploadSSTableMetrics;
import org.apache.cassandra.sidecar.snapshots.SnapshotUtils;
import org.assertj.core.data.Percentage;
@@ -60,6 +62,7 @@ import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
import static org.apache.cassandra.sidecar.utils.TestFileUtils.prepareTestFile;
+import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.when;
@@ -415,6 +418,16 @@ class SSTableUploadHandlerTest extends BaseUploadsHandlerTest
"; actual: " + httpResponse.statusCode());
return;
}
+ UploadSSTableMetrics.UploadSSTableComponentMetrics componentMetrics
+ = new InstanceMetricsImpl(registry(1)).uploadSSTable().forComponent("db");
+ if (expectedRetCode == HttpResponseStatus.INSUFFICIENT_STORAGE.code())
+ {
+ assertThat(componentMetrics.diskUsageHigh.metric.getCount()).isOne();
+ }
+ if (expectedRetCode == HttpResponseStatus.TOO_MANY_REQUESTS.code())
+ {
+ assertThat(componentMetrics.rateLimitedCalls.metric.getCount()).isOne();
+ }
if (responseValidator != null)
{
diff --git a/src/test/java/org/apache/cassandra/sidecar/server/ServerSSLTest.java b/src/test/java/org/apache/cassandra/sidecar/server/ServerSSLTest.java
index 24536cd1..bacefd73 100644
--- a/src/test/java/org/apache/cassandra/sidecar/server/ServerSSLTest.java
+++ b/src/test/java/org/apache/cassandra/sidecar/server/ServerSSLTest.java
@@ -106,7 +106,6 @@ class ServerSSLTest
injector = Guice.createInjector(Modules.override(new MainModule())
.with(Modules.override(new TestModule())
.with(new ServerSSLTestModule(builder))));
- vertx = injector.getInstance(Vertx.class);
}
@AfterEach
@@ -124,6 +123,7 @@ class ServerSSLTest
void failsWhenKeyStoreIsNotConfigured()
{
builder.sslConfiguration(SslConfigurationImpl.builder().enabled(true).build());
+ vertx = vertx();
server = server();
assertThatRuntimeException()
@@ -143,6 +143,7 @@ class ServerSSLTest
.keystore(new KeyStoreConfigurationImpl(serverKeyStoreP12Path.toString(), "badpassword"))
.build();
builder.sslConfiguration(ssl);
+ vertx = vertx();
server = server();
assertThatRuntimeException()
@@ -162,6 +163,7 @@ class ServerSSLTest
.truststore(new KeyStoreConfigurationImpl(trustStoreP12Path.toString(), "badpassword"))
.build();
builder.sslConfiguration(ssl);
+ vertx = vertx();
server = server();
assertThatRuntimeException()
@@ -186,6 +188,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(0)
.build());
+ vertx = vertx();
server = server();
server.start()
@@ -208,6 +211,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(0)
.build());
+ vertx = vertx();
server = server();
server.start()
@@ -231,7 +235,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(9043)
.build());
-
+ vertx = vertx();
server = server();
server.start()
@@ -256,6 +260,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(0)
.build());
+ vertx = vertx();
server = server();
server.start()
@@ -280,6 +285,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(0)
.build());
+ vertx = vertx();
server = server();
server.start()
@@ -307,6 +313,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(0)
.build());
+ vertx = vertx();
server = server();
server.start()
@@ -329,6 +336,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(0)
.build());
+ vertx = vertx();
server = server();
server.start()
@@ -357,7 +365,7 @@ class ServerSSLTest
.host("127.0.0.1")
.port(9043)
.build());
-
+ vertx = vertx();
server = server();
// Remove default protocols enabled for the server and only add an unsupported protocol to the client
@@ -400,6 +408,7 @@ class ServerSSLTest
.serverVerticleInstances(serverVerticleInstances)
.port(9043)
.build());
+ vertx = vertx();
server = server();
WebClient client = clientWithP12Keystore(true, false);
@@ -460,6 +469,14 @@ class ServerSSLTest
return injector.getInstance(Server.class);
}
+ /**
+ * @return vertx, once {@link SidecarConfiguration} is built
+ */
+ Vertx vertx()
+ {
+ return injector.getInstance(Vertx.class);
+ }
+
Future<Void> validateHealthEndpoint(WebClient client)
{
LOGGER.info("Checking server health localhost:{}/api/v1/__health", server.actualPort());
diff --git a/src/test/java/org/apache/cassandra/sidecar/snapshots/SnapshotUtils.java b/src/test/java/org/apache/cassandra/sidecar/snapshots/SnapshotUtils.java
index 994ec01d..744daa2e 100644
--- a/src/test/java/org/apache/cassandra/sidecar/snapshots/SnapshotUtils.java
+++ b/src/test/java/org/apache/cassandra/sidecar/snapshots/SnapshotUtils.java
@@ -124,6 +124,7 @@ public class SnapshotUtils
.dataDirs(Collections.singletonList(rootPath + "/d1"))
.stagingDir(stagingDir)
.delegate(delegate)
+ .globalMetricRegistryName("cassandra_sidecar")
.build();
InstanceMetadataImpl localhost2 = InstanceMetadataImpl.builder()
.id(2)
@@ -132,6 +133,7 @@ public class SnapshotUtils
.dataDirs(Collections.singletonList(rootPath + "/d2"))
.stagingDir(stagingDir)
.delegate(delegate)
+ .globalMetricRegistryName("cassandra_sidecar")
.build();
List<InstanceMetadata> instanceMetas = Arrays.asList(localhost, localhost2);
return new InstancesConfigImpl(instanceMetas, DnsResolver.DEFAULT);
diff --git a/src/test/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTaskTest.java b/src/test/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTaskTest.java
index c4695caa..88db8830 100644
--- a/src/test/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTaskTest.java
+++ b/src/test/java/org/apache/cassandra/sidecar/tasks/HealthCheckPeriodicTaskTest.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -39,8 +40,10 @@ import org.apache.cassandra.sidecar.concurrent.ExecutorPools;
import org.apache.cassandra.sidecar.config.HealthCheckConfiguration;
import org.apache.cassandra.sidecar.config.SidecarConfiguration;
import org.apache.cassandra.sidecar.config.yaml.ServiceConfigurationImpl;
+import org.apache.cassandra.sidecar.metrics.ServerMetrics;
import org.mockito.stubbing.Answer;
+import static org.apache.cassandra.sidecar.utils.TestMetricUtils.registry;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doThrow;
@@ -57,6 +60,7 @@ class HealthCheckPeriodicTaskTest
HealthCheckConfiguration mockHealthCheckConfiguration;
HealthCheckPeriodicTask healthCheck;
InstancesConfig mockInstancesConfig;
+ ServerMetrics serverMetrics;
@BeforeEach
void setup()
@@ -70,8 +74,16 @@ class HealthCheckPeriodicTaskTest
mockInstancesConfig = mock(InstancesConfig.class);
Vertx vertx = Vertx.vertx();
+ serverMetrics = new ServerMetrics(registry());
ExecutorPools executorPools = new ExecutorPools(vertx, new ServiceConfigurationImpl());
- healthCheck = new HealthCheckPeriodicTask(vertx, mockConfiguration, mockInstancesConfig, executorPools);
+ healthCheck = new HealthCheckPeriodicTask(vertx, mockConfiguration, mockInstancesConfig,
+ executorPools, serverMetrics);
+ }
+
+ @AfterEach
+ void cleanup()
+ {
+ registry().removeMatching((name, metric) -> true);
}
@Test
@@ -85,46 +97,66 @@ class HealthCheckPeriodicTaskTest
@Test
void testHealthCheckPromiseCompletesWhenNoInstancesAreConfigured(VertxTestContext context)
{
+ int expectedUpInstances = 0;
+ int expectedDownInstances = 0;
List<InstanceMetadata> mockInstanceMetadata = Collections.emptyList();
when(mockInstancesConfig.instances()).thenReturn(mockInstanceMetadata);
Promise<Void> promise = Promise.promise();
healthCheck.execute(promise);
- promise.future().onComplete(context.succeedingThenComplete());
+ promise.future().onComplete(context.succeeding(v -> {
+ assertThat(serverMetrics.cassandraInstancesUp.metric.getValue()).isEqualTo(expectedUpInstances);
+ assertThat(serverMetrics.cassandraInstancesDown.metric.getValue()).isEqualTo(expectedDownInstances);
+ context.completeNow();
+ }));
}
@Test
void testHealthCheckInvokedForAllInstances(VertxTestContext context)
{
int numberOfInstances = 5;
+ int expectedUpInstances = 5;
+ int expectedDownInstances = 0;
Checkpoint healthCheckCheckPoint = context.checkpoint(numberOfInstances);
List<InstanceMetadata> mockInstanceMetadata =
buildMockInstanceMetadata(healthCheckCheckPoint, numberOfInstances);
when(mockInstancesConfig.instances()).thenReturn(mockInstanceMetadata);
Promise<Void> promise = Promise.promise();
healthCheck.execute(promise);
- promise.future().onComplete(context.succeedingThenComplete());
+ promise.future().onComplete(context.succeeding(v -> {
+ assertThat(serverMetrics.cassandraInstancesUp.metric.getValue()).isEqualTo(expectedUpInstances);
+ assertThat(serverMetrics.cassandraInstancesDown.metric.getValue()).isEqualTo(expectedDownInstances);
+ context.completeNow();
+ }));
}
@Test
void testInstanceMetadataExceptionDoesntPreventChecksOnOtherInstances(VertxTestContext context)
{
int numberOfInstances = 5;
+ int expectedUpInstances = 4;
+ int expectedDownInstances = 1;
Checkpoint healthCheckCheckPoint = context.checkpoint(numberOfInstances);
List<InstanceMetadata> mockInstanceMetadata =
buildMockInstanceMetadata(healthCheckCheckPoint, numberOfInstances);
InstanceMetadata mockInstance = mock(InstanceMetadata.class);
when(mockInstance.delegate()).thenThrow(new RuntimeException());
- mockInstanceMetadata.add(3, mockInstance);
+ mockInstanceMetadata.set(3, mockInstance);
when(mockInstancesConfig.instances()).thenReturn(mockInstanceMetadata);
Promise<Void> promise = Promise.promise();
healthCheck.execute(promise);
- promise.future().onComplete(context.failingThenComplete());
+ promise.future().onComplete(context.failing(v -> {
+ assertThat(serverMetrics.cassandraInstancesUp.metric.getValue()).isEqualTo(expectedUpInstances);
+ assertThat(serverMetrics.cassandraInstancesDown.metric.getValue()).isEqualTo(expectedDownInstances);
+ context.completeNow();
+ }));
}
@Test
void testDelegateExceptionDoesntPreventChecksOnOtherInstances(VertxTestContext context)
{
int numberOfInstances = 5;
+ int expectedUpInstances = 4;
+ int expectedDownInstances = 1;
Checkpoint healthCheckCheckPoint = context.checkpoint(numberOfInstances);
List<InstanceMetadata> mockInstanceMetadata =
buildMockInstanceMetadata(healthCheckCheckPoint, numberOfInstances);
@@ -132,11 +164,15 @@ class HealthCheckPeriodicTaskTest
CassandraAdapterDelegate mockDelegate = mock(CassandraAdapterDelegate.class);
when(mockInstance.delegate()).thenReturn(mockDelegate);
doThrow(new RuntimeException()).when(mockDelegate).healthCheck();
- mockInstanceMetadata.add(3, mockInstance);
+ mockInstanceMetadata.set(3, mockInstance);
when(mockInstancesConfig.instances()).thenReturn(mockInstanceMetadata);
Promise<Void> promise = Promise.promise();
healthCheck.execute(promise);
- promise.future().onComplete(context.succeedingThenComplete());
+ promise.future().onComplete(context.failing(v -> {
+ assertThat(serverMetrics.cassandraInstancesUp.metric.getValue()).isEqualTo(expectedUpInstances);
+ assertThat(serverMetrics.cassandraInstancesDown.metric.getValue()).isEqualTo(expectedDownInstances);
+ context.completeNow();
+ }));
}
private List<InstanceMetadata> buildMockInstanceMetadata(Checkpoint healthCheckCheckPoint, int numberOfInstances)
diff --git a/src/test/java/org/apache/cassandra/sidecar/utils/TestMetricUtils.java b/src/test/java/org/apache/cassandra/sidecar/utils/TestMetricUtils.java
new file mode 100644
index 00000000..6a8436d7
--- /dev/null
+++ b/src/test/java/org/apache/cassandra/sidecar/utils/TestMetricUtils.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.sidecar.utils;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.SharedMetricRegistries;
+
+/**
+ * Utility methods to retrieve {@link MetricRegistry} during testing
+ */
+public class TestMetricUtils
+{
+ public static MetricRegistry registry()
+ {
+ return SharedMetricRegistries.getOrCreate("cassandra_sidecar");
+ }
+
+ public static MetricRegistry registry(int instanceId)
+ {
+ String registryName = "cassandra_sidecar_" + instanceId;
+ return SharedMetricRegistries.getOrCreate(registryName);
+ }
+}
diff --git a/src/test/resources/config/sidecar_metrics.yaml b/src/test/resources/config/sidecar_metrics.yaml
new file mode 100644
index 00000000..689f157e
--- /dev/null
+++ b/src/test/resources/config/sidecar_metrics.yaml
@@ -0,0 +1,69 @@
+#
+# Cassandra SideCar configuration file
+#
+cassandra:
+ host: localhost
+ port: 9042
+ data_dirs: /cassandra/d1/data, /cassandra/d2/data
+ jmx_host: 127.0.0.1
+ jmx_port: 7199
+ jmx_role: controlRole
+ jmx_role_password: controlPassword
+ jmx_ssl_enabled: true
+
+cassandra_instances:
+ - id: 1
+ host: localhost1
+ port: 9042
+ data_dirs: /cassandra/d1/data, /cassandra/d2/data
+ jmx_host: 127.0.0.1
+ jmx_port: 7100
+ jmx_role: controlRole
+ jmx_role_password: controlPassword
+ jmx_ssl_enabled: true
+ - id: 2
+ host: localhost2
+ port: 9042
+ data_dirs: /cassandra/d3/data, /cassandra/d4/data
+ jmx_host: 127.0.0.1
+ jmx_port: 7200
+ jmx_role: controlRole
+ jmx_role_password: controlPassword
+ jmx_ssl_enabled: true
+
+sidecar:
+ host: 0.0.0.0
+ port: 9043
+ throttle:
+ stream_requests_per_sec: 5000
+ delay_sec: 5
+ timeout_sec: 10
+ allowable_time_skew_in_minutes: 60
+ jmx:
+ connection:
+ max_retries: 40
+ retry_delay_millis: 2000
+#
+# Enable SSL configuration (Disabled by default)
+#
+# - ssl:
+# - enabled: true
+# - keystore:
+# - path: "path/to/keystore.p12"
+# - password: password
+# - truststore:
+# - path: "path/to/truststore.p12"
+# - password: password
+
+metrics:
+ registry_name: cassandra_sidecar
+ vertx:
+ enabled: true
+ expose_via_jmx: false
+ jmx_domain_name: sidecar.vertx.jmx_domain
+ monitored_server_route_regexes: # regex list to match server routes
+ - /api/v1/keyspaces/.*
+ - /api/v1/cassandra/.*
+
+healthcheck:
+ poll_freq_millis: 30000
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org