You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2022/08/17 21:59:50 UTC
[flink] branch master updated (3c2fa303074 -> 3e6f47c9d36)
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 3c2fa303074 [FLINK-28488][kafka] Only forward measurable Kafka metrics and ignore others
new 56a7c847df1 [FLINK-28996][datadog] Move parameter parsing into factory
new 7e7fd76078e [FLINK-28997][datadog] Add switch to use logical identifier
new 62e53e0e672 [hotfix][docs] Fix typo
new 3e6f47c9d36 [FLINK-29002][datadog] Deprecate 'tags' option
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../content.zh/docs/deployment/metric_reporters.md | 10 ++--
docs/content/docs/deployment/metric_reporters.md | 6 +-
.../flink/metrics/datadog/DatadogHttpReporter.java | 66 ++++++++++++----------
.../datadog/DatadogHttpReporterFactory.java | 39 ++++++++++++-
4 files changed, 82 insertions(+), 39 deletions(-)
[flink] 01/04: [FLINK-28996][datadog] Move parameter parsing into factory
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 56a7c847df16b70f321d5445e447ca4dc0d8bba4
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Tue Aug 16 11:14:57 2022 +0200
[FLINK-28996][datadog] Move parameter parsing into factory
---
.../flink/metrics/datadog/DatadogHttpReporter.java | 53 ++++++++++------------
.../datadog/DatadogHttpReporterFactory.java | 26 ++++++++++-
2 files changed, 47 insertions(+), 32 deletions(-)
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
index 0b138d85648..9c616725131 100644
--- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
@@ -53,18 +53,31 @@ public class DatadogHttpReporter implements MetricReporter, Scheduled {
private final Map<Meter, DMeter> meters = new ConcurrentHashMap<>();
private final Map<Histogram, DHistogram> histograms = new ConcurrentHashMap<>();
- private DatadogHttpClient client;
- private List<String> configTags;
- private int maxMetricsPerRequestValue;
+ private final DatadogHttpClient client;
+ private final List<String> configTags;
+ private final int maxMetricsPerRequestValue;
private final Clock clock = () -> System.currentTimeMillis() / 1000L;
- public static final String API_KEY = "apikey";
- public static final String PROXY_HOST = "proxyHost";
- public static final String PROXY_PORT = "proxyPort";
- public static final String DATA_CENTER = "dataCenter";
- public static final String TAGS = "tags";
- public static final String MAX_METRICS_PER_REQUEST = "maxMetricsPerRequest";
+ public DatadogHttpReporter(
+ String apiKey,
+ String proxyHost,
+ int proxyPort,
+ int maxMetricsPerRequestValue,
+ DataCenter dataCenter,
+ String tags) {
+ this.maxMetricsPerRequestValue = maxMetricsPerRequestValue;
+ this.client = new DatadogHttpClient(apiKey, proxyHost, proxyPort, dataCenter, true);
+ this.configTags = getTagsFromConfig(tags);
+
+ LOGGER.info(
+ "Configured DatadogHttpReporter with {tags={}, proxyHost={}, proxyPort={}, dataCenter={}, maxMetricsPerRequest={}",
+ tags,
+ proxyHost,
+ proxyPort,
+ dataCenter,
+ maxMetricsPerRequestValue);
+ }
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
@@ -124,27 +137,7 @@ public class DatadogHttpReporter implements MetricReporter, Scheduled {
}
@Override
- public void open(MetricConfig config) {
- String apiKey = config.getString(API_KEY, null);
- String proxyHost = config.getString(PROXY_HOST, null);
- Integer proxyPort = config.getInteger(PROXY_PORT, 8080);
- String rawDataCenter = config.getString(DATA_CENTER, "US");
- maxMetricsPerRequestValue = config.getInteger(MAX_METRICS_PER_REQUEST, 2000);
- DataCenter dataCenter = DataCenter.valueOf(rawDataCenter);
- String tags = config.getString(TAGS, "");
-
- client = new DatadogHttpClient(apiKey, proxyHost, proxyPort, dataCenter, true);
-
- configTags = getTagsFromConfig(tags);
-
- LOGGER.info(
- "Configured DatadogHttpReporter with {tags={}, proxyHost={}, proxyPort={}, dataCenter={}, maxMetricsPerRequest={}",
- tags,
- proxyHost,
- proxyPort,
- dataCenter,
- maxMetricsPerRequestValue);
- }
+ public void open(MetricConfig config) {}
@Override
public void close() {
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
index e291e064f82..0f568bdfae3 100644
--- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
@@ -26,8 +26,30 @@ import java.util.Properties;
/** {@link MetricReporterFactory} for {@link DatadogHttpReporter}. */
public class DatadogHttpReporterFactory implements MetricReporterFactory {
+ private static final String API_KEY = "apikey";
+ private static final String PROXY_HOST = "proxyHost";
+ private static final String PROXY_PORT = "proxyPort";
+ private static final String DATA_CENTER = "dataCenter";
+ private static final String TAGS = "tags";
+ private static final String MAX_METRICS_PER_REQUEST = "maxMetricsPerRequest";
+
@Override
- public MetricReporter createMetricReporter(Properties properties) {
- return new DatadogHttpReporter();
+ public MetricReporter createMetricReporter(Properties config) {
+ final String apiKey = config.getProperty(API_KEY, null);
+ final String proxyHost = config.getProperty(PROXY_HOST, null);
+ final int proxyPort = Integer.valueOf(config.getProperty(PROXY_PORT, "8080"));
+ final String rawDataCenter = config.getProperty(DATA_CENTER, "US");
+ final int maxMetricsPerRequestValue =
+ Integer.valueOf(config.getProperty(MAX_METRICS_PER_REQUEST, "2000"));
+ final DataCenter dataCenter = DataCenter.valueOf(rawDataCenter);
+ final String tags = config.getProperty(TAGS, "");
+
+ return new DatadogHttpReporter(
+ apiKey,
+ proxyHost,
+ proxyPort,
+ maxMetricsPerRequestValue,
+ dataCenter,
+ tags);
}
}
[flink] 03/04: [hotfix][docs] Fix typo
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 62e53e0e672d8283d201d5c48e0cd5406eb86c92
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Tue Aug 16 21:36:47 2022 +0200
[hotfix][docs] Fix typo
---
docs/content.zh/docs/deployment/metric_reporters.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/content.zh/docs/deployment/metric_reporters.md b/docs/content.zh/docs/deployment/metric_reporters.md
index 01a88ebd423..5ab5b1c39da 100644
--- a/docs/content.zh/docs/deployment/metric_reporters.md
+++ b/docs/content.zh/docs/deployment/metric_reporters.md
@@ -263,8 +263,8 @@ metrics.reporter.stsd.interval: 60 SECONDS
<span class="label label-danger">Note</span> For legacy reasons the reporter uses _both_ the metric identifier _and_ tags. This redundancy can be avoided by enabling `useLogicalIdentifier`.
-<span class="label label-info">注意</span> 按照 Datedog 的 Histograms 命名约定，Histograms 类的运行指标会作为一系列 gauges 显示（`<metric_name>.<aggregation>`）。
-默认情况下 `min` 即最小值被发送到 Datedog，`sum` 不会被发送。
+<span class="label label-info">注意</span> 按照 Datadog 的 Histograms 命名约定，Histograms 类的运行指标会作为一系列 gauges 显示（`<metric_name>.<aggregation>`）。
+默认情况下 `min` 即最小值被发送到 Datadog，`sum` 不会被发送。
与 Datadog 提供的 Histograms 相比，Histograms 类的运行指标不会按照指定的发送间隔进行聚合计算。
参数:
[flink] 02/04: [FLINK-28997][datadog] Add switch to use logical identifier
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7e7fd76078e6499a50edb2c69fff5088326390b6
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Tue Aug 16 12:06:21 2022 +0200
[FLINK-28997][datadog] Add switch to use logical identifier
---
docs/content.zh/docs/deployment/metric_reporters.md | 4 ++++
docs/content/docs/deployment/metric_reporters.md | 4 ++++
.../apache/flink/metrics/datadog/DatadogHttpReporter.java | 15 +++++++++++++--
.../flink/metrics/datadog/DatadogHttpReporterFactory.java | 6 +++++-
4 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/docs/content.zh/docs/deployment/metric_reporters.md b/docs/content.zh/docs/deployment/metric_reporters.md
index 36eba808a7f..01a88ebd423 100644
--- a/docs/content.zh/docs/deployment/metric_reporters.md
+++ b/docs/content.zh/docs/deployment/metric_reporters.md
@@ -261,6 +261,8 @@ metrics.reporter.stsd.interval: 60 SECONDS
使用 Datadog 时，Flink 运行指标中的任何变量，例如 `<host>`、`<job_name>`、 `<tm_id>`、 `<subtask_index>`、`<task_name>`、 `<operator_name>`，都会被当作 `host:localhost`、`job_name:myjobname` 这样的 tag 发送。
+<span class="label label-danger">Note</span> For legacy reasons the reporter uses _both_ the metric identifier _and_ tags. This redundancy can be avoided by enabling `useLogicalIdentifier`.
+
<span class="label label-info">注意</span> 按照 Datedog 的 Histograms 命名约定，Histograms 类的运行指标会作为一系列 gauges 显示（`<metric_name>.<aggregation>`）。
默认情况下 `min` 即最小值被发送到 Datedog，`sum` 不会被发送。
与 Datadog 提供的 Histograms 相比，Histograms 类的运行指标不会按照指定的发送间隔进行聚合计算。
@@ -273,6 +275,7 @@ metrics.reporter.stsd.interval: 60 SECONDS
- `proxyPort` - （可选的）发送到 Datadog 时使用的代理端口，默认为 8080。
- `dataCenter` - （可选的）要连接的数据中心（`EU`/`US`），默认为 `US`。
- `maxMetricsPerRequest` - （可选的）每次请求携带的最大运行指标个数，默认为 2000。
+- `useLogicalIdentifier` -> (optional) Whether the reporter uses a logical metric identifier, defaults to `false`.
配置示例:
@@ -285,6 +288,7 @@ metrics.reporter.dghttp.proxyPort: 8080
metrics.reporter.dghttp.dataCenter: US
metrics.reporter.dghttp.maxMetricsPerRequest: 2000
metrics.reporter.dghttp.interval: 60 SECONDS
+metrics.reporter.dghttp.useLogicalIdentifier: true
```
<a name="slf4j"></a>
diff --git a/docs/content/docs/deployment/metric_reporters.md b/docs/content/docs/deployment/metric_reporters.md
index 32b4ae16de1..f980745a138 100644
--- a/docs/content/docs/deployment/metric_reporters.md
+++ b/docs/content/docs/deployment/metric_reporters.md
@@ -245,6 +245,8 @@ Type: push/tags
Note any variables in Flink metrics, such as `<host>`, `<job_name>`, `<tm_id>`, `<subtask_index>`, `<task_name>`, and `<operator_name>`,
will be sent to Datadog as tags. Tags will look like `host:localhost` and `job_name:myjobname`.
+<span class="label label-danger">Note</span> For legacy reasons the reporter uses _both_ the metric identifier _and_ tags. This redundancy can be avoided by enabling `useLogicalIdentifier`.
+
<span class="label label-info">Note</span> Histograms are exposed as a series of gauges following the naming convention of Datadog histograms (`<metric_name>.<aggregation>`).
The `min` aggregation is reported by default, whereas `sum` is not available.
In contrast to Datadog-provided Histograms the reported aggregations are not computed for a specific reporting interval.
@@ -257,6 +259,7 @@ Parameters:
- `proxyPort` - (optional) The proxy port to use when sending to Datadog, defaults to 8080.
- `dataCenter` - (optional) The data center (`EU`/`US`) to connect to, defaults to `US`.
- `maxMetricsPerRequest` - (optional) The maximum number of metrics to include in each request, defaults to 2000.
+- `useLogicalIdentifier` -> (optional) Whether the reporter uses a logical metric identifier, defaults to `false`.
Example configuration:
@@ -269,6 +272,7 @@ metrics.reporter.dghttp.proxyPort: 8080
metrics.reporter.dghttp.dataCenter: US
metrics.reporter.dghttp.maxMetricsPerRequest: 2000
metrics.reporter.dghttp.interval: 60 SECONDS
+metrics.reporter.dghttp.useLogicalIdentifier: true
```
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
index 9c616725131..5778728db2e 100644
--- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporter.java
@@ -18,9 +18,11 @@
package org.apache.flink.metrics.datadog;
+import org.apache.flink.metrics.CharacterFilter;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.LogicalScopeProvider;
import org.apache.flink.metrics.Meter;
import org.apache.flink.metrics.Metric;
import org.apache.flink.metrics.MetricConfig;
@@ -56,6 +58,7 @@ public class DatadogHttpReporter implements MetricReporter, Scheduled {
private final DatadogHttpClient client;
private final List<String> configTags;
private final int maxMetricsPerRequestValue;
+ private final boolean useLogicalIdentifier;
private final Clock clock = () -> System.currentTimeMillis() / 1000L;
@@ -65,8 +68,10 @@ public class DatadogHttpReporter implements MetricReporter, Scheduled {
int proxyPort,
int maxMetricsPerRequestValue,
DataCenter dataCenter,
- String tags) {
+ String tags,
+ boolean useLogicalIdentifier) {
this.maxMetricsPerRequestValue = maxMetricsPerRequestValue;
+ this.useLogicalIdentifier = useLogicalIdentifier;
this.client = new DatadogHttpClient(apiKey, proxyHost, proxyPort, dataCenter, true);
this.configTags = getTagsFromConfig(tags);
@@ -81,7 +86,13 @@ public class DatadogHttpReporter implements MetricReporter, Scheduled {
@Override
public void notifyOfAddedMetric(Metric metric, String metricName, MetricGroup group) {
- final String name = group.getMetricIdentifier(metricName);
+ final String name =
+ this.useLogicalIdentifier
+ ? ((LogicalScopeProvider) group)
+ .getLogicalScope(CharacterFilter.NO_OP_FILTER)
+ + "."
+ + metricName
+ : group.getMetricIdentifier(metricName);
List<String> tags = new ArrayList<>(configTags);
tags.addAll(getTagsFromMetricGroup(group));
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
index 0f568bdfae3..c9ba359f52b 100644
--- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
@@ -32,6 +32,7 @@ public class DatadogHttpReporterFactory implements MetricReporterFactory {
private static final String DATA_CENTER = "dataCenter";
private static final String TAGS = "tags";
private static final String MAX_METRICS_PER_REQUEST = "maxMetricsPerRequest";
+ private static final String USE_LOGICAL_IDENTIFIER = "useLogicalIdentifier";
@Override
public MetricReporter createMetricReporter(Properties config) {
@@ -43,6 +44,8 @@ public class DatadogHttpReporterFactory implements MetricReporterFactory {
Integer.valueOf(config.getProperty(MAX_METRICS_PER_REQUEST, "2000"));
final DataCenter dataCenter = DataCenter.valueOf(rawDataCenter);
final String tags = config.getProperty(TAGS, "");
+ final boolean useLogicalIdentifier =
+ Boolean.parseBoolean(config.getProperty(USE_LOGICAL_IDENTIFIER, "false"));
return new DatadogHttpReporter(
apiKey,
@@ -50,6 +53,7 @@ public class DatadogHttpReporterFactory implements MetricReporterFactory {
proxyPort,
maxMetricsPerRequestValue,
dataCenter,
- tags);
+ tags,
+ useLogicalIdentifier);
}
}
[flink] 04/04: [FLINK-29002][datadog] Deprecate 'tags' option
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3e6f47c9d3623c240bbb86845893f9a022dde03d
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Tue Aug 16 21:42:15 2022 +0200
[FLINK-29002][datadog] Deprecate 'tags' option
---
docs/content.zh/docs/deployment/metric_reporters.md | 2 --
docs/content/docs/deployment/metric_reporters.md | 2 --
.../apache/flink/metrics/datadog/DatadogHttpReporterFactory.java | 9 +++++++++
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/docs/content.zh/docs/deployment/metric_reporters.md b/docs/content.zh/docs/deployment/metric_reporters.md
index 5ab5b1c39da..2c4cdb9b033 100644
--- a/docs/content.zh/docs/deployment/metric_reporters.md
+++ b/docs/content.zh/docs/deployment/metric_reporters.md
@@ -270,7 +270,6 @@ metrics.reporter.stsd.interval: 60 SECONDS
参数:
- `apikey` - Datadog 的 API KEY。
-- `tags` - （可选的）发送到 Datadog 时将会转换为指标的全局 tag。tag 间只能以逗号分隔。
- `proxyHost` - （可选的）发送到 Datadog 时使用的代理主机。
- `proxyPort` - （可选的）发送到 Datadog 时使用的代理端口，默认为 8080。
- `dataCenter` - （可选的）要连接的数据中心（`EU`/`US`），默认为 `US`。
@@ -282,7 +281,6 @@ metrics.reporter.stsd.interval: 60 SECONDS
```yaml
metrics.reporter.dghttp.factory.class: org.apache.flink.metrics.datadog.DatadogHttpReporterFactory
metrics.reporter.dghttp.apikey: xxx
-metrics.reporter.dghttp.tags: myflinkapp,prod
metrics.reporter.dghttp.proxyHost: my.web.proxy.com
metrics.reporter.dghttp.proxyPort: 8080
metrics.reporter.dghttp.dataCenter: US
diff --git a/docs/content/docs/deployment/metric_reporters.md b/docs/content/docs/deployment/metric_reporters.md
index f980745a138..e8b3ccb2e20 100644
--- a/docs/content/docs/deployment/metric_reporters.md
+++ b/docs/content/docs/deployment/metric_reporters.md
@@ -254,7 +254,6 @@ In contrast to Datadog-provided Histograms the reported aggregations are not com
Parameters:
- `apikey` - the Datadog API key
-- `tags` - (optional) the global tags that will be applied to metrics when sending to Datadog. Tags should be separated by comma only
- `proxyHost` - (optional) The proxy host to use when sending to Datadog.
- `proxyPort` - (optional) The proxy port to use when sending to Datadog, defaults to 8080.
- `dataCenter` - (optional) The data center (`EU`/`US`) to connect to, defaults to `US`.
@@ -266,7 +265,6 @@ Example configuration:
```yaml
metrics.reporter.dghttp.factory.class: org.apache.flink.metrics.datadog.DatadogHttpReporterFactory
metrics.reporter.dghttp.apikey: xxx
-metrics.reporter.dghttp.tags: myflinkapp,prod
metrics.reporter.dghttp.proxyHost: my.web.proxy.com
metrics.reporter.dghttp.proxyPort: 8080
metrics.reporter.dghttp.dataCenter: US
diff --git a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
index c9ba359f52b..ebee3b545c7 100644
--- a/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
+++ b/flink-metrics/flink-metrics-datadog/src/main/java/org/apache/flink/metrics/datadog/DatadogHttpReporterFactory.java
@@ -21,11 +21,16 @@ package org.apache.flink.metrics.datadog;
import org.apache.flink.metrics.reporter.MetricReporter;
import org.apache.flink.metrics.reporter.MetricReporterFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.Properties;
/** {@link MetricReporterFactory} for {@link DatadogHttpReporter}. */
public class DatadogHttpReporterFactory implements MetricReporterFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(DatadogHttpReporterFactory.class);
+
private static final String API_KEY = "apikey";
private static final String PROXY_HOST = "proxyHost";
private static final String PROXY_PORT = "proxyPort";
@@ -43,6 +48,10 @@ public class DatadogHttpReporterFactory implements MetricReporterFactory {
final int maxMetricsPerRequestValue =
Integer.valueOf(config.getProperty(MAX_METRICS_PER_REQUEST, "2000"));
final DataCenter dataCenter = DataCenter.valueOf(rawDataCenter);
+ if (config.containsKey(TAGS)) {
+ LOG.warn(
+ "The 'tags' option is deprecated; please use 'scope.variables.additional' instead.");
+ }
final String tags = config.getProperty(TAGS, "");
final boolean useLogicalIdentifier =
Boolean.parseBoolean(config.getProperty(USE_LOGICAL_IDENTIFIER, "false"));