You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2016/05/22 18:58:12 UTC
[1/6] flink git commit: [FLINK-1502] [core] Various improvements to
the metrics infrastructure code
Repository: flink
Updated Branches:
refs/heads/master 8ed368582 -> 707606ac4
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java
index 7b3286d..6b9a5fc 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java
@@ -17,15 +17,112 @@
*/
package org.apache.flink.metrics.groups;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.util.AbstractID;
import org.junit.Test;
import java.util.List;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
public class TaskManagerGroupTest {
+
+ // ------------------------------------------------------------------------
+ // adding and removing jobs
+ // ------------------------------------------------------------------------
+
+ @Test
+ public void addAndRemoveJobs() {
+ final TaskManagerMetricGroup group = new TaskManagerMetricGroup(
+ new MetricRegistry(new Configuration()), "localhost", new AbstractID().toString());
+
+
+ final JobID jid1 = new JobID();
+ final JobID jid2 = new JobID();
+
+ final String jobName1 = "testjob";
+ final String jobName2 = "anotherJob";
+
+ final AbstractID vertex11 = new AbstractID();
+ final AbstractID vertex12 = new AbstractID();
+ final AbstractID vertex13 = new AbstractID();
+ final AbstractID vertex21 = new AbstractID();
+
+ final AbstractID execution11 = new AbstractID();
+ final AbstractID execution12 = new AbstractID();
+ final AbstractID execution13 = new AbstractID();
+ final AbstractID execution21 = new AbstractID();
+
+ TaskMetricGroup tmGroup11 = group.addTaskForJob(jid1, jobName1, vertex11, execution11, 17, "test");
+ TaskMetricGroup tmGroup12 = group.addTaskForJob(jid1, jobName1, vertex12, execution12, 13, "test");
+ TaskMetricGroup tmGroup21 = group.addTaskForJob(jid2, jobName2, vertex21, execution21, 7, "test");
+
+ assertEquals(2, group.numRegisteredJobMetricGroups());
+ assertFalse(tmGroup11.parent().isClosed());
+ assertFalse(tmGroup12.parent().isClosed());
+ assertFalse(tmGroup21.parent().isClosed());
+
+ // close all for job 2 and one from job 1
+ tmGroup11.close();
+ tmGroup21.close();
+ assertTrue(tmGroup11.isClosed());
+ assertTrue(tmGroup21.isClosed());
+
+ // job 2 should be removed, job should still be there
+ assertFalse(tmGroup11.parent().isClosed());
+ assertFalse(tmGroup12.parent().isClosed());
+ assertTrue(tmGroup21.parent().isClosed());
+ assertEquals(1, group.numRegisteredJobMetricGroups());
+
+ // add one more to job one
+ TaskMetricGroup tmGroup13 = group.addTaskForJob(jid1, jobName1, vertex13, execution13, 0, "test");
+ tmGroup12.close();
+ tmGroup13.close();
+
+ assertTrue(tmGroup11.parent().isClosed());
+ assertTrue(tmGroup12.parent().isClosed());
+ assertTrue(tmGroup13.parent().isClosed());
+
+ assertEquals(0, group.numRegisteredJobMetricGroups());
+ }
+
+ @Test
+ public void testCloseClosesAll() {
+ final TaskManagerMetricGroup group = new TaskManagerMetricGroup(
+ new MetricRegistry(new Configuration()), "localhost", new AbstractID().toString());
+
+
+ final JobID jid1 = new JobID();
+ final JobID jid2 = new JobID();
+
+ final String jobName1 = "testjob";
+ final String jobName2 = "anotherJob";
+
+ final AbstractID vertex11 = new AbstractID();
+ final AbstractID vertex12 = new AbstractID();
+ final AbstractID vertex21 = new AbstractID();
+
+ final AbstractID execution11 = new AbstractID();
+ final AbstractID execution12 = new AbstractID();
+ final AbstractID execution21 = new AbstractID();
+
+ TaskMetricGroup tmGroup11 = group.addTaskForJob(jid1, jobName1, vertex11, execution11, 17, "test");
+ TaskMetricGroup tmGroup12 = group.addTaskForJob(jid1, jobName1, vertex12, execution12, 13, "test");
+ TaskMetricGroup tmGroup21 = group.addTaskForJob(jid2, jobName2, vertex21, execution21, 7, "test");
+
+ group.close();
+
+ assertTrue(tmGroup11.isClosed());
+ assertTrue(tmGroup12.isClosed());
+ assertTrue(tmGroup21.isClosed());
+ }
+
+ // ------------------------------------------------------------------------
+ // scope tests
+ // ------------------------------------------------------------------------
+
@Test
public void testGenerateScopeDefault() {
MetricRegistry registry = new MetricRegistry(new Configuration());
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java
index d607072..ab78288 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java
@@ -24,6 +24,7 @@ import org.apache.flink.metrics.groups.JobMetricGroup;
import org.apache.flink.util.AbstractID;
public class DummyJobMetricGroup extends JobMetricGroup {
+
public DummyJobMetricGroup() {
super(new DummyMetricRegistry(), new DummyTaskManagerMetricGroup(), new JobID(), "job");
}
@@ -34,14 +35,10 @@ public class DummyJobMetricGroup extends JobMetricGroup {
}
@Override
- protected MetricGroup addMetric(String name, Metric metric) {
- return this;
- }
+ public void removeTaskMetricGroup(AbstractID executionId) {}
@Override
- public MetricGroup addGroup(int name) {
- return addGroup("" + name);
- }
+ protected void addMetric(String name, Metric metric) {}
@Override
public MetricGroup addGroup(String name) {
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java
index 26df874..77ddd17 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java
@@ -19,6 +19,7 @@ package org.apache.flink.metrics.util;
import org.apache.flink.metrics.Metric;
import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
import org.apache.flink.metrics.groups.AbstractMetricGroup;
import org.apache.flink.metrics.groups.Scope;
@@ -26,8 +27,13 @@ import java.util.ArrayList;
import java.util.List;
public class DummyMetricGroup extends AbstractMetricGroup {
+
public DummyMetricGroup() {
- super(new DummyMetricRegistry());
+ this(new DummyMetricRegistry());
+ }
+
+ public DummyMetricGroup(MetricRegistry registry) {
+ super(registry);
}
@Override
@@ -41,14 +47,8 @@ public class DummyMetricGroup extends AbstractMetricGroup {
}
@Override
- protected MetricGroup addMetric(String name, Metric metric) {
- return this;
- }
-
- @Override
- public MetricGroup addGroup(int name) {
- return addGroup("" + name);
- }
+ protected void addMetric(String name, Metric metric) {}
+
@Override
public MetricGroup addGroup(String name) {
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java
index f0d6d3f..f8b73a9 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java
@@ -15,18 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.util;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.MetricRegistry;
public class DummyMetricRegistry extends MetricRegistry {
- private static final Configuration config;
-
- static {
- config = new Configuration();
- config.setString(KEY_METRICS_REPORTER_CLASS, DummyReporter.class.getCanonicalName());
- }
public DummyMetricRegistry() {
super(new Configuration());
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java
index eb45f6a..e271d6e 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java
@@ -22,19 +22,13 @@ import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.OperatorMetricGroup;
public class DummyOperatorMetricGroup extends OperatorMetricGroup {
+
public DummyOperatorMetricGroup() {
super(new DummyMetricRegistry(), new DummyTaskMetricGroup(), "operator", 0);
}
@Override
- protected MetricGroup addMetric(String name, Metric metric) {
- return this;
- }
-
- @Override
- public MetricGroup addGroup(int name) {
- return addGroup("" + name);
- }
+ protected void addMetric(String name, Metric metric) {}
@Override
public MetricGroup addGroup(String name) {
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java
index 1c7d33b..8b7714f 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java
@@ -23,6 +23,7 @@ import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.TaskManagerMetricGroup;
public class DummyTaskManagerMetricGroup extends TaskManagerMetricGroup {
+
public DummyTaskManagerMetricGroup() {
super(new DummyMetricRegistry(), "host", "id");
}
@@ -32,14 +33,7 @@ public class DummyTaskManagerMetricGroup extends TaskManagerMetricGroup {
}
@Override
- protected MetricGroup addMetric(String name, Metric metric) {
- return this;
- }
-
- @Override
- public MetricGroup addGroup(int name) {
- return addGroup("" + name);
- }
+ protected void addMetric(String name, Metric metric) {}
@Override
public MetricGroup addGroup(String name) {
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java
index 53683f4..db2c557 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java
@@ -23,6 +23,7 @@ import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.util.AbstractID;
public class DummyTaskMetricGroup extends TaskMetricGroup {
+
public DummyTaskMetricGroup() {
super(new DummyMetricRegistry(), new DummyJobMetricGroup(), new AbstractID(), new AbstractID(), 0, "task");
}
@@ -32,14 +33,7 @@ public class DummyTaskMetricGroup extends TaskMetricGroup {
}
@Override
- protected MetricGroup addMetric(String name, Metric metric) {
- return this;
- }
-
- @Override
- public MetricGroup addGroup(int name) {
- return addGroup("" + name);
- }
+ protected void addMetric(String name, Metric metric) {}
@Override
public MetricGroup addGroup(String name) {
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java b/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java
index 482d1e8..5d8a8e0 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java
@@ -15,23 +15,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.util;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.metrics.Metric;
import org.apache.flink.metrics.reporter.AbstractReporter;
-import org.apache.flink.metrics.reporter.MetricReporter;
import java.util.List;
public class TestReporter extends AbstractReporter {
+
@Override
- public void open(Configuration config) {
- }
+ public void open(Configuration config) {}
@Override
- public void close() {
- }
+ public void close() {}
@Override
public String generateName(String name, List<String> scope) {
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-dropwizard/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-dropwizard/pom.xml b/flink-metric-reporters/flink-metrics-dropwizard/pom.xml
new file mode 100644
index 0000000..a386880
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-dropwizard/pom.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metric-reporters</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-dropwizard</artifactId>
+ <name>flink-metrics-dropwizard</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java b/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
new file mode 100644
index 0000000..059704d
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.dropwizard;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.dropwizard.metrics.CounterWrapper;
+import org.apache.flink.dropwizard.metrics.GaugeWrapper;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.reporter.MetricReporter;
+import org.apache.flink.metrics.reporter.Scheduled;
+
+import java.util.List;
+
+/**
+ * Base class for {@link org.apache.flink.metrics.reporter.MetricReporter} that wraps a
+ * Dropwizard {@link com.codahale.metrics.Reporter}.
+ */
+@PublicEvolving
+public abstract class ScheduledDropwizardReporter implements MetricReporter, Scheduled {
+ protected MetricRegistry registry;
+ protected ScheduledReporter reporter;
+
+ public static final String ARG_HOST = "host";
+ public static final String ARG_PORT = "port";
+ public static final String ARG_PREFIX = "prefix";
+ public static final String ARG_CONVERSION_RATE = "rateConversion";
+ public static final String ARG_CONVERSION_DURATION = "durationConversion";
+
+ protected ScheduledDropwizardReporter() {
+ this.registry = new MetricRegistry();
+ }
+
+ @Override
+ public synchronized void notifyOfAddedMetric(Metric metric, String name) {
+ if (metric instanceof Counter) {
+ registry.register(name, new CounterWrapper((Counter) metric));
+ } else if (metric instanceof Gauge) {
+ registry.register(name, new GaugeWrapper((Gauge<?>) metric));
+ }
+ }
+
+ @Override
+ public synchronized void notifyOfRemovedMetric(Metric metric, String name) {
+ registry.remove(name);
+ }
+
+ public abstract ScheduledReporter getReporter(Configuration config);
+
+ @Override
+ public void open(Configuration config) {
+ this.reporter = getReporter(config);
+ }
+
+ @Override
+ public void close() {
+ this.reporter.stop();
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : scope) {
+ sb.append(s);
+ sb.append('.');
+ }
+ sb.append(name);
+ return sb.toString();
+ }
+
+ @Override
+ public synchronized void report() {
+ this.reporter.report(
+ this.registry.getGauges(),
+ this.registry.getCounters(),
+ this.registry.getHistograms(),
+ this.registry.getMeters(),
+ this.registry.getTimers());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java b/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
new file mode 100644
index 0000000..f6630b9
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.dropwizard.metrics;
+
+import org.apache.flink.metrics.Counter;
+
+public class CounterWrapper extends com.codahale.metrics.Counter {
+ private final Counter counter;
+
+ public CounterWrapper(Counter counter) {
+ this.counter = counter;
+ }
+
+ @Override
+ public long getCount() {
+ return this.counter.getCount();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java b/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
new file mode 100644
index 0000000..fcb629a
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.dropwizard.metrics;
+
+import org.apache.flink.metrics.Gauge;
+
+public class GaugeWrapper<T> implements com.codahale.metrics.Gauge<T> {
+ private final Gauge<T> gauge;
+
+ public GaugeWrapper(Gauge<T> gauge) {
+ this.gauge = gauge;
+ }
+
+ @Override
+ public T getValue() {
+ return this.gauge.getValue();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-ganglia/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-ganglia/pom.xml b/flink-metric-reporters/flink-metrics-ganglia/pom.xml
new file mode 100644
index 0000000..a457ca1
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-ganglia/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metric-reporters</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-ganglia</artifactId>
+ <name>flink-metrics-ganglia</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics-dropwizard</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>info.ganglia.gmetric4j</groupId>
+ <artifactId>gmetric4j</artifactId>
+ <version>1.0.7</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-ganglia</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java b/flink-metric-reporters/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
new file mode 100644
index 0000000..a1dafc9
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.graphite;
+
+import com.codahale.metrics.ScheduledReporter;
+import info.ganglia.gmetric4j.gmetric.GMetric;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.dropwizard.ScheduledDropwizardReporter;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public class GangliaReporter extends ScheduledDropwizardReporter {
+ public static final String ARG_DMAX = "dmax";
+ public static final String ARG_TMAX = "tmax";
+ public static final String ARG_TTL = "ttl";
+ public static final String ARG_MODE_ADDRESSING = "addressingMode";
+
+ @Override
+ public ScheduledReporter getReporter(Configuration config) {
+
+ try {
+ String host = config.getString(ARG_HOST, null);
+ int port = config.getInteger(ARG_PORT, -1);
+ if (host == null || host.length() == 0 || port < 1) {
+ throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
+ }
+ String addressingMode = config.getString(ARG_MODE_ADDRESSING, "MULTICAST");
+ int ttl = config.getInteger(ARG_TTL, -1);
+ GMetric gMetric = new GMetric(host, port, GMetric.UDPAddressingMode.valueOf(addressingMode), ttl);
+
+ String prefix = config.getString(ARG_PREFIX, null);
+ String conversionRate = config.getString(ARG_CONVERSION_RATE, null);
+ String conversionDuration = config.getString(ARG_CONVERSION_DURATION, null);
+ int dMax = config.getInteger(ARG_DMAX, 0);
+ int tMax = config.getInteger(ARG_TMAX, 60);
+
+ com.codahale.metrics.ganglia.GangliaReporter.Builder builder =
+ com.codahale.metrics.ganglia.GangliaReporter.forRegistry(registry);
+
+ if (prefix != null) {
+ builder.prefixedWith(prefix);
+ }
+ if (conversionRate != null) {
+ builder.convertRatesTo(TimeUnit.valueOf(conversionRate));
+ }
+ if (conversionDuration != null) {
+ builder.convertDurationsTo(TimeUnit.valueOf(conversionDuration));
+ }
+ builder.withDMax(dMax);
+ builder.withTMax(tMax);
+
+ return builder.build(gMetric);
+ } catch (IOException e) {
+ throw new RuntimeException("Error while instantiating GangliaReporter.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-graphite/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-graphite/pom.xml b/flink-metric-reporters/flink-metrics-graphite/pom.xml
new file mode 100644
index 0000000..714b77f
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-graphite/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metric-reporters</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-graphite</artifactId>
+ <name>flink-metrics-graphite</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics-dropwizard</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-graphite</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java b/flink-metric-reporters/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
new file mode 100644
index 0000000..b28d7a4
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.graphite;
+
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.graphite.Graphite;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.dropwizard.ScheduledDropwizardReporter;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class GraphiteReporter extends ScheduledDropwizardReporter {
+ @Override
+ public ScheduledReporter getReporter(Configuration config) {
+ String host = config.getString(ARG_HOST, null);
+ int port = config.getInteger(ARG_PORT, -1);
+
+ if (host == null || host.length() == 0 || port < 1) {
+ throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
+ }
+
+ String prefix = config.getString(ARG_PREFIX, null);
+ String conversionRate = config.getString(ARG_CONVERSION_RATE, null);
+ String conversionDuration = config.getString(ARG_CONVERSION_DURATION, null);
+
+ com.codahale.metrics.graphite.GraphiteReporter.Builder builder =
+ com.codahale.metrics.graphite.GraphiteReporter.forRegistry(registry);
+
+ if (prefix != null) {
+ builder.prefixedWith(prefix);
+ }
+
+ if (conversionRate != null) {
+ builder.convertRatesTo(TimeUnit.valueOf(conversionRate));
+ }
+
+ if (conversionDuration != null) {
+ builder.convertDurationsTo(TimeUnit.valueOf(conversionDuration));
+ }
+
+ return builder.build(new Graphite(host, port));
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : scope) {
+ sb.append(s.replace(".", "_").replace("\"", ""));
+ sb.append(".");
+ }
+ sb.append(name);
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-statsd/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-statsd/pom.xml b/flink-metric-reporters/flink-metrics-statsd/pom.xml
new file mode 100644
index 0000000..3052a10
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-statsd/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metric-reporters</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-statsd</artifactId>
+ <name>flink-metrics-statsd</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java b/flink-metric-reporters/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
new file mode 100644
index 0000000..e57001f
--- /dev/null
+++ b/flink-metric-reporters/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.statsd;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.reporter.AbstractReporter;
+import org.apache.flink.metrics.reporter.Scheduled;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
+import java.net.SocketException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Largely based on the StatsDReporter class by ReadyTalk
+ * https://github.com/ReadyTalk/metrics-statsd/blob/master/metrics3-statsd/src/main/java/com/readytalk/metrics/StatsDReporter.java
+ *
+ * Ported since it was not present in maven central.
+ */
+public class StatsDReporter extends AbstractReporter implements Scheduled {
+ private static final Logger LOG = LoggerFactory.getLogger(StatsDReporter.class);
+
+ public static final String ARG_HOST = "host";
+ public static final String ARG_PORT = "port";
+ public static final String ARG_CONVERSION_RATE = "rateConversion";
+ public static final String ARG_CONVERSION_DURATION = "durationConversion";
+
+ private DatagramSocket socket;
+ private InetSocketAddress address;
+
+ private double durationFactor;
+ private double rateFactor;
+
+ @Override
+ public void open(Configuration config) {
+ String host = config.getString(ARG_HOST, null);
+ int port = config.getInteger(ARG_PORT, -1);
+
+ if (host == null || host.length() == 0 || port < 1) {
+ throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
+ }
+
+ String conversionRate = config.getString(ARG_CONVERSION_RATE, "SECONDS");
+ String conversionDuration = config.getString(ARG_CONVERSION_DURATION, "MILLISECONDS");
+
+ this.address = new InetSocketAddress(host, port);
+ this.rateFactor = TimeUnit.valueOf(conversionRate).toSeconds(1);
+ this.durationFactor = 1.0 / TimeUnit.valueOf(conversionDuration).toNanos(1);
+ try {
+ this.socket = new DatagramSocket(0);
+ } catch (SocketException e) {
+ throw new RuntimeException("Failure while creating socket. ", e);
+ }
+ }
+
+ @Override
+ public void close() {
+ if (socket != null && !socket.isClosed()) {
+ socket.close();
+ }
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : scope) {
+ sb.append(s);
+ sb.append('.');
+ }
+ sb.append(name);
+ return sb.toString();
+ }
+
+ public void send(final String name, final double value) {
+ send(name, "" + value);
+ }
+
+ public void send(final String name, final String value) {
+ try {
+ String formatted = String.format("%s:%s|g", name, value);
+ byte[] data = formatted.getBytes();
+ socket.send(new DatagramPacket(data, data.length, this.address));
+ } catch (IOException e) {
+ LOG.error("unable to send packet to statsd at '{}:{}'", address.getHostName(), address.getPort());
+ }
+ }
+
+ @Override
+ public void report() {
+ for (Map.Entry<String, Gauge<?>> entry : gauges.entrySet()) {
+ reportGauge(entry.getKey(), entry.getValue());
+ }
+
+ for (Map.Entry<String, Counter> entry : counters.entrySet()) {
+ reportCounter(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private void reportCounter(final String name, final Counter counter) {
+ send(name, counter.getCount());
+ }
+
+ private void reportGauge(final String name, final Gauge<?> gauge) {
+ final String value = gauge.getValue().toString();
+ if (value != null) {
+ send((name), value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metric-reporters/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metric-reporters/pom.xml b/flink-metric-reporters/pom.xml
new file mode 100644
index 0000000..01a809c
--- /dev/null
+++ b/flink-metric-reporters/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metric-reporters</artifactId>
+ <name>flink-metric-reporters</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>flink-metrics-dropwizard</module>
+ <module>flink-metrics-ganglia</module>
+ <module>flink-metrics-graphite</module>
+ <module>flink-metrics-statsd</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-dropwizard/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/pom.xml b/flink-metrics/flink-metrics-dropwizard/pom.xml
deleted file mode 100644
index 84d9722..0000000
--- a/flink-metrics/flink-metrics-dropwizard/pom.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-metrics</artifactId>
- <version>1.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-metrics-dropwizard</artifactId>
- <name>flink-metrics-dropwizard</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.4</version>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
deleted file mode 100644
index a7309be..0000000
--- a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.dropwizard;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.ScheduledReporter;
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.dropwizard.metrics.CounterWrapper;
-import org.apache.flink.dropwizard.metrics.GaugeWrapper;
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.Gauge;
-import org.apache.flink.metrics.Metric;
-import org.apache.flink.metrics.reporter.MetricReporter;
-import org.apache.flink.metrics.reporter.Scheduled;
-
-import java.util.List;
-
-/**
- * Base class for {@link org.apache.flink.metrics.reporter.MetricReporter} that wraps a
- * Dropwizard {@link com.codahale.metrics.Reporter}.
- */
-@PublicEvolving
-public abstract class ScheduledDropwizardReporter implements MetricReporter, Scheduled {
- protected MetricRegistry registry;
- protected ScheduledReporter reporter;
-
- public static final String ARG_HOST = "host";
- public static final String ARG_PORT = "port";
- public static final String ARG_PREFIX = "prefix";
- public static final String ARG_CONVERSION_RATE = "rateConversion";
- public static final String ARG_CONVERSION_DURATION = "durationConversion";
-
- protected ScheduledDropwizardReporter() {
- this.registry = new MetricRegistry();
- }
-
- @Override
- public synchronized void notifyOfAddedMetric(Metric metric, String name) {
- if (metric instanceof Counter) {
- registry.register(name, new CounterWrapper((Counter) metric));
- } else if (metric instanceof Gauge) {
- registry.register(name, new GaugeWrapper((Gauge) metric));
- }
- }
-
- @Override
- public synchronized void notifyOfRemovedMetric(Metric metric, String name) {
- registry.remove(name);
- }
-
- public abstract ScheduledReporter getReporter(Configuration config);
-
- @Override
- public void open(Configuration config) {
- this.reporter = getReporter(config);
- }
-
- @Override
- public void close() {
- this.reporter.stop();
- }
-
- @Override
- public String generateName(String name, List<String> scope) {
- StringBuilder sb = new StringBuilder();
- for (String s : scope) {
- sb.append(s);
- sb.append('.');
- }
- sb.append(name);
- return sb.toString();
- }
-
- @Override
- public synchronized void report() {
- this.reporter.report(
- this.registry.getGauges(),
- this.registry.getCounters(),
- this.registry.getHistograms(),
- this.registry.getMeters(),
- this.registry.getTimers());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
deleted file mode 100644
index f6630b9..0000000
--- a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.dropwizard.metrics;
-
-import org.apache.flink.metrics.Counter;
-
-public class CounterWrapper extends com.codahale.metrics.Counter {
- private final Counter counter;
-
- public CounterWrapper(Counter counter) {
- this.counter = counter;
- }
-
- @Override
- public long getCount() {
- return this.counter.getCount();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
deleted file mode 100644
index d47090d..0000000
--- a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.dropwizard.metrics;
-
-import org.apache.flink.metrics.Gauge;
-
-public class GaugeWrapper implements com.codahale.metrics.Gauge {
- private final Gauge gauge;
-
- public GaugeWrapper(Gauge gauge) {
- this.gauge = gauge;
- }
-
- @Override
- public Object getValue() {
- return this.gauge.getValue();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-ganglia/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-ganglia/pom.xml b/flink-metrics/flink-metrics-ganglia/pom.xml
deleted file mode 100644
index c4f51da..0000000
--- a/flink-metrics/flink-metrics-ganglia/pom.xml
+++ /dev/null
@@ -1,90 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-metrics</artifactId>
- <version>1.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-metrics-ganglia</artifactId>
- <name>flink-metrics-ganglia</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-metrics-dropwizard</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>info.ganglia.gmetric4j</groupId>
- <artifactId>gmetric4j</artifactId>
- <version>1.0.7</version>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-ganglia</artifactId>
- <version>${metrics.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.4</version>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java b/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
deleted file mode 100644
index a1dafc9..0000000
--- a/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.metrics.graphite;
-
-import com.codahale.metrics.ScheduledReporter;
-import info.ganglia.gmetric4j.gmetric.GMetric;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.dropwizard.ScheduledDropwizardReporter;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-public class GangliaReporter extends ScheduledDropwizardReporter {
- public static final String ARG_DMAX = "dmax";
- public static final String ARG_TMAX = "tmax";
- public static final String ARG_TTL = "ttl";
- public static final String ARG_MODE_ADDRESSING = "addressingMode";
-
- @Override
- public ScheduledReporter getReporter(Configuration config) {
-
- try {
- String host = config.getString(ARG_HOST, null);
- int port = config.getInteger(ARG_PORT, -1);
- if (host == null || host.length() == 0 || port < 1) {
- throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
- }
- String addressingMode = config.getString(ARG_MODE_ADDRESSING, "MULTICAST");
- int ttl = config.getInteger(ARG_TTL, -1);
- GMetric gMetric = new GMetric(host, port, GMetric.UDPAddressingMode.valueOf(addressingMode), ttl);
-
- String prefix = config.getString(ARG_PREFIX, null);
- String conversionRate = config.getString(ARG_CONVERSION_RATE, null);
- String conversionDuration = config.getString(ARG_CONVERSION_DURATION, null);
- int dMax = config.getInteger(ARG_DMAX, 0);
- int tMax = config.getInteger(ARG_TMAX, 60);
-
- com.codahale.metrics.ganglia.GangliaReporter.Builder builder =
- com.codahale.metrics.ganglia.GangliaReporter.forRegistry(registry);
-
- if (prefix != null) {
- builder.prefixedWith(prefix);
- }
- if (conversionRate != null) {
- builder.convertRatesTo(TimeUnit.valueOf(conversionRate));
- }
- if (conversionDuration != null) {
- builder.convertDurationsTo(TimeUnit.valueOf(conversionDuration));
- }
- builder.withDMax(dMax);
- builder.withTMax(tMax);
-
- return builder.build(gMetric);
- } catch (IOException e) {
- throw new RuntimeException("Error while instantiating GangliaReporter.", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-graphite/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-graphite/pom.xml b/flink-metrics/flink-metrics-graphite/pom.xml
deleted file mode 100644
index 45fb018..0000000
--- a/flink-metrics/flink-metrics-graphite/pom.xml
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-metrics</artifactId>
- <version>1.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-metrics-graphite</artifactId>
- <name>flink-metrics-graphite</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-metrics-dropwizard</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-graphite</artifactId>
- <version>${metrics.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.4</version>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- <executions>
- <execution>
- <id>make-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java b/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
deleted file mode 100644
index b28d7a4..0000000
--- a/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.metrics.graphite;
-
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.graphite.Graphite;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.dropwizard.ScheduledDropwizardReporter;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-public class GraphiteReporter extends ScheduledDropwizardReporter {
- @Override
- public ScheduledReporter getReporter(Configuration config) {
- String host = config.getString(ARG_HOST, null);
- int port = config.getInteger(ARG_PORT, -1);
-
- if (host == null || host.length() == 0 || port < 1) {
- throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
- }
-
- String prefix = config.getString(ARG_PREFIX, null);
- String conversionRate = config.getString(ARG_CONVERSION_RATE, null);
- String conversionDuration = config.getString(ARG_CONVERSION_DURATION, null);
-
- com.codahale.metrics.graphite.GraphiteReporter.Builder builder =
- com.codahale.metrics.graphite.GraphiteReporter.forRegistry(registry);
-
- if (prefix != null) {
- builder.prefixedWith(prefix);
- }
-
- if (conversionRate != null) {
- builder.convertRatesTo(TimeUnit.valueOf(conversionRate));
- }
-
- if (conversionDuration != null) {
- builder.convertDurationsTo(TimeUnit.valueOf(conversionDuration));
- }
-
- return builder.build(new Graphite(host, port));
- }
-
- @Override
- public String generateName(String name, List<String> scope) {
- StringBuilder sb = new StringBuilder();
- for (String s : scope) {
- sb.append(s.replace(".", "_").replace("\"", ""));
- sb.append(".");
- }
- sb.append(name);
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-statsd/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-statsd/pom.xml b/flink-metrics/flink-metrics-statsd/pom.xml
deleted file mode 100644
index 5d63908..0000000
--- a/flink-metrics/flink-metrics-statsd/pom.xml
+++ /dev/null
@@ -1,43 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-metrics</artifactId>
- <version>1.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-metrics-statsd</artifactId>
- <name>flink-metrics-statsd</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java b/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
deleted file mode 100644
index 288b8b8..0000000
--- a/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.metrics.statsd;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.Gauge;
-import org.apache.flink.metrics.reporter.AbstractReporter;
-import org.apache.flink.metrics.reporter.Scheduled;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.DatagramPacket;
-import java.net.DatagramSocket;
-import java.net.InetSocketAddress;
-import java.net.SocketException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Largely based on the StatsDReporter class by ReadyTalk
- * https://github.com/ReadyTalk/metrics-statsd/blob/master/metrics3-statsd/src/main/java/com/readytalk/metrics/StatsDReporter.java
- *
- * Ported since it was not present in maven central.
- */
-public class StatsDReporter extends AbstractReporter implements Scheduled {
- private static final Logger LOG = LoggerFactory.getLogger(StatsDReporter.class);
-
- public static final String ARG_HOST = "host";
- public static final String ARG_PORT = "port";
- public static final String ARG_CONVERSION_RATE = "rateConversion";
- public static final String ARG_CONVERSION_DURATION = "durationConversion";
-
- private DatagramSocket socket;
- private InetSocketAddress address;
-
- private double durationFactor;
- private double rateFactor;
-
- @Override
- public void open(Configuration config) {
- String host = config.getString(ARG_HOST, null);
- int port = config.getInteger(ARG_PORT, -1);
-
- if (host == null || host.length() == 0 || port < 1) {
- throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
- }
-
- String conversionRate = config.getString(ARG_CONVERSION_RATE, "SECONDS");
- String conversionDuration = config.getString(ARG_CONVERSION_DURATION, "MILLISECONDS");
-
- this.address = new InetSocketAddress(host, port);
- this.rateFactor = TimeUnit.valueOf(conversionRate).toSeconds(1);
- this.durationFactor = 1.0 / TimeUnit.valueOf(conversionDuration).toNanos(1);
- try {
- this.socket = new DatagramSocket(0);
- } catch (SocketException e) {
- throw new RuntimeException("Failure while creating socket. ", e);
- }
- }
-
- @Override
- public void close() {
- if (socket != null && !socket.isClosed()) {
- socket.close();
- }
- }
-
- @Override
- public String generateName(String name, List<String> scope) {
- StringBuilder sb = new StringBuilder();
- for (String s : scope) {
- sb.append(s);
- sb.append('.');
- }
- sb.append(name);
- return sb.toString();
- }
-
- public void send(final String name, final double value) {
- send(name, "" + value);
- }
-
- public void send(final String name, final String value) {
- try {
- String formatted = String.format("%s:%s|g", name, value);
- byte[] data = formatted.getBytes();
- socket.send(new DatagramPacket(data, data.length, this.address));
- } catch (IOException e) {
- LOG.error("unable to send packet to statsd at '{}:{}'", address.getHostName(), address.getPort());
- }
- }
-
- @Override
- public void report() {
- for (Map.Entry<String, Gauge> entry : gauges.entrySet()) {
- reportGauge(entry.getKey(), entry.getValue());
- }
-
- for (Map.Entry<String, Counter> entry : counters.entrySet()) {
- reportCounter(entry.getKey(), entry.getValue());
- }
- }
-
- private void reportCounter(final String name, final Counter counter) {
- send(name, counter.getCount());
- }
-
- private void reportGauge(final String name, final Gauge<?> gauge) {
- final String value = gauge.getValue().toString();
- if (value != null) {
- send((name), value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/pom.xml b/flink-metrics/pom.xml
deleted file mode 100644
index 542f49c..0000000
--- a/flink-metrics/pom.xml
+++ /dev/null
@@ -1,42 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-parent</artifactId>
- <version>1.1-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-metrics</artifactId>
- <name>flink-metrics</name>
- <packaging>pom</packaging>
-
- <modules>
- <module>flink-metrics-dropwizard</module>
- <module>flink-metrics-ganglia</module>
- <module>flink-metrics-graphite</module>
- <module>flink-metrics-statsd</module>
- </modules>
-</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index bc3ba0d..1f766e1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -60,6 +60,7 @@ import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint;
import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.runtime.state.StateUtils;
import org.apache.flink.util.SerializedValue;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,15 +92,15 @@ import static com.google.common.base.Preconditions.checkNotNull;
* {@link org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable} have only data
* readers, -writers, and certain event callbacks. The task connects those to the
* network stack and actor messages, and tracks the state of the execution and
- * handles exceptions.</p>
+ * handles exceptions.
*
* <p>Tasks have no knowledge about how they relate to other tasks, or whether they
* are the first attempt to execute the task, or a repeated attempt. All of that
* is only known to the JobManager. All the task knows are its own runnable code,
* the task's configuration, and the IDs of the intermediate results to consume and
- * produce (if any).</p>
+ * produce (if any).
*
- * <p>Each Task is run by one dedicated thread.</p>
+ * <p>Each Task is run by one dedicated thread.
*/
public class Task implements Runnable {
@@ -129,6 +130,7 @@ public class Task implements Runnable {
/** TaskInfo object for this task */
private final TaskInfo taskInfo;
+ /** The name of the task, including subtask indexes */
private final String taskNameWithSubtask;
/** The job-wide configuration object */
@@ -158,6 +160,9 @@ public class Task implements Runnable {
/** The BroadcastVariableManager to be used by this task */
private final BroadcastVariableManager broadcastVariableManager;
+ /** Serialized version of the job specific execution configuration (see {@link ExecutionConfig}). */
+ private final SerializedValue<ExecutionConfig> serializedExecutionConfig;
+
private final ResultPartition[] producedPartitions;
private final ResultPartitionWriter[] writers;
@@ -193,6 +198,9 @@ public class Task implements Runnable {
/** The thread that executes the task */
private final Thread executingThread;
+ /** Parent group for all metrics of this task */
+ private final TaskMetricGroup metrics;
+
// ------------------------------------------------------------------------
// Fields that control the task execution. All these fields are volatile
// (which means that they introduce memory barriers), to establish
@@ -202,8 +210,6 @@ public class Task implements Runnable {
/** atomic flag that makes sure the invokable is canceled exactly once upon error */
private final AtomicBoolean invokableHasBeenCanceled;
- private final TaskMetricGroup metrics;
-
/** The invokable of this task, if initialized */
private volatile AbstractInvokable invokable;
@@ -215,16 +221,13 @@ public class Task implements Runnable {
/** Serial executor for asynchronous calls (checkpoints, etc), lazily initialized */
private volatile ExecutorService asyncCallDispatcher;
-
+
/** The handle to the state that the operator was initialized with. Will be set to null after the
* initialization, to be memory friendly */
private volatile SerializedValue<StateHandle<?>> operatorState;
private volatile long recoveryTs;
- /** Serialized version of the job specific execution configuration (see {@link ExecutionConfig}). */
- private final SerializedValue<ExecutionConfig> serializedExecutionConfig;
-
/** Initialized from the Flink configuration. May also be set at the ExecutionConfig */
private long taskCancellationInterval;
@@ -688,9 +691,6 @@ public class Task implements Runnable {
// remove all of the tasks library resources
libraryCache.unregisterTask(jobId, executionId);
-
- //Uncomment before Merging!!!
- //metrics.close();
// remove all files in the distributed cache
removeCachedFiles(distributedCacheEntries, fileCache);
@@ -703,6 +703,16 @@ public class Task implements Runnable {
LOG.error(message, t);
notifyFatalError(message, t);
}
+
+ // un-register the metrics at the end so that the task may already be
+ // counted as finished when this happens
+ // errors here will only be logged
+ try {
+ metrics.close();
+ }
+ catch (Throwable t) {
+ LOG.error("Error during metrics de-registration", t);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index beb012c..a5cc18d 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -155,8 +155,8 @@ class TaskManager(
/** Registry of metrics periodically transmitted to the JobManager */
private val metricRegistry = TaskManager.createMetricsRegistry()
- private var metricsRegistry : FlinkMetricRegistry = null
- private var taskManagerMetricGroup : TaskManagerMetricGroup = null
+ private var metricsRegistry : FlinkMetricRegistry = _
+ private var taskManagerMetricGroup : TaskManagerMetricGroup = _
/** Metric serialization */
private val metricRegistryMapper: ObjectMapper = new ObjectMapper()
@@ -938,7 +938,7 @@ class TaskManager(
libraryCacheManager = Some(new FallbackLibraryCacheManager)
}
- metricsRegistry = new FlinkMetricRegistry(this.config.configuration);
+ metricsRegistry = new FlinkMetricRegistry(config.configuration)
taskManagerMetricGroup =
new TaskManagerMetricGroup(metricsRegistry, this.runtimeInfo.getHostname, id.toString)
@@ -1011,6 +1011,10 @@ class TaskManager(
// disassociate the network environment
network.disassociate()
+
+ // stop the metrics reporters
+ metricsRegistry.shutdown()
+ metricsRegistry = null
}
protected def handleJobManagerDisconnect(jobManager: ActorRef, msg: String): Unit = {
@@ -1085,8 +1089,9 @@ class TaskManager(
}
val taskMetricGroup = taskManagerMetricGroup
- .addJob(tdd.getJobID, jobName)
- .addTask(tdd.getVertexID, tdd.getExecutionId, tdd.getIndexInSubtaskGroup, tdd.getTaskName)
+ .addTaskForJob(
+ tdd.getJobID, jobName,
+ tdd.getVertexID, tdd.getExecutionId, tdd.getIndexInSubtaskGroup, tdd.getTaskName)
val task = new Task(
tdd,
@@ -1224,16 +1229,16 @@ class TaskManager(
registry.getSnapshot
}
- self ! decorateMessage(
- UpdateTaskExecutionState(
- new TaskExecutionState(
- task.getJobID,
- task.getExecutionId,
- task.getExecutionState,
- task.getFailureCause,
- accumulators)
- )
+ self ! decorateMessage(
+ UpdateTaskExecutionState(
+ new TaskExecutionState(
+ task.getJobID,
+ task.getExecutionId,
+ task.getExecutionState,
+ task.getFailureCause,
+ accumulators)
)
+ )
}
else {
log.error(s"Cannot find task with ID $executionID to unregister.")
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c97815a..784aa40 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,7 +74,7 @@ under the License.
<module>flink-quickstart</module>
<module>flink-contrib</module>
<module>flink-dist</module>
- <module>flink-metrics</module>
+ <module>flink-metric-reporters</module>
</modules>
<properties>
[6/6] flink git commit: [FLINK-1502] [core] Add basic metric system
Posted by se...@apache.org.
[FLINK-1502] [core] Add basic metric system
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/003ce18e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/003ce18e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/003ce18e
Branch: refs/heads/master
Commit: 003ce18efc0249fae874e56c3df6acf19f5f2429
Parents: 8ed3685
Author: zentol <ch...@apache.org>
Authored: Mon May 9 14:54:32 2016 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Sun May 22 19:58:17 2016 +0200
----------------------------------------------------------------------
.../flink-statebackend-rocksdb/pom.xml | 7 +
flink-contrib/flink-storm/pom.xml | 8 +
.../flink/storm/wrappers/BoltWrapperTest.java | 7 +
flink-core/pom.xml | 6 +
.../api/common/functions/RuntimeContext.java | 8 +
.../util/AbstractRuntimeUDFContext.java | 12 +-
.../functions/util/RuntimeUDFContext.java | 6 +-
.../common/operators/CollectionExecutor.java | 48 +++-
.../java/org/apache/flink/metrics/Counter.java | 69 ++++++
.../java/org/apache/flink/metrics/Gauge.java | 33 +++
.../java/org/apache/flink/metrics/Metric.java | 27 +++
.../org/apache/flink/metrics/MetricGroup.java | 99 +++++++++
.../apache/flink/metrics/MetricRegistry.java | 170 +++++++++++++++
.../metrics/groups/AbstractMetricGroup.java | 145 +++++++++++++
.../metrics/groups/ComponentMetricGroup.java | 113 ++++++++++
.../metrics/groups/GenericMetricGroup.java | 57 +++++
.../flink/metrics/groups/IOMetricGroup.java | 70 ++++++
.../flink/metrics/groups/JobMetricGroup.java | 71 ++++++
.../metrics/groups/OperatorMetricGroup.java | 46 ++++
.../org/apache/flink/metrics/groups/Scope.java | 120 ++++++++++
.../metrics/groups/TaskManagerMetricGroup.java | 70 ++++++
.../flink/metrics/groups/TaskMetricGroup.java | 87 ++++++++
.../metrics/reporter/AbstractReporter.java | 48 ++++
.../flink/metrics/reporter/JMXReporter.java | 167 ++++++++++++++
.../flink/metrics/reporter/MetricReporter.java | 73 +++++++
.../flink/metrics/reporter/Scheduled.java | 32 +++
.../functions/util/RuntimeUDFContextTest.java | 11 +-
.../api/common/io/RichInputFormatTest.java | 3 +-
.../api/common/io/RichOutputFormatTest.java | 3 +-
.../operators/GenericDataSinkBaseTest.java | 5 +-
.../operators/GenericDataSourceBaseTest.java | 5 +-
.../base/FlatMapOperatorCollectionTest.java | 3 +-
.../base/InnerJoinOperatorBaseTest.java | 5 +-
.../common/operators/base/MapOperatorTest.java | 5 +-
.../base/PartitionMapOperatorTest.java | 5 +-
.../apache/flink/metrics/MetricGroupTest.java | 93 ++++++++
.../flink/metrics/MetricRegistryTest.java | 217 +++++++++++++++++++
.../flink/metrics/groups/JobGroupTest.java | 71 ++++++
.../flink/metrics/groups/OperatorGroupTest.java | 86 ++++++++
.../flink/metrics/groups/TaskGroupTest.java | 80 +++++++
.../metrics/groups/TaskManagerGroupTest.java | 70 ++++++
.../flink/metrics/reporter/JMXReporterTest.java | 43 ++++
.../flink/metrics/util/DummyJobMetricGroup.java | 50 +++++
.../flink/metrics/util/DummyMetricGroup.java | 57 +++++
.../flink/metrics/util/DummyMetricRegistry.java | 34 +++
.../metrics/util/DummyOperatorMetricGroup.java | 43 ++++
.../flink/metrics/util/DummyReporter.java | 47 ++++
.../util/DummyTaskManagerMetricGroup.java | 48 ++++
.../metrics/util/DummyTaskMetricGroup.java | 48 ++++
.../apache/flink/metrics/util/TestReporter.java | 40 ++++
flink-dist/src/main/flink-bin/bin/config.sh | 6 +
.../src/main/flink-bin/bin/flink-daemon.sh | 18 +-
.../base/CoGroupOperatorCollectionTest.java | 3 +-
.../operators/base/GroupReduceOperatorTest.java | 5 +-
.../base/InnerJoinOperatorBaseTest.java | 5 +-
.../operators/base/ReduceOperatorTest.java | 5 +-
flink-metrics/flink-metrics-dropwizard/pom.xml | 72 ++++++
.../dropwizard/ScheduledDropwizardReporter.java | 99 +++++++++
.../dropwizard/metrics/CounterWrapper.java | 33 +++
.../flink/dropwizard/metrics/GaugeWrapper.java | 33 +++
flink-metrics/flink-metrics-ganglia/pom.xml | 90 ++++++++
.../flink/metrics/graphite/GangliaReporter.java | 73 +++++++
flink-metrics/flink-metrics-graphite/pom.xml | 84 +++++++
.../metrics/graphite/GraphiteReporter.java | 70 ++++++
flink-metrics/flink-metrics-statsd/pom.xml | 43 ++++
.../flink/metrics/statsd/StatsDReporter.java | 132 +++++++++++
flink-metrics/pom.xml | 42 ++++
flink-runtime/pom.xml | 4 -
.../deployment/TaskDeploymentDescriptor.java | 7 +
.../flink/runtime/execution/Environment.java | 8 +
.../runtime/executiongraph/ExecutionVertex.java | 1 +
.../api/reader/AbstractRecordReader.java | 8 +
.../io/network/api/reader/BufferReader.java | 5 +
.../io/network/api/reader/ReaderBase.java | 8 +
.../AdaptiveSpanningRecordDeserializer.java | 20 ++
.../api/serialization/RecordDeserializer.java | 8 +
.../api/serialization/RecordSerializer.java | 8 +
.../serialization/SpanningRecordSerializer.java | 19 ++
...llingAdaptiveSpanningRecordDeserializer.java | 21 ++
.../io/network/api/writer/RecordWriter.java | 11 +
.../iterative/task/AbstractIterativeTask.java | 9 +-
.../flink/runtime/operators/BatchTask.java | 19 +-
.../flink/runtime/operators/DataSinkTask.java | 5 +-
.../flink/runtime/operators/DataSourceTask.java | 5 +-
.../runtime/operators/GroupReduceDriver.java | 2 +-
.../flink/runtime/operators/TaskContext.java | 3 +
.../operators/chaining/ChainedDriver.java | 8 +-
.../util/DistributedRuntimeUDFContext.java | 5 +-
.../runtime/taskmanager/RuntimeEnvironment.java | 11 +-
.../apache/flink/runtime/taskmanager/Task.java | 12 +-
.../flink/runtime/taskmanager/TaskManager.scala | 169 ++++++++++++++-
.../TaskDeploymentDescriptorTest.java | 3 +-
.../network/api/reader/AbstractReaderTest.java | 5 +
.../operators/drivers/TestTaskContext.java | 7 +
.../testutils/BinaryOperatorTestBase.java | 7 +
.../operators/testutils/DriverTestBase.java | 7 +
.../operators/testutils/DummyEnvironment.java | 7 +
.../operators/testutils/MockEnvironment.java | 7 +
.../testutils/UnaryOperatorTestBase.java | 6 +
.../runtime/taskmanager/TaskAsyncCallTest.java | 6 +-
.../runtime/taskmanager/TaskManagerTest.java | 27 +--
.../flink/runtime/taskmanager/TaskStopTest.java | 3 +-
.../flink/runtime/taskmanager/TaskTest.java | 6 +-
.../flink/streaming/api/graph/StreamConfig.java | 9 +
.../api/graph/StreamingJobGraphGenerator.java | 11 +-
.../api/operators/AbstractStreamOperator.java | 9 +
.../streaming/api/operators/StreamOperator.java | 3 +
.../api/operators/StreamingRuntimeContext.java | 3 +-
.../runtime/io/StreamInputProcessor.java | 12 +
.../runtime/io/StreamTwoInputProcessor.java | 12 +
.../runtime/tasks/OneInputStreamTask.java | 1 +
.../streaming/runtime/tasks/OperatorChain.java | 1 +
.../runtime/tasks/TwoInputStreamTask.java | 1 +
...AlignedProcessingTimeWindowOperatorTest.java | 2 +
...AlignedProcessingTimeWindowOperatorTest.java | 2 +
.../runtime/tasks/StreamMockEnvironment.java | 7 +
.../streaming/runtime/tasks/StreamTaskTest.java | 6 +-
pom.xml | 2 +
118 files changed, 3936 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-contrib/flink-statebackend-rocksdb/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-statebackend-rocksdb/pom.xml b/flink-contrib/flink-statebackend-rocksdb/pom.xml
index b966371..cccdc20 100644
--- a/flink-contrib/flink-statebackend-rocksdb/pom.xml
+++ b/flink-contrib/flink-statebackend-rocksdb/pom.xml
@@ -54,6 +54,13 @@ under the License.
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
<artifactId>flink-runtime_2.10</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-contrib/flink-storm/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-storm/pom.xml b/flink-contrib/flink-storm/pom.xml
index 24b0b4b..a080a03 100644
--- a/flink-contrib/flink-storm/pom.xml
+++ b/flink-contrib/flink-storm/pom.xml
@@ -49,6 +49,14 @@ under the License.
</dependency>
<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>0.9.4</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
index e65dc45..cb9ac1c 100644
--- a/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
+++ b/flink-contrib/flink-storm/src/test/java/org/apache/flink/storm/wrappers/BoltWrapperTest.java
@@ -32,7 +32,10 @@ import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.UnmodifiableConfiguration;
+import org.apache.flink.metrics.util.DummyMetricGroup;
+import org.apache.flink.metrics.util.DummyTaskMetricGroup;
import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.taskmanager.RuntimeEnvironment;
import org.apache.flink.storm.util.AbstractTest;
import org.apache.flink.storm.util.SplitStreamType;
import org.apache.flink.storm.util.StormConfig;
@@ -141,6 +144,7 @@ public class BoltWrapperTest extends AbstractTest {
final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class);
when(taskContext.getExecutionConfig()).thenReturn(mock(ExecutionConfig.class));
when(taskContext.getTaskName()).thenReturn("name");
+ when(taskContext.getMetricGroup()).thenReturn(new DummyMetricGroup());
final IRichBolt bolt = mock(IRichBolt.class);
@@ -225,6 +229,7 @@ public class BoltWrapperTest extends AbstractTest {
final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class);
when(taskContext.getExecutionConfig()).thenReturn(taskConfig);
when(taskContext.getTaskName()).thenReturn("name");
+ when(taskContext.getMetricGroup()).thenReturn(new DummyMetricGroup());
final SetupOutputFieldsDeclarer declarer = new SetupOutputFieldsDeclarer();
declarer.declare(new Fields("dummy"));
@@ -289,6 +294,7 @@ public class BoltWrapperTest extends AbstractTest {
final StreamingRuntimeContext taskContext = mock(StreamingRuntimeContext.class);
when(taskContext.getExecutionConfig()).thenReturn(taskConfig);
when(taskContext.getTaskName()).thenReturn("name");
+ when(taskContext.getMetricGroup()).thenReturn(new DummyMetricGroup());
final IRichBolt bolt = mock(IRichBolt.class);
BoltWrapper<Object, Object> wrapper = new BoltWrapper<Object, Object>(bolt);
@@ -361,6 +367,7 @@ public class BoltWrapperTest extends AbstractTest {
Environment env = mock(Environment.class);
when(env.getTaskInfo()).thenReturn(new TaskInfo("Mock Task", 0, 1, 0));
when(env.getUserClassLoader()).thenReturn(BoltWrapperTest.class.getClassLoader());
+ when(env.getMetricGroup()).thenReturn(new DummyTaskMetricGroup());
StreamTask<?, ?> mockTask = mock(StreamTask.class);
when(mockTask.getCheckpointLock()).thenReturn(new Object());
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-core/pom.xml b/flink-core/pom.xml
index eb55bdd..0e0b54b 100644
--- a/flink-core/pom.xml
+++ b/flink-core/pom.xml
@@ -47,6 +47,12 @@ under the License.
<!-- managed version -->
</dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
<!-- Avro is needed for the interoperability with Avro types for serialization -->
<dependency>
<groupId>org.apache.avro</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
index 3225c00..ed2f613 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/RuntimeContext.java
@@ -39,6 +39,7 @@ import org.apache.flink.api.common.state.ReducingStateDescriptor;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.metrics.MetricGroup;
/**
* A RuntimeContext contains information about the context in which functions are executed. Each parallel instance
@@ -59,6 +60,13 @@ public interface RuntimeContext {
String getTaskName();
/**
+ * Returns the metric group for this parallel subtask.
+ *
+ * @return The metric group for this parallel subtask.
+ */
+ MetricGroup getMetricGroup();
+
+ /**
* Gets the parallelism with which the parallel task runs.
*
* @return The parallelism with which the parallel task runs.
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
index 74b78df..6645964 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
@@ -43,6 +43,7 @@ import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.MetricGroup;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -61,17 +62,21 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
private final Map<String, Accumulator<?, ?>> accumulators;
private final DistributedCache distributedCache;
+
+ private final MetricGroup metrics;
public AbstractRuntimeUDFContext(TaskInfo taskInfo,
ClassLoader userCodeClassLoader,
ExecutionConfig executionConfig,
Map<String, Accumulator<?,?>> accumulators,
- Map<String, Future<Path>> cpTasks) {
+ Map<String, Future<Path>> cpTasks,
+ MetricGroup metrics) {
this.taskInfo = checkNotNull(taskInfo);
this.userCodeClassLoader = userCodeClassLoader;
this.executionConfig = executionConfig;
this.distributedCache = new DistributedCache(checkNotNull(cpTasks));
this.accumulators = checkNotNull(accumulators);
+ this.metrics = metrics;
}
@Override
@@ -93,6 +98,11 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
public int getIndexOfThisSubtask() {
return taskInfo.getIndexOfThisSubtask();
}
+
+ @Override
+ public MetricGroup getMetricGroup() {
+ return metrics;
+ }
@Override
public int getAttemptNumber() {
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
index 2337afa..6571d0d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.MetricGroup;
/**
* A standalone implementation of the {@link RuntimeContext}, created by runtime UDF operators.
@@ -42,8 +43,9 @@ public class RuntimeUDFContext extends AbstractRuntimeUDFContext {
private final HashMap<String, List<?>> uninitializedBroadcastVars = new HashMap<String, List<?>>();
public RuntimeUDFContext(TaskInfo taskInfo, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
- Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?,?>> accumulators) {
- super(taskInfo, userCodeClassLoader, executionConfig, accumulators, cpTasks);
+ Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?, ?>> accumulators,
+ MetricGroup metrics) {
+ super(taskInfo, userCodeClassLoader, executionConfig, accumulators, cpTasks, metrics);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
index 44042c4..7e5269e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
@@ -36,6 +36,7 @@ import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.Plan;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.api.common.accumulators.Accumulator;
@@ -58,9 +59,15 @@ import org.apache.flink.api.common.operators.util.TypeComparable;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.core.fs.local.LocalFileSystem;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.metrics.groups.JobMetricGroup;
+import org.apache.flink.metrics.groups.TaskManagerMetricGroup;
import org.apache.flink.types.Value;
+import org.apache.flink.util.AbstractID;
import org.apache.flink.util.Visitor;
/**
@@ -86,6 +93,8 @@ public class CollectionExecutor {
private final ExecutionConfig executionConfig;
private int iterationSuperstep;
+
+ private JobMetricGroup jobMetricGroup;
// --------------------------------------------------------------------------------------------
@@ -106,6 +115,14 @@ public class CollectionExecutor {
public JobExecutionResult execute(Plan program) throws Exception {
long startTime = System.currentTimeMillis();
+
+ JobID jobID = program.getJobId();
+ if (jobID == null) {
+ jobID = new JobID();
+ }
+ this.jobMetricGroup =
+ new TaskManagerMetricGroup(new MetricRegistry(new Configuration()), "localhost", new AbstractID().toString())
+ .addJob(jobID, program.getJobName());
initCache(program.getCachedFiles());
Collection<? extends GenericDataSinkBase<?>> sinks = program.getDataSinks();
for (Operator<?> sink : sinks) {
@@ -184,9 +201,12 @@ public class CollectionExecutor {
// build the runtime context and compute broadcast variables, if necessary
TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 0, 1, 0);
RuntimeUDFContext ctx;
+
+ MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, typedSink.getName());
+
if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
} else {
ctx = null;
}
@@ -200,10 +220,13 @@ public class CollectionExecutor {
GenericDataSourceBase<OUT, ?> typedSource = (GenericDataSourceBase<OUT, ?>) source;
// build the runtime context and compute broadcast variables, if necessary
TaskInfo taskInfo = new TaskInfo(typedSource.getName(), 0, 1, 0);
+
RuntimeUDFContext ctx;
+
+ MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, source.getName());
if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
} else {
ctx = null;
}
@@ -225,9 +248,11 @@ public class CollectionExecutor {
// build the runtime context and compute broadcast variables, if necessary
TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 0, 1, 0);
RuntimeUDFContext ctx;
+
+ MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, typedOp.getName());
if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
List<?> bcData = execute(bcInputs.getValue());
@@ -265,9 +290,11 @@ public class CollectionExecutor {
// build the runtime context and compute broadcast variables, if necessary
TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 0, 1, 0);
RuntimeUDFContext ctx;
+
+ MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, typedOp.getName());
if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
- ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators) :
- new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators);
+ ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
+ new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) {
List<?> bcData = execute(bcInputs.getValue());
@@ -523,8 +550,9 @@ public class CollectionExecutor {
private class IterationRuntimeUDFContext extends RuntimeUDFContext implements IterationRuntimeContext {
public IterationRuntimeUDFContext(TaskInfo taskInfo, ClassLoader classloader, ExecutionConfig executionConfig,
- Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?,?>> accumulators) {
- super(taskInfo, classloader, executionConfig, cpTasks, accumulators);
+ Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?, ?>> accumulators,
+ MetricGroup metrics) {
+ super(taskInfo, classloader, executionConfig, cpTasks, accumulators, metrics);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/Counter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/Counter.java b/flink-core/src/main/java/org/apache/flink/metrics/Counter.java
new file mode 100644
index 0000000..b18da4f
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/Counter.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * A Counter is a {@link org.apache.flink.metrics.Metric} that measures a count.
+ */
+@PublicEvolving
+public final class Counter implements Metric {
+ private long count = 0;
+
+ /**
+ * Increment the current count by 1.
+ */
+ public void inc() {
+ count++;
+ }
+
+ /**
+ * Increment the current count by the given value.
+ *
+ * @param n value to increment the current count by
+ */
+ public void inc(long n) {
+ count += n;
+ }
+
+ /**
+ * Decrement the current count by 1.
+ */
+ public void dec() {
+ count--;
+ }
+
+ /**
+ * Decrement the current count by the given value.
+ *
+ * @param n value to decrement the current count by
+ */
+ public void dec(long n) {
+ count -= n;
+ }
+
+ /**
+ * Returns the current count.
+ *
+ * @return current count
+ */
+ public long getCount() {
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java b/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java
new file mode 100644
index 0000000..455587a
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * A Gauge is a {@link org.apache.flink.metrics.Metric} that calculates a specific value at a point in time.
+ */
+@PublicEvolving
+public abstract class Gauge<T> implements Metric {
+ /**
+ * Calculates and returns the measured value.
+ *
+ * @return calculated value
+ */
+ public abstract T getValue();
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/Metric.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/Metric.java b/flink-core/src/main/java/org/apache/flink/metrics/Metric.java
new file mode 100644
index 0000000..11cfcc6
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/Metric.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * Common interface for all metrics.
+ */
+@PublicEvolving
+public interface Metric {
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java
new file mode 100644
index 0000000..4036129
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * A MetricGroup is a named container for {@link org.apache.flink.metrics.Metric}s and
+ * {@link org.apache.flink.metrics.MetricGroup}s.
+ * <p>
+ * Instances of this class can be used to register new metrics with Flink and to create a nested hierarchy based on the
+ * group names.
+ * <p>
+ * A MetricGroup is uniquely identified by it's place in the hierarchy and name.
+ */
+@PublicEvolving
+public interface MetricGroup {
+
+ /**
+ * Recursively unregisters all {@link org.apache.flink.metrics.Metric}s contained in this
+ * {@link org.apache.flink.metrics.MetricGroup}
+ */
+ void close();
+
+ // -----------------------------------------------------------------------------------------------------------------
+ // Metrics
+ // -----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Creates and registers a new {@link org.apache.flink.metrics.Counter} with Flink.
+ *
+ * @param name name of the counter
+ * @return the registered counter
+ */
+ Counter counter(int name);
+
+ /**
+ * Creates and registers a new {@link org.apache.flink.metrics.Counter} with Flink.
+ *
+ * @param name name of the counter
+ * @return the registered counter
+ */
+ Counter counter(String name);
+
+ /**
+ * Registers a new {@link org.apache.flink.metrics.Gauge} with Flink.
+ *
+ * @param name name of the gauge
+ * @param gauge gauge to register
+ * @param <T> return type of the gauge
+ * @return the registered gauge
+ */
+ <T> Gauge<T> gauge(int name, Gauge<T> gauge);
+
+ /**
+ * Registers a new {@link org.apache.flink.metrics.Gauge} with Flink.
+ *
+ * @param name name of the gauge
+ * @param gauge gauge to register
+ * @param <T> return type of the gauge
+ * @return the registered gauge
+ */
+ <T> Gauge<T> gauge(String name, Gauge<T> gauge);
+
+ // -----------------------------------------------------------------------------------------------------------------
+ // Groups
+ // -----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Creates a new {@link org.apache.flink.metrics.MetricGroup} and adds it to this groups sub-groups.
+ *
+ * @param name name of the group
+ * @return the created group
+ */
+ MetricGroup addGroup(int name);
+
+ /**
+ * Creates a new {@link org.apache.flink.metrics.MetricGroup} and adds it to this groups sub-groups.
+ *
+ * @param name name of the group
+ * @return the created group
+ */
+ MetricGroup addGroup(String name);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java b/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java
new file mode 100644
index 0000000..0e8b0d5
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.AbstractMetricGroup;
+import org.apache.flink.metrics.groups.Scope;
+import org.apache.flink.metrics.reporter.JMXReporter;
+import org.apache.flink.metrics.reporter.MetricReporter;
+import org.apache.flink.metrics.reporter.Scheduled;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.TimerTask;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.flink.metrics.groups.JobMetricGroup.DEFAULT_SCOPE_JOB;
+import static org.apache.flink.metrics.groups.OperatorMetricGroup.DEFAULT_SCOPE_OPERATOR;
+import static org.apache.flink.metrics.groups.TaskManagerMetricGroup.DEFAULT_SCOPE_TM;
+import static org.apache.flink.metrics.groups.TaskMetricGroup.DEFAULT_SCOPE_TASK;
+
+/**
+ * A MetricRegistry keeps track of all registered {@link org.apache.flink.metrics.Metric}s. It serves as the
+ * connection between {@link org.apache.flink.metrics.MetricGroup}s and {@link org.apache.flink.metrics.reporter.MetricReporter}s.
+ */
+@Internal
+public class MetricRegistry {
+ private static final Logger LOG = LoggerFactory.getLogger(MetricRegistry.class);
+
+ private final MetricReporter reporter;
+ private java.util.Timer timer;
+
+ public static final String KEY_METRICS_REPORTER_CLASS = "metrics.reporter.class";
+ public static final String KEY_METRICS_REPORTER_ARGUMENTS = "metrics.reporter.arguments";
+ public static final String KEY_METRICS_REPORTER_INTERVAL = "metrics.reporter.interval";
+
+ public static final String KEY_METRICS_SCOPE_NAMING_TM = "metrics.scope.tm";
+ public static final String KEY_METRICS_SCOPE_NAMING_JOB = "metrics.scope.job";
+ public static final String KEY_METRICS_SCOPE_NAMING_TASK = "metrics.scope.task";
+ public static final String KEY_METRICS_SCOPE_NAMING_OPERATOR = "metrics.scope.operator";
+
+ private final Scope.ScopeFormat scopeConfig;
+
+ /**
+ * Creates a new {@link MetricRegistry} and starts the configured reporter.
+ */
+ public MetricRegistry(Configuration config) {
+ try {
+ String className = config.getString(KEY_METRICS_REPORTER_CLASS, null);
+ if (className == null) {
+ LOG.info("No reporter class name defined in flink-conf.yaml, defaulting to " + JMXReporter.class.getName() + ".");
+ className = JMXReporter.class.getName();
+ }
+
+ this.scopeConfig = createScopeConfig(config);
+
+ Configuration reporterConfig = createReporterConfig(config);
+ Class<?> reporterClass = Class.forName(className);
+ reporter = (MetricReporter) reporterClass.newInstance();
+ reporter.open(reporterConfig);
+
+ if (reporter instanceof Scheduled) {
+ String[] interval = config.getString(KEY_METRICS_REPORTER_INTERVAL, "10 SECONDS").split(" ");
+ long millis = TimeUnit.valueOf(interval[1]).toMillis(Long.parseLong(interval[0]));
+ timer = new java.util.Timer(true);
+ timer.schedule(new TimerTask() {
+ @Override
+ public void run() {
+ ((Scheduled) reporter).report();
+ }
+ }, millis, millis);
+ }
+ } catch (InstantiationException | ClassNotFoundException e) {
+ throw new RuntimeException("Error while instantiating reporter.", e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Implementation error.", e);
+ }
+ }
+
+ private static Configuration createReporterConfig(Configuration config) {
+ String[] interval = config.getString(KEY_METRICS_REPORTER_INTERVAL, "10 SECONDS").split(" ");
+
+ String[] arguments = config.getString(KEY_METRICS_REPORTER_ARGUMENTS, "").split(" ");
+
+ Configuration reporterConfig = new Configuration();
+ reporterConfig.setString("period", interval[0]);
+ reporterConfig.setString("timeunit", interval[1]);
+
+ if (arguments.length > 1) {
+ for (int x = 0; x < arguments.length; x += 2) {
+ reporterConfig.setString(arguments[x].replace("--", ""), arguments[x + 1]);
+ }
+ }
+ return reporterConfig;
+ }
+
+ private static Scope.ScopeFormat createScopeConfig(Configuration config) {
+ String tmFormat = config.getString(KEY_METRICS_SCOPE_NAMING_TM, DEFAULT_SCOPE_TM);
+ String jobFormat = config.getString(KEY_METRICS_SCOPE_NAMING_JOB, DEFAULT_SCOPE_JOB);
+ String taskFormat = config.getString(KEY_METRICS_SCOPE_NAMING_TASK, DEFAULT_SCOPE_TASK);
+ String operatorFormat = config.getString(KEY_METRICS_SCOPE_NAMING_OPERATOR, DEFAULT_SCOPE_OPERATOR);
+
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setTaskManagerFormat(tmFormat);
+ format.setJobFormat(jobFormat);
+ format.setTaskFormat(taskFormat);
+ format.setOperatorFormat(operatorFormat);
+ return format;
+ }
+
+ public Scope.ScopeFormat getScopeConfig() {
+ return this.scopeConfig;
+ }
+
+ /**
+ * Shuts down this registry and the associated {@link org.apache.flink.metrics.reporter.MetricReporter}.
+ */
+ public void shutdown() {
+ if (timer != null) {
+ timer.cancel();
+ }
+ if (reporter != null) {
+ reporter.close();
+ }
+ }
+
+ /**
+ * Registers a new {@link org.apache.flink.metrics.Metric} with this registry.
+ *
+ * @param metric metric to register
+ * @param name name of the metric
+ * @param parent group that contains the metric
+ */
+ public void register(Metric metric, String name, AbstractMetricGroup parent) {
+ String metricName = reporter.generateName(name, parent.generateScope());
+
+
+ this.reporter.notifyOfAddedMetric(metric, metricName);
+ }
+
+ /**
+ * Un-registers the given {@link org.apache.flink.metrics.Metric} with this registry.
+ *
+ * @param metric metric to un-register
+ * @param name name of the metric
+ * @param parent group that contains the metric
+ */
+ public void unregister(Metric metric, String name, AbstractMetricGroup parent) {
+ String metricName = reporter.generateName(name, parent.generateScope());
+
+ this.reporter.notifyOfRemovedMetric(metric, metricName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java
new file mode 100644
index 0000000..373ac09
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract {@link org.apache.flink.metrics.MetricGroup} that contains key functionality for adding metrics and groups.
+ */
+@Internal
+public abstract class AbstractMetricGroup implements MetricGroup {
+ private static final Logger LOG = LoggerFactory.getLogger(MetricGroup.class);
+ protected final MetricRegistry registry;
+
+ // all metrics that are directly contained in this group
+ protected final Map<String, Metric> metrics = new HashMap<>();
+ // all generic groups that are directly contained in this group
+ protected final Map<String, MetricGroup> groups = new HashMap<>();
+
+ public AbstractMetricGroup(MetricRegistry registry) {
+ this.registry = registry;
+ }
+
+ @Override
+ public void close() {
+ for (MetricGroup group : groups.values()) {
+ group.close();
+ }
+ this.groups.clear();
+ for (Map.Entry<String, Metric> metric : metrics.entrySet()) {
+ registry.unregister(metric.getValue(), metric.getKey(), this);
+ }
+ this.metrics.clear();
+ }
+
+ // -----------------------------------------------------------------------------------------------------------------
+ // Scope
+ // -----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Generates the full scope based on the default/configured format that applies to all metrics within this group.
+ *
+ * @return generated scope
+ */
+ public abstract List<String> generateScope();
+
+ /**
+ * Generates the full scope based on the given format that applies to all metrics within this group.
+ *
+ * @param format format string
+ * @return generated scope
+ */
+ public abstract List<String> generateScope(Scope.ScopeFormat format);
+
+ // -----------------------------------------------------------------------------------------------------------------
+ // Metrics
+ // -----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public Counter counter(int name) {
+ return counter("" + name);
+ }
+
+ @Override
+ public Counter counter(String name) {
+ Counter counter = new Counter();
+ addMetric(name, counter);
+ return counter;
+ }
+
+ @Override
+ public <T> Gauge<T> gauge(int name, Gauge<T> gauge) {
+ return gauge("" + name, gauge);
+ }
+
+ @Override
+ public <T> Gauge<T> gauge(String name, Gauge<T> gauge) {
+ addMetric(name, gauge);
+ return gauge;
+ }
+
+ protected MetricGroup addMetric(String name, Metric metric) {
+ if (!name.matches("[a-zA-Z0-9]*")) {
+ throw new IllegalArgumentException("Metric names may not contain special characters.");
+ }
+ if (metrics.containsKey(name)) {
+ LOG.warn("Detected metric name collision. This group already contains a group for the given group name. " +
+ this.generateScope().toString() + "." + name);
+ }
+ if (groups.containsKey(name)) {
+ LOG.warn("Detected metric name collision. This group already contains a group for the given metric name." +
+ this.generateScope().toString() + ")." + name);
+ }
+ metrics.put(name, metric);
+ registry.register(metric, name, this);
+ return this;
+ }
+
+ // -----------------------------------------------------------------------------------------------------------------
+ // Groups
+ // -----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public MetricGroup addGroup(int name) {
+ return addGroup("" + name);
+ }
+
+ @Override
+ public MetricGroup addGroup(String name) {
+ if (metrics.containsKey(name)) {
+ LOG.warn("Detected metric name collision. This group already contains a metric for the given group name."
+ + this.generateScope().toString() + "." + name);
+ }
+ if (!groups.containsKey(name)) {
+ groups.put(name, new GenericMetricGroup(registry, this, name));
+ }
+ return groups.get(name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java
new file mode 100644
index 0000000..81851e2
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.MetricRegistry;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.flink.metrics.groups.Scope.SCOPE_WILDCARD;
+
+/**
+ * Abstract {@link org.apache.flink.metrics.MetricGroup} that contains key functionality for modifying the scope.
+ */
+@Internal
+public abstract class ComponentMetricGroup extends AbstractMetricGroup {
+ private final ComponentMetricGroup parent;
+ private final String format;
+
+
+ // Map: scope variable -> specific value
+ protected final Map<String, String> formats;
+
+ /**
+ * Creates a new ComponentMetricGroup.
+ *
+ * @param registry registry to register new metrics with
+ * @param parentGroup parent group, may be null
+ * @param scopeFormat default format string
+ */
+ public ComponentMetricGroup(MetricRegistry registry, ComponentMetricGroup parentGroup, String scopeFormat) {
+ super(registry);
+ this.formats = new HashMap<>();
+ this.parent = parentGroup;
+ this.format = scopeFormat;
+ }
+
+ @Override
+ public List<String> generateScope() {
+ return this.generateScope(this.format);
+ }
+
+ @Override
+ public List<String> generateScope(Scope.ScopeFormat format) {
+ return generateScope(getScopeFormat(format));
+ }
+
+ protected abstract String getScopeFormat(Scope.ScopeFormat format);
+
+ private List<String> generateScope(String format) {
+ String[] components = Scope.split(format);
+
+ List<String> scope = new ArrayList<>();
+ if (components[0].equals(SCOPE_WILDCARD)) {
+ if (this.parent != null) {
+ scope = this.parent.generateScope();
+ }
+ this.replaceFormats(components);
+ addToList(scope, components, 1);
+ } else {
+ if (this.parent != null) {
+ this.parent.replaceFormats(components);
+ }
+ this.replaceFormats(components);
+ addToList(scope, components, 0);
+ }
+ return scope;
+ }
+
+ private void replaceFormats(String[] components) {
+ if (this.parent != null) {
+ this.parent.replaceFormats(components);
+ }
+ for (int x = 0; x < components.length; x++) {
+ if (components[x].startsWith("<")) {
+ if (this.formats.containsKey(components[x])) {
+ components[x] = this.formats.get(components[x]);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds all elements from the given array, starting from the given index, to the given list.
+ *
+ * @param list destination
+ * @param array source
+ * @param startIndex array index to start from
+ */
+ private static void addToList(List<String> list, String[] array, int startIndex) {
+ for (int x = startIndex; x < array.length; x++) {
+ list.add(array[x]);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java
new file mode 100644
index 0000000..5886312
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.MetricRegistry;
+
+import java.util.List;
+
+/**
+ * A simple named {@link org.apache.flink.metrics.MetricGroup} with no special properties.
+ */
+@Internal
+public class GenericMetricGroup extends AbstractMetricGroup {
+ private final AbstractMetricGroup parent;
+
+ private final String name;
+
+ protected GenericMetricGroup(MetricRegistry registry, AbstractMetricGroup parent, int name) {
+ this(registry, parent, "" + name);
+ }
+
+ protected GenericMetricGroup(MetricRegistry registry, AbstractMetricGroup parent, String name) {
+ super(registry);
+ this.parent = parent;
+ this.name = name;
+ }
+
+ @Override
+ public List<String> generateScope() {
+ List<String> scope = parent.generateScope();
+ scope.add(name);
+ return scope;
+ }
+
+ @Override
+ public List<String> generateScope(Scope.ScopeFormat format) {
+ List<String> scope = parent.generateScope(format);
+ scope.add(name);
+ return scope;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java
new file mode 100644
index 0000000..68d91c4
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.MetricRegistry;
+
+import java.util.List;
+
+/**
+ * Special {@link org.apache.flink.metrics.MetricGroup} that contains shareable pre-defined IO-related metrics.
+ */
+public class IOMetricGroup extends AbstractMetricGroup {
+ private final TaskMetricGroup parent;
+
+ private transient final Counter numBytesIn;
+ private transient final Counter numBytesOut;
+ private transient final Counter numRecordsIn;
+ private transient final Counter numRecordsOut;
+
+ public IOMetricGroup(MetricRegistry registry, TaskMetricGroup parent) {
+ super(registry);
+ this.parent = parent;
+ this.numBytesIn = parent.counter("numBytesIn");
+ this.numBytesOut = parent.counter("numBytesOut");
+ this.numRecordsIn = parent.counter("numRecordsIn");
+ this.numRecordsOut = parent.counter("numRecordsOut");
+ }
+
+ @Override
+ public List<String> generateScope() {
+ return parent.generateScope();
+ }
+
+ @Override
+ public List<String> generateScope(Scope.ScopeFormat format) {
+ return parent.generateScope(format);
+ }
+
+ public Counter getBytesInCounter() {
+ return this.numBytesIn;
+ }
+
+ public Counter getBytesOutCounter() {
+ return this.numBytesOut;
+ }
+
+ public Counter getRecordsInCounter() {
+ return this.numRecordsIn;
+ }
+
+ public Counter getRecordsOutCounter() {
+ return this.numRecordsOut;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java
new file mode 100644
index 0000000..35a01f8
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.util.AbstractID;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.flink.metrics.groups.TaskManagerMetricGroup.DEFAULT_SCOPE_TM;
+
+/**
+ * Special {@link org.apache.flink.metrics.MetricGroup} representing a Job.
+ * <p<
+ * Contains extra logic for adding tasks.
+ */
+@Internal
+public class JobMetricGroup extends ComponentMetricGroup {
+ public static final String SCOPE_JOB_DESCRIPTOR = "job";
+ public static final String SCOPE_JOB_ID = Scope.format("job_id");
+ public static final String SCOPE_JOB_NAME = Scope.format("job_name");
+ public static final String DEFAULT_SCOPE_JOB_COMPONENT = Scope.concat(SCOPE_JOB_NAME);
+ public static final String DEFAULT_SCOPE_JOB = Scope.concat(DEFAULT_SCOPE_TM, DEFAULT_SCOPE_JOB_COMPONENT);
+
+ private Map<AbstractID, TaskMetricGroup> tasks = new HashMap<>();
+
+ public JobMetricGroup(MetricRegistry registry, TaskManagerMetricGroup taskManager, JobID id, String name) {
+ super(registry, taskManager, registry.getScopeConfig().getJobFormat());
+ this.formats.put(SCOPE_JOB_ID, id.toString());
+ this.formats.put(SCOPE_JOB_NAME, name);
+ }
+
+ public TaskMetricGroup addTask(AbstractID id, AbstractID attemptID, int subtaskIndex, String name) {
+ TaskMetricGroup task = new TaskMetricGroup(this.registry, this, id, attemptID, subtaskIndex, name);
+ tasks.put(id, task);
+ return task;
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ for (MetricGroup group : tasks.values()) {
+ group.close();
+ }
+ tasks.clear();
+ }
+
+ @Override
+ protected String getScopeFormat(Scope.ScopeFormat format) {
+ return format.getJobFormat();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java
new file mode 100644
index 0000000..6475eec
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.MetricRegistry;
+
+import static org.apache.flink.metrics.groups.JobMetricGroup.DEFAULT_SCOPE_JOB;
+
+/**
+ * Special {@link org.apache.flink.metrics.MetricGroup} representing an Operator.
+ */
+@Internal
+public class OperatorMetricGroup extends ComponentMetricGroup {
+ public static final String SCOPE_OPERATOR_DESCRIPTOR = "operator";
+ public static final String SCOPE_OPERATOR_NAME = Scope.format("operator_name");
+ public static final String SCOPE_OPERATOR_SUBTASK_INDEX = Scope.format("subtask_index");
+ public static final String DEFAULT_SCOPE_OPERATOR_COMPONENT = Scope.concat(SCOPE_OPERATOR_NAME, SCOPE_OPERATOR_SUBTASK_INDEX);
+ public static final String DEFAULT_SCOPE_OPERATOR = Scope.concat(DEFAULT_SCOPE_JOB, DEFAULT_SCOPE_OPERATOR_COMPONENT);
+
+ protected OperatorMetricGroup(MetricRegistry registry, TaskMetricGroup task, String name, int subTaskIndex) {
+ super(registry, task, registry.getScopeConfig().getOperatorFormat());
+ this.formats.put(SCOPE_OPERATOR_NAME, name);
+ this.formats.put(SCOPE_OPERATOR_SUBTASK_INDEX, "" + subTaskIndex);
+ }
+
+ @Override
+ protected String getScopeFormat(Scope.ScopeFormat format) {
+ return format.getOperatorFormat();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java
new file mode 100644
index 0000000..47bae37
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * This class provides utility-functions for handling scopes.
+ */
+@Internal
+public class Scope {
+ public static final String SCOPE_WILDCARD = "*";
+
+ private static final String SCOPE_PREFIX = "<";
+ private static final String SCOPE_SUFFIX = ">";
+ private static final String SCOPE_SPLIT = ".";
+
+ private Scope() {
+ }
+
+ /**
+ * Modifies the given string to resemble a scope variable.
+ *
+ * @param scope string to format
+ * @return formatted string
+ */
+ public static String format(String scope) {
+ return SCOPE_PREFIX + scope + SCOPE_SUFFIX;
+ }
+
+ /**
+ * Joins the given components into a single scope.
+ *
+ * @param components components to join
+ * @return joined scoped
+ */
+ public static String concat(String... components) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(components[0]);
+ for (int x = 1; x < components.length; x++) {
+ sb.append(SCOPE_SPLIT);
+ sb.append(components[x]);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Splits the given scope into it's individual components.
+ *
+ * @param scope scope to split
+ * @return array of components
+ */
+ public static String[] split(String scope) {
+ return scope.split("\\" + SCOPE_SPLIT);
+ }
+
+ /**
+ * Simple container for component scope format strings.
+ */
+ public static class ScopeFormat {
+ private String operatorFormat = OperatorMetricGroup.DEFAULT_SCOPE_OPERATOR;
+ private String taskFormat = TaskMetricGroup.DEFAULT_SCOPE_TASK;
+ private String jobFormat = JobMetricGroup.DEFAULT_SCOPE_JOB;
+ private String taskManagerFormat = TaskManagerMetricGroup.DEFAULT_SCOPE_TM;
+
+ public ScopeFormat() {
+ }
+
+ public ScopeFormat setOperatorFormat(String format) {
+ this.operatorFormat = format;
+ return this;
+ }
+
+ public ScopeFormat setTaskFormat(String format) {
+ this.taskFormat = format;
+ return this;
+ }
+
+ public ScopeFormat setJobFormat(String format) {
+ this.jobFormat = format;
+ return this;
+ }
+
+ public ScopeFormat setTaskManagerFormat(String format) {
+ this.taskManagerFormat = format;
+ return this;
+ }
+
+ public String getOperatorFormat() {
+ return this.operatorFormat;
+ }
+
+ public String getTaskFormat() {
+ return this.taskFormat;
+ }
+
+ public String getJobFormat() {
+ return this.jobFormat;
+ }
+
+ public String getTaskManagerFormat() {
+ return this.taskManagerFormat;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java
new file mode 100644
index 0000000..e199ca7
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Special {@link org.apache.flink.metrics.MetricGroup} representing a TaskManager.
+ * <p<
+ * Contains extra logic for adding jobs.
+ */
+@Internal
+public class TaskManagerMetricGroup extends ComponentMetricGroup {
+ public static final String SCOPE_HOST_DESCRIPTOR = "host";
+ public static final String SCOPE_TM_DESCRIPTOR = "taskmanager";
+ public static final String SCOPE_TM_HOST = Scope.format("host");
+ public static final String SCOPE_TM_ID = Scope.format("tm_id");
+ public static final String DEFAULT_SCOPE_TM_COMPONENT = Scope.concat(SCOPE_TM_HOST, "taskmanager", SCOPE_TM_ID);
+ public static final String DEFAULT_SCOPE_TM = DEFAULT_SCOPE_TM_COMPONENT;
+
+ private Map<JobID, JobMetricGroup> jobs = new HashMap<>();
+
+ public TaskManagerMetricGroup(MetricRegistry registry, String host, String id) {
+ super(registry, null, registry.getScopeConfig().getTaskManagerFormat());
+ this.formats.put(SCOPE_TM_HOST, host);
+ this.formats.put(SCOPE_TM_ID, id);
+ }
+
+ public JobMetricGroup addJob(JobID id, String name) {
+ JobMetricGroup task = new JobMetricGroup(this.registry, this, id, name);
+ jobs.put(id, task);
+ return task;
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ for (MetricGroup group : jobs.values()) {
+ group.close();
+ }
+ jobs.clear();
+ }
+
+ @Override
+ protected String getScopeFormat(Scope.ScopeFormat format) {
+ return format.getTaskManagerFormat();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java
new file mode 100644
index 0000000..4f8e010
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.util.AbstractID;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.flink.metrics.groups.JobMetricGroup.DEFAULT_SCOPE_JOB;
+
+/**
+ * Special {@link org.apache.flink.metrics.MetricGroup} representing a Task.
+ * <p<
+ * Contains extra logic for adding operators.
+ */
+@Internal
+public class TaskMetricGroup extends ComponentMetricGroup {
+ public static final String SCOPE_TASK_DESCRIPTOR = "task";
+ public static final String SCOPE_TASK_ID = Scope.format("task_id");
+ public static final String SCOPE_TASK_NAME = Scope.format("task_name");
+ public static final String SCOPE_TASK_ATTEMPT = Scope.format("task_attempt");
+ public static final String SCOPE_TASK_SUBTASK_INDEX = Scope.format("subtask_index");
+ public static final String DEFAULT_SCOPE_TASK_COMPONENT = SCOPE_TASK_NAME;
+ public static final String DEFAULT_SCOPE_TASK = Scope.concat(DEFAULT_SCOPE_JOB, DEFAULT_SCOPE_TASK_COMPONENT);
+ private final int subtaskIndex;
+
+ private Map<String, OperatorMetricGroup> operators = new HashMap<>();
+ private IOMetricGroup ioMetrics;
+
+ protected TaskMetricGroup(MetricRegistry registry, JobMetricGroup job, AbstractID id, AbstractID attemptID, int subtaskIndex, String name) {
+ super(registry, job, registry.getScopeConfig().getTaskFormat());
+ this.formats.put(SCOPE_TASK_ID, id.toString());
+ this.formats.put(SCOPE_TASK_ATTEMPT, attemptID.toString());
+ this.formats.put(SCOPE_TASK_NAME, name);
+ this.formats.put(SCOPE_TASK_SUBTASK_INDEX, "" + subtaskIndex);
+ this.subtaskIndex = subtaskIndex;
+ this.ioMetrics = new IOMetricGroup(registry, this);
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ for (MetricGroup group : operators.values()) {
+ group.close();
+ }
+ operators.clear();
+ }
+
+ public OperatorMetricGroup addOperator(String name) {
+ OperatorMetricGroup operator = new OperatorMetricGroup(this.registry, this, name, this.subtaskIndex);
+ operators.put(name, operator);
+ return operator;
+ }
+
+ /**
+ * Returns the IOMetricGroup for this task.
+ *
+ * @return IOMetricGroup for this task.
+ */
+ public IOMetricGroup getIOMetricGroup() {
+ return this.ioMetrics;
+ }
+
+ @Override
+ protected String getScopeFormat(Scope.ScopeFormat format) {
+ return format.getTaskFormat();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java b/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java
new file mode 100644
index 0000000..43f09b2
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.reporter;
+
+import org.apache.flink.hadoop.shaded.org.jboss.netty.util.internal.ConcurrentHashMap;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+
+import java.util.Map;
+
+public abstract class AbstractReporter implements MetricReporter {
+ protected Map<String, Gauge> gauges = new ConcurrentHashMap<>();
+ protected Map<String, Counter> counters = new ConcurrentHashMap<>();
+
+ @Override
+ public void notifyOfAddedMetric(Metric metric, String name) {
+ if (metric instanceof Counter) {
+ counters.put(name, (Counter) metric);
+ } else if (metric instanceof Gauge) {
+ gauges.put(name, (Gauge) metric);
+ }
+ }
+
+ @Override
+ public void notifyOfRemovedMetric(Metric metric, String name) {
+ if (metric instanceof Counter) {
+ counters.remove(name);
+ } else if (metric instanceof Gauge) {
+ gauges.remove(name);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java b/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java
new file mode 100644
index 0000000..0b2efe4
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.reporter;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.management.InstanceAlreadyExistsException;
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanRegistrationException;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+
+/**
+ * {@link org.apache.flink.metrics.reporter.MetricReporter} that exports {@link org.apache.flink.metrics.Metric}s via JMX.
+ *
+ * Largely based on the JmxReporter class of the dropwizard metrics library
+ * https://github.com/dropwizard/metrics/blob/master/metrics-core/src/main/java/io/dropwizard/metrics/JmxReporter.java
+ */
+@Internal
+public class JMXReporter implements MetricReporter {
+ private static final Logger LOG = LoggerFactory.getLogger(JMXReporter.class);
+
+ private MBeanServer mBeanServer;
+
+ private static final String PREFIX = "org.apache.flink.metrics:";
+ private static final String KEY_PREFIX = "key";
+
+ public JMXReporter() {
+ this.mBeanServer = ManagementFactory.getPlatformMBeanServer();
+ }
+
+ @Override
+ public void notifyOfAddedMetric(Metric metric, String name) {
+ AbstractBean jmxMetric;
+ ObjectName jmxName;
+ try {
+ jmxName = new ObjectName(name);
+ } catch (MalformedObjectNameException e) {
+ throw new IllegalArgumentException("Metric name did not conform to JMX ObjectName rules: " + name, e);
+ }
+
+ if (metric instanceof Gauge) {
+ jmxMetric = new JmxGauge((Gauge<?>) metric);
+ } else if (metric instanceof Counter) {
+ jmxMetric = new JmxCounter((Counter) metric);
+ } else {
+ throw new IllegalArgumentException("Unknown metric type: " + metric.getClass());
+ }
+
+ try {
+ mBeanServer.registerMBean(jmxMetric, jmxName);
+ } catch (NotCompliantMBeanException e) { //implementation error on our side
+ LOG.error("Metric did not comply with JMX MBean naming rules.", e);
+ } catch (InstanceAlreadyExistsException e) {
+ LOG.error("A metric with the name " + jmxName + " was already registered.", e);
+ } catch (MBeanRegistrationException e) {
+ LOG.error("Failed to register metric.", e);
+ }
+ }
+
+ @Override
+ public void notifyOfRemovedMetric(Metric metric, String name) {
+ try {
+ mBeanServer.unregisterMBean(new ObjectName(name));
+ } catch (MBeanRegistrationException e) {
+ LOG.error("Un-registering metric failed.", e);
+ } catch (MalformedObjectNameException e) {
+ LOG.error("Un-registering metric failed due to invalid name.", e);
+ } catch (InstanceNotFoundException e) {
+ //alright then
+ }
+ }
+
+ @Override
+ public void open(Configuration config) {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public String generateName(String name, List<String> origin) {
+ StringBuilder fullName = new StringBuilder();
+
+ fullName.append(PREFIX);
+ for (int x = 0; x < origin.size(); x++) {
+ fullName.append(KEY_PREFIX);
+ fullName.append(x);
+ fullName.append("=");
+ String value = origin.get(x);
+ value = value.replaceAll("\"", "");
+ value = value.replaceAll(" ", "_");
+ value = value.replaceAll("[,=;:?'*]", "-");
+ fullName.append(value);
+ fullName.append(",");
+ }
+ fullName.append("name=" + name);
+
+ return fullName.toString();
+ }
+
+ public interface MetricMBean {
+ }
+
+ private abstract static class AbstractBean implements MetricMBean {
+ }
+
+ public interface JmxCounterMBean extends MetricMBean {
+ long getCount();
+ }
+
+ private static class JmxCounter extends AbstractBean implements JmxCounterMBean {
+ private Counter counter;
+
+ public JmxCounter(Counter counter) {
+ this.counter = counter;
+ }
+
+ @Override
+ public long getCount() {
+ return counter.getCount();
+ }
+ }
+
+ public interface JmxGaugeMBean extends MetricMBean {
+ Object getValue();
+ }
+
+ private static class JmxGauge extends AbstractBean implements JmxGaugeMBean {
+ private final Gauge gauge;
+
+ public JmxGauge(Gauge gauge) {
+ this.gauge = gauge;
+ }
+
+ @Override
+ public Object getValue() {
+ return gauge.getValue();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java b/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java
new file mode 100644
index 0000000..2bca606
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.reporter;
+
+import com.codahale.metrics.Reporter;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Metric;
+
+import java.util.List;
+
+/**
+ * Reporters are used to export {@link org.apache.flink.metrics.Metric}s to an external backend.
+ * <p>
+ * Reporters are instantiated generically and must have a no-argument constructor.
+ */
+@PublicEvolving
+public interface MetricReporter extends Reporter {
+ /**
+ * Configures this reporter. Since reporters are instantiated generically and hence parameter-less,
+ * this method is the place where the reporters set their basic fields based on configuration values.
+ * <p>
+ * This method is always called first on a newly instantiated reporter.
+ *
+ * @param config The configuration with all parameters.
+ */
+ void open(Configuration config);
+
+ /**
+ * Closes this reporter. Should be used to close channels, streams and release resources.
+ */
+ void close();
+
+ /**
+ * Called when a new {@link org.apache.flink.metrics.Metric} was added.
+ *
+ * @param metric metric that was added
+ * @param name name of the metric
+ */
+ void notifyOfAddedMetric(Metric metric, String name);
+
+ /**
+ * Called when a {@link org.apache.flink.metrics.Metric} was removed.
+ *
+ * @param metric metric that was removed
+ * @param name name of the metric
+ */
+ void notifyOfRemovedMetric(Metric metric, String name);
+
+ /**
+ * Generates the reported name of a metric based on it's hierarchy/scope and associated name.
+ *
+ * @param name name of the metric
+ * @param scope hierarchy/scope of the metric
+ * @return reported name
+ */
+ String generateName(String name, List<String> scope);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/main/java/org/apache/flink/metrics/reporter/Scheduled.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/reporter/Scheduled.java b/flink-core/src/main/java/org/apache/flink/metrics/reporter/Scheduled.java
new file mode 100644
index 0000000..3638f7a
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/reporter/Scheduled.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.reporter;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+/**
+ * Marker interface for reporters that actively send out data periodically.
+ */
+@PublicEvolving
+public interface Scheduled {
+ /**
+ * Report the current measurements.
+ * This method is called in regular intervals
+ */
+ void report();
+}
[5/6] flink git commit: [FLINK-1502] [core] Add basic metric system
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/functions/util/RuntimeUDFContextTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/functions/util/RuntimeUDFContextTest.java b/flink-core/src/test/java/org/apache/flink/api/common/functions/util/RuntimeUDFContextTest.java
index 858bc49..83c88cc 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/functions/util/RuntimeUDFContextTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/functions/util/RuntimeUDFContextTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class RuntimeUDFContextTest {
@Test
public void testBroadcastVariableNotFound() {
try {
- RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(),new HashMap<String, Accumulator<?, ?>>());
+ RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(),new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup());
try {
ctx.getBroadcastVariable("some name");
@@ -71,7 +72,7 @@ public class RuntimeUDFContextTest {
@Test
public void testBroadcastVariableSimple() {
try {
- RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>());
+ RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup());
ctx.setBroadcastVariable("name1", Arrays.asList(1, 2, 3, 4));
ctx.setBroadcastVariable("name2", Arrays.asList(1.0, 2.0, 3.0, 4.0));
@@ -105,7 +106,7 @@ public class RuntimeUDFContextTest {
@Test
public void testBroadcastVariableWithInitializer() {
try {
- RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>());
+ RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup());
ctx.setBroadcastVariable("name", Arrays.asList(1, 2, 3, 4));
@@ -130,7 +131,7 @@ public class RuntimeUDFContextTest {
@Test
public void testResetBroadcastVariableWithInitializer() {
try {
- RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>());
+ RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup());
ctx.setBroadcastVariable("name", Arrays.asList(1, 2, 3, 4));
@@ -153,7 +154,7 @@ public class RuntimeUDFContextTest {
@Test
public void testBroadcastVariableWithInitializerAndMismatch() {
try {
- RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>());
+ RuntimeUDFContext ctx = new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup());
ctx.setBroadcastVariable("name", Arrays.asList(1, 2, 3, 4));
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/io/RichInputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/RichInputFormatTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/RichInputFormatTest.java
index 7ea0071..554820e 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/RichInputFormatTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/RichInputFormatTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.types.Value;
import org.junit.Assert;
import org.junit.Test;
@@ -40,7 +41,7 @@ public class RichInputFormatTest {
public void testCheckRuntimeContextAccess() {
final SerializedInputFormat<Value> inputFormat = new SerializedInputFormat<Value>();
final TaskInfo taskInfo = new TaskInfo("test name", 1, 3, 0);
- inputFormat.setRuntimeContext(new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()));
+ inputFormat.setRuntimeContext(new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()));
Assert.assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
Assert.assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/io/RichOutputFormatTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/io/RichOutputFormatTest.java b/flink-core/src/test/java/org/apache/flink/api/common/io/RichOutputFormatTest.java
index 273f4f5..09db3a9 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/io/RichOutputFormatTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/io/RichOutputFormatTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.types.Value;
import org.junit.Assert;
import org.junit.Test;
@@ -41,7 +42,7 @@ public class RichOutputFormatTest {
public void testCheckRuntimeContextAccess() {
final SerializedOutputFormat<Value> inputFormat = new SerializedOutputFormat<Value>();
final TaskInfo taskInfo = new TaskInfo("test name", 1, 3, 0);
- inputFormat.setRuntimeContext(new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()));
+ inputFormat.setRuntimeContext(new RuntimeUDFContext(taskInfo, getClass().getClassLoader(), new ExecutionConfig(), new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()));
Assert.assertEquals(inputFormat.getRuntimeContext().getIndexOfThisSubtask(), 1);
Assert.assertEquals(inputFormat.getRuntimeContext().getNumberOfParallelSubtasks(),3);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSinkBaseTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSinkBaseTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSinkBaseTest.java
index 5ca4c4c..7c905c1 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSinkBaseTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSinkBaseTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.api.common.operators.util.TestNonRichInputFormat;
import org.apache.flink.api.common.operators.util.TestRichOutputFormat;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.types.Nothing;
import org.junit.Test;
@@ -94,13 +95,13 @@ public class GenericDataSinkBaseTest implements java.io.Serializable {
final TaskInfo taskInfo = new TaskInfo("test_sink", 0, 1, 0);
executionConfig.disableObjectReuse();
in.reset();
- sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
assertEquals(out.output, asList(TestIOData.RICH_NAMES));
executionConfig.enableObjectReuse();
out.clear();
in.reset();
- sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
assertEquals(out.output, asList(TestIOData.RICH_NAMES));
} catch(Exception e){
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSourceBaseTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSourceBaseTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSourceBaseTest.java
index 083039a..c360c62 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSourceBaseTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/GenericDataSourceBaseTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.api.common.operators.util.TestNonRichInputFormat;
import org.apache.flink.api.common.operators.util.TestRichInputFormat;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.junit.Test;
import java.util.HashMap;
@@ -83,7 +84,7 @@ public class GenericDataSourceBaseTest implements java.io.Serializable {
executionConfig.disableObjectReuse();
assertEquals(false, in.hasBeenClosed());
assertEquals(false, in.hasBeenOpened());
- List<String> resultMutableSafe = source.executeOnCollections(new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ List<String> resultMutableSafe = source.executeOnCollections(new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
assertEquals(true, in.hasBeenClosed());
assertEquals(true, in.hasBeenOpened());
@@ -91,7 +92,7 @@ public class GenericDataSourceBaseTest implements java.io.Serializable {
executionConfig.enableObjectReuse();
assertEquals(false, in.hasBeenClosed());
assertEquals(false, in.hasBeenOpened());
- List<String> resultRegular = source.executeOnCollections(new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ List<String> resultRegular = source.executeOnCollections(new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
assertEquals(true, in.hasBeenClosed());
assertEquals(true, in.hasBeenOpened());
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/operators/base/FlatMapOperatorCollectionTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/FlatMapOperatorCollectionTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/FlatMapOperatorCollectionTest.java
index cda3245..9447efd 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/FlatMapOperatorCollectionTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/FlatMapOperatorCollectionTest.java
@@ -29,6 +29,7 @@ import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
@@ -78,7 +79,7 @@ public class FlatMapOperatorCollectionTest implements Serializable {
final TaskInfo taskInfo = new TaskInfo("Test UDF", 0, 4, 0);
// run on collections
final List<String> result = getTestFlatMapOperator(udf)
- .executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ .executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
Assert.assertEquals(input.size(), result.size());
Assert.assertEquals(input, result);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
index d119fe2..a610a4d 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.common.operators.BinaryOperatorInformation;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.util.Collector;
import org.junit.Test;
@@ -125,9 +126,9 @@ public class InnerJoinOperatorBaseTest implements Serializable {
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.disableObjectReuse();
- List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
executionConfig.enableObjectReuse();
- List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
assertEquals(expected, resultSafe);
assertEquals(expected, resultRegular);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/operators/base/MapOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/MapOperatorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/MapOperatorTest.java
index 6059ab1..7ecdefa 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/MapOperatorTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/MapOperatorTest.java
@@ -39,6 +39,7 @@ import org.apache.flink.api.common.operators.base.MapOperatorBase;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.junit.Test;
@SuppressWarnings("serial")
@@ -112,9 +113,9 @@ public class MapOperatorTest implements java.io.Serializable {
final TaskInfo taskInfo = new TaskInfo(taskName, 0, 1, 0);
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.disableObjectReuse();
- List<Integer> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ List<Integer> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
executionConfig.enableObjectReuse();
- List<Integer> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap), executionConfig);
+ List<Integer> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulatorMap, new DummyMetricGroup()), executionConfig);
assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/api/common/operators/base/PartitionMapOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/PartitionMapOperatorTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/PartitionMapOperatorTest.java
index 71486a5..5012718 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/base/PartitionMapOperatorTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/base/PartitionMapOperatorTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.util.Collector;
import org.apache.flink.api.common.functions.MapPartitionFunction;
import org.apache.flink.api.common.functions.RichMapPartitionFunction;
@@ -85,9 +86,9 @@ public class PartitionMapOperatorTest implements java.io.Serializable {
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.disableObjectReuse();
- List<Integer> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Integer> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
executionConfig.enableObjectReuse();
- List<Integer> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Integer> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java
new file mode 100644
index 0000000..f7502e5
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.TaskManagerMetricGroup;
+import org.apache.flink.metrics.util.TestReporter;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MetricGroupTest {
+ /**
+ * Verifies that group methods instantiate the correct metric with the given name.
+ */
+ @Test
+ public void testMetricInstantiation() {
+ Configuration config = new Configuration();
+
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter1.class.getName());
+
+ MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
+
+ root.counter("counter");
+ Assert.assertTrue(TestReporter1.lastPassedMetric instanceof Counter);
+ Assert.assertEquals("counter", TestReporter1.lastPassedName);
+
+ root.gauge("gauge", new Gauge<Object>() {
+ @Override
+ public Object getValue() {
+ return null;
+ }
+ });
+ Assert.assertTrue(TestReporter1.lastPassedMetric instanceof Gauge);
+ Assert.assertEquals("gauge", TestReporter1.lastPassedName);
+ }
+
+ protected static class TestReporter1 extends TestReporter {
+ public static Metric lastPassedMetric;
+ public static String lastPassedName;
+
+ @Override
+ public void notifyOfAddedMetric(Metric metric, String name) {
+ lastPassedMetric = metric;
+ lastPassedName = name;
+ }
+
+ @Override
+ public void notifyOfRemovedMetric(Metric metric, String name) {
+ }
+ }
+
+ /**
+ * Verifies that metric names containing special characters are rejected.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidMetricName() {
+ Configuration config = new Configuration();
+
+ MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
+ root.counter("=)(/!");
+ }
+
+ /**
+ * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
+ */
+ @Test
+ public void testDuplicateGroupName() {
+ Configuration config = new Configuration();
+
+ MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
+
+ MetricGroup group1 = root.addGroup("group");
+ MetricGroup group2 = root.addGroup("group");
+ MetricGroup group3 = root.addGroup("group");
+ Assert.assertTrue(group1 == group2 && group2 == group3);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/MetricRegistryTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/MetricRegistryTest.java b/flink-core/src/test/java/org/apache/flink/metrics/MetricRegistryTest.java
new file mode 100644
index 0000000..32cc11c
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/MetricRegistryTest.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.JobMetricGroup;
+import org.apache.flink.metrics.groups.OperatorMetricGroup;
+import org.apache.flink.metrics.groups.Scope;
+import org.apache.flink.metrics.groups.TaskManagerMetricGroup;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
+import org.apache.flink.metrics.reporter.Scheduled;
+import org.apache.flink.metrics.util.TestReporter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+public class MetricRegistryTest {
+ /**
+ * Verifies that the reporter class argument is correctly used to instantiate and open the reporter.
+ */
+ @Test
+ public void testReporterInstantiation() {
+ Configuration config = new Configuration();
+
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter1.class.getName());
+
+ new MetricRegistry(config);
+
+ Assert.assertTrue(TestReporter1.wasOpened);
+ }
+
+ protected static class TestReporter1 extends TestReporter {
+ public static boolean wasOpened = false;
+
+ @Override
+ public void open(Configuration config) {
+ wasOpened = true;
+ }
+ }
+
+ /**
+ * Verifies that configured arguments are properly forwarded to the reporter.
+ */
+ @Test
+ public void testReporterArgumentForwarding() {
+ Configuration config = new Configuration();
+
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter2.class.getName());
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_ARGUMENTS, "--arg1 hello --arg2 world");
+
+ new MetricRegistry(config);
+ }
+
+ protected static class TestReporter2 extends TestReporter {
+ @Override
+ public void open(Configuration config) {
+ Assert.assertEquals("hello", config.getString("arg1", null));
+ Assert.assertEquals("world", config.getString("arg2", null));
+ }
+ }
+
+ /**
+ * Verifies that reporters implementing the Scheduled interface are regularly called to report the metrics.
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testReporterScheduling() throws InterruptedException {
+ Configuration config = new Configuration();
+
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter3.class.getName());
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_INTERVAL, "50 MILLISECONDS");
+
+ new MetricRegistry(config);
+
+ long start = System.currentTimeMillis();
+ for (int x = 0; x < 10; x++) {
+ Thread.sleep(100);
+ int reportCount = TestReporter3.reportCount;
+ long curT = System.currentTimeMillis();
+ /**
+ * Within a given time-frame T only T/500 reports may be triggered due to the interval between reports.
+ * This value however does not not take the first triggered report into account (=> +1).
+ * Furthermore we have to account for the mis-alignment between reports being triggered and our time
+ * measurement (=> +1); for T=200 a total of 4-6 reports may have been
+ * triggered depending on whether the end of the interval for the first reports ends before
+ * or after T=50.
+ */
+ long maxAllowedReports = (curT - start) / 50 + 2;
+ Assert.assertTrue("Too many report were triggered.", maxAllowedReports >= reportCount);
+ }
+ Assert.assertTrue("No report was triggered.", TestReporter3.reportCount > 0);
+ }
+
+ protected static class TestReporter3 extends TestReporter implements Scheduled {
+ public static int reportCount = 0;
+
+ @Override
+ public void report() {
+ reportCount++;
+ }
+ }
+
+ /**
+ * Verifies that groups are correctly created, nesting works, and names are properly forwarded to generate names.
+ */
+ @Test
+ public void testMetricGroupGeneration() {
+ Configuration config = new Configuration();
+
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter4.class.getName());
+
+ MetricRegistry registry = new MetricRegistry(config);
+
+ MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");
+ root.counter("rootCounter");
+ root.addGroup("top").counter("topCounter");
+ }
+
+ protected static class TestReporter4 extends TestReporter {
+ @Override
+ public String generateName(String name, List<String> scope) {
+ if (name.compareTo("rootCounter") == 0) {
+ Assert.assertEquals("host", scope.get(0));
+ return "success";
+ } else if (name.compareTo("topCounter") == 0) {
+ Assert.assertEquals("host", scope.get(0));
+ Assert.assertEquals("taskmanager", scope.get(1));
+ return "success";
+ } else {
+ Assert.fail();
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Verifies that reporters implementing the Listener interface are notified when Metrics are added or removed.
+ */
+ @Test
+ public void testListener() {
+ Configuration config = new Configuration();
+
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter6.class.getName());
+
+ MetricRegistry registry = new MetricRegistry(config);
+
+ MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");
+ root.counter("rootCounter");
+ root.close();
+
+ Assert.assertTrue(TestReporter6.addCalled);
+ Assert.assertTrue(TestReporter6.removeCalled);
+ }
+
+ protected static class TestReporter6 extends TestReporter {
+ public static boolean addCalled = false;
+ public static boolean removeCalled = false;
+
+ @Override
+ public void notifyOfAddedMetric(Metric metric, String name) {
+ addCalled = true;
+ Assert.assertTrue(metric instanceof Counter);
+ Assert.assertEquals("rootCounter", name);
+ }
+
+ @Override
+ public void notifyOfRemovedMetric(Metric metric, String name) {
+ removeCalled = true;
+ Assert.assertTrue(metric instanceof Counter);
+ Assert.assertEquals("rootCounter", name);
+ }
+ }
+
+ /**
+ * Verifies that the scope configuration is properly extracted.
+ */
+ @Test
+ public void testScopeConfig() {
+ Configuration config = new Configuration();
+
+ config.setString(MetricRegistry.KEY_METRICS_SCOPE_NAMING_TM, "A");
+ config.setString(MetricRegistry.KEY_METRICS_SCOPE_NAMING_JOB, "B");
+ config.setString(MetricRegistry.KEY_METRICS_SCOPE_NAMING_TASK, "C");
+ config.setString(MetricRegistry.KEY_METRICS_SCOPE_NAMING_OPERATOR, "D");
+
+ Scope.ScopeFormat scopeConfig = new MetricRegistry(config).getScopeConfig();
+
+ Assert.assertEquals("A", scopeConfig.getTaskManagerFormat());
+ Assert.assertEquals("B", scopeConfig.getJobFormat());
+ Assert.assertEquals("C", scopeConfig.getTaskFormat());
+ Assert.assertEquals("D", scopeConfig.getOperatorFormat());
+
+ Scope.ScopeFormat emptyScopeConfig = new MetricRegistry(new Configuration()).getScopeConfig();
+
+ Assert.assertEquals(TaskManagerMetricGroup.DEFAULT_SCOPE_TM, emptyScopeConfig.getTaskManagerFormat());
+ Assert.assertEquals(JobMetricGroup.DEFAULT_SCOPE_JOB, emptyScopeConfig.getJobFormat());
+ Assert.assertEquals(TaskMetricGroup.DEFAULT_SCOPE_TASK, emptyScopeConfig.getTaskFormat());
+ Assert.assertEquals(OperatorMetricGroup.DEFAULT_SCOPE_OPERATOR, emptyScopeConfig.getOperatorFormat());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java
new file mode 100644
index 0000000..89483b3
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricRegistry;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class JobGroupTest {
+ @Test
+ public void testGenerateScopeDefault() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ JobMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job");
+
+ List<String> scope = operator.generateScope();
+ assertEquals(4, scope.size());
+ assertEquals("job", scope.get(3));
+ }
+
+ @Test
+ public void testGenerateScopeWildcard() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ JobMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setJobFormat(Scope.concat(Scope.SCOPE_WILDCARD, "superjob", JobMetricGroup.SCOPE_JOB_NAME));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(5, scope.size());
+ assertEquals("superjob", scope.get(3));
+ assertEquals("job", scope.get(4));
+ }
+
+ @Test
+ public void testGenerateScopeCustom() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ JobMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setJobFormat(Scope.concat(TaskManagerMetricGroup.SCOPE_TM_HOST, "superjob", JobMetricGroup.SCOPE_JOB_NAME));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(3, scope.size());
+ assertEquals("host", scope.get(0));
+ assertEquals("superjob", scope.get(1));
+ assertEquals("job", scope.get(2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java
new file mode 100644
index 0000000..4f33d2a
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.util.AbstractID;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class OperatorGroupTest {
+ @Test
+ public void testGenerateScopeDefault() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ OperatorMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job")
+ .addTask(new AbstractID(), new AbstractID(), 0, "task").addOperator("operator");
+
+ List<String> scope = operator.generateScope();
+ assertEquals(6, scope.size());
+ assertEquals("host", scope.get(0));
+ assertEquals("taskmanager", scope.get(1));
+ assertEquals("id", scope.get(2));
+ assertEquals("job", scope.get(3));
+ assertEquals("operator", scope.get(4));
+ assertEquals("0", scope.get(5));
+ }
+
+ @Test
+ public void testGenerateScopeWildcard() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ OperatorMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job")
+ .addTask(new AbstractID(), new AbstractID(), 0, "task").addOperator("operator");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setOperatorFormat(Scope.concat(Scope.SCOPE_WILDCARD, "op", OperatorMetricGroup.SCOPE_OPERATOR_NAME));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(7, scope.size());
+ assertEquals("host", scope.get(0));
+ assertEquals("taskmanager", scope.get(1));
+ assertEquals("id", scope.get(2));
+ assertEquals("job", scope.get(3));
+ assertEquals("task", scope.get(4));
+ assertEquals("op", scope.get(5));
+ assertEquals("operator", scope.get(6));
+ }
+
+ @Test
+ public void testGenerateScopeCustom() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ OperatorMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job")
+ .addTask(new AbstractID(), new AbstractID(), 0, "task").addOperator("operator");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setOperatorFormat(Scope.concat("jobs", JobMetricGroup.SCOPE_JOB_NAME, "op", OperatorMetricGroup.SCOPE_OPERATOR_NAME));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(4, scope.size());
+ assertEquals("jobs", scope.get(0));
+ assertEquals("job", scope.get(1));
+ assertEquals("op", scope.get(2));
+ assertEquals("operator", scope.get(3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java
new file mode 100644
index 0000000..c49fdcd
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.util.AbstractID;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TaskGroupTest {
+ @Test
+ public void testGenerateScopeDefault() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ TaskMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job")
+ .addTask(new AbstractID(), new AbstractID(), 0, "task");
+
+ List<String> scope = operator.generateScope();
+ assertEquals(5, scope.size());
+ assertEquals("task", scope.get(4));
+ }
+
+ @Test
+ public void testGenerateScopeWilcard() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ TaskMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job")
+ .addTask(new AbstractID(), new AbstractID(), 0, "task");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setTaskFormat(Scope.concat(Scope.SCOPE_WILDCARD, "supertask", TaskMetricGroup.SCOPE_TASK_NAME));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(6, scope.size());
+ assertEquals("host", scope.get(0));
+ assertEquals("taskmanager", scope.get(1));
+ assertEquals("id", scope.get(2));
+ assertEquals("job", scope.get(3));
+ assertEquals("supertask", scope.get(4));
+ assertEquals("task", scope.get(5));
+ }
+
+ @Test
+ public void testGenerateScopeCustom() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ TaskMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
+ .addJob(new JobID(), "job")
+ .addTask(new AbstractID(), new AbstractID(), 0, "task");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setTaskFormat(Scope.concat(TaskManagerMetricGroup.SCOPE_TM_HOST, JobMetricGroup.SCOPE_JOB_NAME, "supertask", TaskMetricGroup.SCOPE_TASK_NAME));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(4, scope.size());
+ assertEquals("host", scope.get(0));
+ assertEquals("job", scope.get(1));
+ assertEquals("supertask", scope.get(2));
+ assertEquals("task", scope.get(3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java
new file mode 100644
index 0000000..7b3286d
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskManagerGroupTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricRegistry;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TaskManagerGroupTest {
+ @Test
+ public void testGenerateScopeDefault() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ TaskManagerMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id");
+
+ List<String> scope = operator.generateScope();
+ assertEquals(3, scope.size());
+ assertEquals("host", scope.get(0));
+ assertEquals("taskmanager", scope.get(1));
+ assertEquals("id", scope.get(2));
+ }
+
+ @Test
+ public void testGenerateScopeWildcard() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ TaskManagerMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setTaskManagerFormat(Scope.concat(Scope.SCOPE_WILDCARD, "superhost", TaskManagerMetricGroup.SCOPE_TM_HOST));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(2, scope.size());
+ assertEquals("superhost", scope.get(0));
+ assertEquals("host", scope.get(1));
+ }
+
+ @Test
+ public void testGenerateScopeCustom() {
+ MetricRegistry registry = new MetricRegistry(new Configuration());
+ TaskManagerMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id");
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setTaskManagerFormat(Scope.concat("h", TaskManagerMetricGroup.SCOPE_TM_HOST, "t", TaskManagerMetricGroup.SCOPE_TM_ID));
+
+ List<String> scope = operator.generateScope(format);
+ assertEquals(4, scope.size());
+ assertEquals("h", scope.get(0));
+ assertEquals("host", scope.get(1));
+ assertEquals("t", scope.get(2));
+ assertEquals("id", scope.get(3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/reporter/JMXReporterTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/reporter/JMXReporterTest.java b/flink-core/src/test/java/org/apache/flink/metrics/reporter/JMXReporterTest.java
new file mode 100644
index 0000000..0d683c2
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/reporter/JMXReporterTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.reporter;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class JMXReporterTest {
+ /**
+ * Verifies that the JMXReporter properly generates the JMX name.
+ */
+ @Test
+ public void testGenerateName() {
+ String name = "metric";
+
+ List<String> scope = new ArrayList<>();
+ scope.add("value0");
+ scope.add("value1");
+ scope.add("\"value2 (test),=;:?'");
+
+ String jmxName = new JMXReporter().generateName(name, scope);
+
+ Assert.assertEquals("org.apache.flink.metrics:key0=value0,key1=value1,key2=value2_(test)------,name=metric", jmxName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java
new file mode 100644
index 0000000..d607072
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyJobMetricGroup.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.JobMetricGroup;
+import org.apache.flink.util.AbstractID;
+
+public class DummyJobMetricGroup extends JobMetricGroup {
+ public DummyJobMetricGroup() {
+ super(new DummyMetricRegistry(), new DummyTaskManagerMetricGroup(), new JobID(), "job");
+ }
+
+ @Override
+ public DummyTaskMetricGroup addTask(AbstractID id, AbstractID attemptID, int subtaskIndex, String name) {
+ return new DummyTaskMetricGroup();
+ }
+
+ @Override
+ protected MetricGroup addMetric(String name, Metric metric) {
+ return this;
+ }
+
+ @Override
+ public MetricGroup addGroup(int name) {
+ return addGroup("" + name);
+ }
+
+ @Override
+ public MetricGroup addGroup(String name) {
+ return new DummyMetricGroup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java
new file mode 100644
index 0000000..26df874
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricGroup.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.AbstractMetricGroup;
+import org.apache.flink.metrics.groups.Scope;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DummyMetricGroup extends AbstractMetricGroup {
+ public DummyMetricGroup() {
+ super(new DummyMetricRegistry());
+ }
+
+ @Override
+ public List<String> generateScope() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public List<String> generateScope(Scope.ScopeFormat format) {
+ return new ArrayList<>();
+ }
+
+ @Override
+ protected MetricGroup addMetric(String name, Metric metric) {
+ return this;
+ }
+
+ @Override
+ public MetricGroup addGroup(int name) {
+ return addGroup("" + name);
+ }
+
+ @Override
+ public MetricGroup addGroup(String name) {
+ return new DummyMetricGroup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java
new file mode 100644
index 0000000..f0d6d3f
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyMetricRegistry.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricRegistry;
+
+public class DummyMetricRegistry extends MetricRegistry {
+ private static final Configuration config;
+
+ static {
+ config = new Configuration();
+ config.setString(KEY_METRICS_REPORTER_CLASS, DummyReporter.class.getCanonicalName());
+ }
+
+ public DummyMetricRegistry() {
+ super(new Configuration());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java
new file mode 100644
index 0000000..eb45f6a
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyOperatorMetricGroup.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.OperatorMetricGroup;
+
+public class DummyOperatorMetricGroup extends OperatorMetricGroup {
+ public DummyOperatorMetricGroup() {
+ super(new DummyMetricRegistry(), new DummyTaskMetricGroup(), "operator", 0);
+ }
+
+ @Override
+ protected MetricGroup addMetric(String name, Metric metric) {
+ return this;
+ }
+
+ @Override
+ public MetricGroup addGroup(int name) {
+ return addGroup("" + name);
+ }
+
+ @Override
+ public MetricGroup addGroup(String name) {
+ return new DummyMetricGroup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/DummyReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyReporter.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyReporter.java
new file mode 100644
index 0000000..23a7768
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyReporter.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.reporter.MetricReporter;
+
+import java.util.List;
+
+public class DummyReporter implements MetricReporter {
+ @Override
+ public void open(Configuration config) {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public void notifyOfAddedMetric(Metric metric, String name) {
+ }
+
+ @Override
+ public void notifyOfRemovedMetric(Metric metric, String name) {
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ return "";
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java
new file mode 100644
index 0000000..1c7d33b
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskManagerMetricGroup.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.TaskManagerMetricGroup;
+
+public class DummyTaskManagerMetricGroup extends TaskManagerMetricGroup {
+ public DummyTaskManagerMetricGroup() {
+ super(new DummyMetricRegistry(), "host", "id");
+ }
+
+ public DummyJobMetricGroup addJob(JobID id, String name) {
+ return new DummyJobMetricGroup();
+ }
+
+ @Override
+ protected MetricGroup addMetric(String name, Metric metric) {
+ return this;
+ }
+
+ @Override
+ public MetricGroup addGroup(int name) {
+ return addGroup("" + name);
+ }
+
+ @Override
+ public MetricGroup addGroup(String name) {
+ return new DummyMetricGroup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java
new file mode 100644
index 0000000..53683f4
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/DummyTaskMetricGroup.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
+import org.apache.flink.util.AbstractID;
+
+public class DummyTaskMetricGroup extends TaskMetricGroup {
+ public DummyTaskMetricGroup() {
+ super(new DummyMetricRegistry(), new DummyJobMetricGroup(), new AbstractID(), new AbstractID(), 0, "task");
+ }
+
+ public DummyOperatorMetricGroup addOperator(String name) {
+ return new DummyOperatorMetricGroup();
+ }
+
+ @Override
+ protected MetricGroup addMetric(String name, Metric metric) {
+ return this;
+ }
+
+ @Override
+ public MetricGroup addGroup(int name) {
+ return addGroup("" + name);
+ }
+
+ @Override
+ public MetricGroup addGroup(String name) {
+ return new DummyMetricGroup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java b/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java
new file mode 100644
index 0000000..482d1e8
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/util/TestReporter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.util;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.reporter.AbstractReporter;
+import org.apache.flink.metrics.reporter.MetricReporter;
+
+import java.util.List;
+
+public class TestReporter extends AbstractReporter {
+ @Override
+ public void open(Configuration config) {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ return name;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-dist/src/main/flink-bin/bin/config.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh
index ffbec07..b6bdbed 100755
--- a/flink-dist/src/main/flink-bin/bin/config.sh
+++ b/flink-dist/src/main/flink-bin/bin/config.sh
@@ -103,6 +103,8 @@ KEY_ENV_SSH_OPTS="env.ssh.opts"
KEY_RECOVERY_MODE="recovery.mode"
KEY_ZK_HEAP_MB="zookeeper.heap.mb"
+KEY_METRICS_JMX_PORT="metrics.jmx.port"
+
########################################################################################################################
# PATHS AND CONFIG
########################################################################################################################
@@ -240,6 +242,10 @@ if [ -z "${RECOVERY_MODE}" ]; then
RECOVERY_MODE=$(readFromConfig ${KEY_RECOVERY_MODE} "standalone" "${YAML_CONF}")
fi
+if [ -z "${JMX_PORT}" ]; then
+ JMX_PORT=$(readFromConfig ${KEY_METRICS_JMX_PORT} 9010 "${YAML_CONF}")
+fi
+
# Arguments for the JVM. Used for job and task manager JVMs.
# DO NOT USE FOR MEMORY SETTINGS! Use conf/flink-conf.yaml with keys
# KEY_JOBM_MEM_SIZE and KEY_TASKM_MEM_SIZE for that!
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-dist/src/main/flink-bin/bin/flink-daemon.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/flink-daemon.sh b/flink-dist/src/main/flink-bin/bin/flink-daemon.sh
index 1ef7439..cc7163f 100644
--- a/flink-dist/src/main/flink-bin/bin/flink-daemon.sh
+++ b/flink-dist/src/main/flink-bin/bin/flink-daemon.sh
@@ -23,14 +23,24 @@ USAGE="Usage: flink-daemon.sh (start|stop|stop-all) (jobmanager|taskmanager|zook
STARTSTOP=$1
DAEMON=$2
ARGS=("${@:3}") # get remaining arguments as array
+JMX_ARGS=""
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/config.sh
case $DAEMON in
(jobmanager)
CLASS_TO_RUN=org.apache.flink.runtime.jobmanager.JobManager
+ if [ "${ARGS[3]}" == "local" ]; then
+ JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${JMX_PORT} -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+ fi
;;
(taskmanager)
CLASS_TO_RUN=org.apache.flink.runtime.taskmanager.TaskManager
+ JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=${JMX_PORT} -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
;;
(zookeeper)
@@ -43,11 +53,6 @@ case $DAEMON in
;;
esac
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/config.sh
-
if [ "$FLINK_IDENT_STRING" = "" ]; then
FLINK_IDENT_STRING="$USER"
fi
@@ -96,12 +101,13 @@ case $STARTSTOP in
count="${#active[@]}"
if [ ${count} -gt 0 ]; then
+ JMX_ARGS=""
echo "[INFO] $count instance(s) of $DAEMON are already running on $HOSTNAME."
fi
fi
echo "Starting $DAEMON daemon on host $HOSTNAME."
- $JAVA_RUN $JVM_ARGS ${FLINK_ENV_JAVA_OPTS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}" > "$out" 2>&1 < /dev/null &
+ $JAVA_RUN $JVM_ARGS ${FLINK_ENV_JAVA_OPTS} ${JMX_ARGS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}" > "$out" 2>&1 < /dev/null &
mypid=$!
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-java/src/test/java/org/apache/flink/api/common/operators/base/CoGroupOperatorCollectionTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/CoGroupOperatorCollectionTest.java b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/CoGroupOperatorCollectionTest.java
index 8ce83f3..31b3ba2 100644
--- a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/CoGroupOperatorCollectionTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/CoGroupOperatorCollectionTest.java
@@ -31,6 +31,7 @@ import org.apache.flink.api.java.tuple.builder.Tuple2Builder;
import org.apache.flink.api.java.typeutils.TypeInfoParser;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.util.Collector;
import org.junit.Assert;
import org.junit.Test;
@@ -76,7 +77,7 @@ public class CoGroupOperatorCollectionTest implements Serializable {
final HashMap<String, Accumulator<?, ?>> accumulators = new HashMap<String, Accumulator<?, ?>>();
final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
final TaskInfo taskInfo = new TaskInfo("Test UDF", 0, 4, 0);
- final RuntimeContext ctx = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulators);
+ final RuntimeContext ctx = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks, accumulators, new DummyMetricGroup());
{
SumCoGroup udf1 = new SumCoGroup();
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-java/src/test/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorTest.java
index a563281..1b627c4 100644
--- a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/GroupReduceOperatorTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeInfoParser;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.util.Collector;
import org.junit.Test;
@@ -168,9 +169,9 @@ public class GroupReduceOperatorTest implements java.io.Serializable {
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.disableObjectReuse();
- List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
executionConfig.enableObjectReuse();
- List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<Tuple2<String, Integer>>(resultMutableSafe);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-java/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
index a6b3deb..1d5668b 100644
--- a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/InnerJoinOperatorBaseTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.util.Collector;
import org.junit.Test;
@@ -107,9 +108,9 @@ public class InnerJoinOperatorBaseTest implements Serializable {
final TaskInfo taskInfo = new TaskInfo("op", 0, 1, 0);
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.disableObjectReuse();
- List<Tuple2<Double, String>> resultSafe = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Tuple2<Double, String>> resultSafe = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
executionConfig.enableObjectReuse();
- List<Tuple2<Double, String>> resultRegular = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Tuple2<Double, String>> resultRegular = base.executeOnCollections(inputData1, inputData2, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
assertEquals(expected, new HashSet<>(resultSafe));
assertEquals(expected, new HashSet<>(resultRegular));
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-java/src/test/java/org/apache/flink/api/common/operators/base/ReduceOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/ReduceOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/ReduceOperatorTest.java
index c04916d..4317c03 100644
--- a/flink-java/src/test/java/org/apache/flink/api/common/operators/base/ReduceOperatorTest.java
+++ b/flink-java/src/test/java/org/apache/flink/api/common/operators/base/ReduceOperatorTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeInfoParser;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.junit.Test;
import java.util.ArrayList;
@@ -145,9 +146,9 @@ public class ReduceOperatorTest implements java.io.Serializable {
ExecutionConfig executionConfig = new ExecutionConfig();
executionConfig.disableObjectReuse();
- List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
executionConfig.enableObjectReuse();
- List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>()), executionConfig);
+ List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, new RuntimeUDFContext(taskInfo, null, executionConfig, new HashMap<String, Future<Path>>(), new HashMap<String, Accumulator<?, ?>>(), new DummyMetricGroup()), executionConfig);
Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<Tuple2<String, Integer>>(resultMutableSafe);
Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<Tuple2<String, Integer>>(resultRegular);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-dropwizard/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/pom.xml b/flink-metrics/flink-metrics-dropwizard/pom.xml
new file mode 100644
index 0000000..84d9722
--- /dev/null
+++ b/flink-metrics/flink-metrics-dropwizard/pom.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-dropwizard</artifactId>
+ <name>flink-metrics-dropwizard</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
new file mode 100644
index 0000000..a7309be
--- /dev/null
+++ b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.dropwizard;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.dropwizard.metrics.CounterWrapper;
+import org.apache.flink.dropwizard.metrics.GaugeWrapper;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.reporter.MetricReporter;
+import org.apache.flink.metrics.reporter.Scheduled;
+
+import java.util.List;
+
+/**
+ * Base class for {@link org.apache.flink.metrics.reporter.MetricReporter} that wraps a
+ * Dropwizard {@link com.codahale.metrics.Reporter}.
+ */
+@PublicEvolving
+public abstract class ScheduledDropwizardReporter implements MetricReporter, Scheduled {
+ protected MetricRegistry registry;
+ protected ScheduledReporter reporter;
+
+ public static final String ARG_HOST = "host";
+ public static final String ARG_PORT = "port";
+ public static final String ARG_PREFIX = "prefix";
+ public static final String ARG_CONVERSION_RATE = "rateConversion";
+ public static final String ARG_CONVERSION_DURATION = "durationConversion";
+
+ protected ScheduledDropwizardReporter() {
+ this.registry = new MetricRegistry();
+ }
+
+ @Override
+ public synchronized void notifyOfAddedMetric(Metric metric, String name) {
+ if (metric instanceof Counter) {
+ registry.register(name, new CounterWrapper((Counter) metric));
+ } else if (metric instanceof Gauge) {
+ registry.register(name, new GaugeWrapper((Gauge) metric));
+ }
+ }
+
+ @Override
+ public synchronized void notifyOfRemovedMetric(Metric metric, String name) {
+ registry.remove(name);
+ }
+
+ public abstract ScheduledReporter getReporter(Configuration config);
+
+ @Override
+ public void open(Configuration config) {
+ this.reporter = getReporter(config);
+ }
+
+ @Override
+ public void close() {
+ this.reporter.stop();
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : scope) {
+ sb.append(s);
+ sb.append('.');
+ }
+ sb.append(name);
+ return sb.toString();
+ }
+
+ @Override
+ public synchronized void report() {
+ this.reporter.report(
+ this.registry.getGauges(),
+ this.registry.getCounters(),
+ this.registry.getHistograms(),
+ this.registry.getMeters(),
+ this.registry.getTimers());
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
new file mode 100644
index 0000000..f6630b9
--- /dev/null
+++ b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/CounterWrapper.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.dropwizard.metrics;
+
+import org.apache.flink.metrics.Counter;
+
+public class CounterWrapper extends com.codahale.metrics.Counter {
+ private final Counter counter;
+
+ public CounterWrapper(Counter counter) {
+ this.counter = counter;
+ }
+
+ @Override
+ public long getCount() {
+ return this.counter.getCount();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
new file mode 100644
index 0000000..d47090d
--- /dev/null
+++ b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/metrics/GaugeWrapper.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.dropwizard.metrics;
+
+import org.apache.flink.metrics.Gauge;
+
+public class GaugeWrapper implements com.codahale.metrics.Gauge {
+ private final Gauge gauge;
+
+ public GaugeWrapper(Gauge gauge) {
+ this.gauge = gauge;
+ }
+
+ @Override
+ public Object getValue() {
+ return this.gauge.getValue();
+ }
+}
[2/6] flink git commit: [FLINK-1502] [core] Various improvements to
the metrics infrastructure code
Posted by se...@apache.org.
[FLINK-1502] [core] Various improvements to the metrics infrastructure code
- Metric groups are generally thread-safe
- Metric groups are closable. Closed groups do not register metric objects and more.
- TaskManager's JobMetricsGroup auto disposes when all TaskMetricGroups are closed
- Maven project with metric reporters renamed to 'flink-metric-reporters'
- Various code style cleanups
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/707606ac
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/707606ac
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/707606ac
Branch: refs/heads/master
Commit: 707606ac40dbbbd497fcbbb5442870fec5468bf3
Parents: 003ce18
Author: Stephan Ewen <se...@apache.org>
Authored: Fri May 20 09:49:08 2016 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Sun May 22 19:58:17 2016 +0200
----------------------------------------------------------------------
.../common/operators/CollectionExecutor.java | 21 +-
.../java/org/apache/flink/metrics/Counter.java | 7 +-
.../java/org/apache/flink/metrics/Gauge.java | 3 +-
.../org/apache/flink/metrics/MetricGroup.java | 50 +++--
.../apache/flink/metrics/MetricRegistry.java | 219 +++++++++++++------
.../metrics/groups/AbstractMetricGroup.java | 179 +++++++++++----
.../metrics/groups/ComponentMetricGroup.java | 56 ++++-
.../metrics/groups/GenericMetricGroup.java | 3 +-
.../flink/metrics/groups/IOMetricGroup.java | 9 +-
.../flink/metrics/groups/JobMetricGroup.java | 78 +++++--
.../groups/NonRegisteringMetricsGroup.java | 87 ++++++++
.../metrics/groups/OperatorMetricGroup.java | 13 +-
.../org/apache/flink/metrics/groups/Scope.java | 5 +-
.../metrics/groups/TaskManagerMetricGroup.java | 96 ++++++--
.../flink/metrics/groups/TaskMetricGroup.java | 78 +++++--
.../metrics/reporter/AbstractReporter.java | 6 +-
.../flink/metrics/reporter/JMXReporter.java | 7 +-
.../flink/metrics/reporter/MetricReporter.java | 1 +
.../apache/flink/metrics/MetricGroupTest.java | 93 --------
.../flink/metrics/groups/JobGroupTest.java | 25 ++-
.../groups/MetricGroupRegistrationTest.java | 100 +++++++++
.../flink/metrics/groups/MetricGroupTest.java | 125 +++++++++++
.../flink/metrics/groups/OperatorGroupTest.java | 13 +-
.../flink/metrics/groups/TaskGroupTest.java | 15 +-
.../metrics/groups/TaskManagerGroupTest.java | 99 ++++++++-
.../flink/metrics/util/DummyJobMetricGroup.java | 9 +-
.../flink/metrics/util/DummyMetricGroup.java | 18 +-
.../flink/metrics/util/DummyMetricRegistry.java | 7 +-
.../metrics/util/DummyOperatorMetricGroup.java | 10 +-
.../util/DummyTaskManagerMetricGroup.java | 10 +-
.../metrics/util/DummyTaskMetricGroup.java | 10 +-
.../apache/flink/metrics/util/TestReporter.java | 10 +-
.../flink-metrics-dropwizard/pom.xml | 72 ++++++
.../dropwizard/ScheduledDropwizardReporter.java | 99 +++++++++
.../dropwizard/metrics/CounterWrapper.java | 33 +++
.../flink/dropwizard/metrics/GaugeWrapper.java | 33 +++
.../flink-metrics-ganglia/pom.xml | 90 ++++++++
.../flink/metrics/graphite/GangliaReporter.java | 73 +++++++
.../flink-metrics-graphite/pom.xml | 84 +++++++
.../metrics/graphite/GraphiteReporter.java | 70 ++++++
.../flink-metrics-statsd/pom.xml | 43 ++++
.../flink/metrics/statsd/StatsDReporter.java | 132 +++++++++++
flink-metric-reporters/pom.xml | 42 ++++
flink-metrics/flink-metrics-dropwizard/pom.xml | 72 ------
.../dropwizard/ScheduledDropwizardReporter.java | 99 ---------
.../dropwizard/metrics/CounterWrapper.java | 33 ---
.../flink/dropwizard/metrics/GaugeWrapper.java | 33 ---
flink-metrics/flink-metrics-ganglia/pom.xml | 90 --------
.../flink/metrics/graphite/GangliaReporter.java | 73 -------
flink-metrics/flink-metrics-graphite/pom.xml | 84 -------
.../metrics/graphite/GraphiteReporter.java | 70 ------
flink-metrics/flink-metrics-statsd/pom.xml | 43 ----
.../flink/metrics/statsd/StatsDReporter.java | 132 -----------
flink-metrics/pom.xml | 42 ----
.../apache/flink/runtime/taskmanager/Task.java | 34 ++-
.../flink/runtime/taskmanager/TaskManager.scala | 33 +--
pom.xml | 2 +-
57 files changed, 1890 insertions(+), 1183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
index 7e5269e..97f0c5e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
@@ -59,15 +59,11 @@ import org.apache.flink.api.common.operators.util.TypeComparable;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.core.fs.local.LocalFileSystem;
import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.metrics.MetricRegistry;
-import org.apache.flink.metrics.groups.JobMetricGroup;
-import org.apache.flink.metrics.groups.TaskManagerMetricGroup;
+import org.apache.flink.metrics.groups.NonRegisteringMetricsGroup;
import org.apache.flink.types.Value;
-import org.apache.flink.util.AbstractID;
import org.apache.flink.util.Visitor;
/**
@@ -93,8 +89,6 @@ public class CollectionExecutor {
private final ExecutionConfig executionConfig;
private int iterationSuperstep;
-
- private JobMetricGroup jobMetricGroup;
// --------------------------------------------------------------------------------------------
@@ -120,9 +114,7 @@ public class CollectionExecutor {
if (jobID == null) {
jobID = new JobID();
}
- this.jobMetricGroup =
- new TaskManagerMetricGroup(new MetricRegistry(new Configuration()), "localhost", new AbstractID().toString())
- .addJob(jobID, program.getJobName());
+
initCache(program.getCachedFiles());
Collection<? extends GenericDataSinkBase<?>> sinks = program.getDataSinks();
for (Operator<?> sink : sinks) {
@@ -202,7 +194,7 @@ public class CollectionExecutor {
TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 0, 1, 0);
RuntimeUDFContext ctx;
- MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, typedSink.getName());
+ MetricGroup metrics = NonRegisteringMetricsGroup.get();
if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
@@ -223,7 +215,7 @@ public class CollectionExecutor {
RuntimeUDFContext ctx;
- MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, source.getName());
+ MetricGroup metrics = NonRegisteringMetricsGroup.get();
if (RichInputFormat.class.isAssignableFrom(typedSource.getUserCodeWrapper().getUserCodeClass())) {
ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
@@ -249,7 +241,7 @@ public class CollectionExecutor {
TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 0, 1, 0);
RuntimeUDFContext ctx;
- MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, typedOp.getName());
+ MetricGroup metrics = NonRegisteringMetricsGroup.get();
if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
@@ -291,7 +283,8 @@ public class CollectionExecutor {
TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 0, 1, 0);
RuntimeUDFContext ctx;
- MetricGroup metrics = this.jobMetricGroup.addTask(new AbstractID(), new AbstractID(), 0, typedOp.getName());
+ MetricGroup metrics = NonRegisteringMetricsGroup.get();
+
if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) {
ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics) :
new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulators, metrics);
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/Counter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/Counter.java b/flink-core/src/main/java/org/apache/flink/metrics/Counter.java
index b18da4f..201a613 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/Counter.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/Counter.java
@@ -20,12 +20,13 @@ package org.apache.flink.metrics;
import org.apache.flink.annotation.PublicEvolving;
/**
- * A Counter is a {@link org.apache.flink.metrics.Metric} that measures a count.
+ * A Counter is a {@link Metric} that measures a count.
*/
@PublicEvolving
public final class Counter implements Metric {
- private long count = 0;
-
+
+ private long count;
+
/**
* Increment the current count by 1.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java b/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java
index 455587a..cca105e 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/Gauge.java
@@ -20,10 +20,11 @@ package org.apache.flink.metrics;
import org.apache.flink.annotation.PublicEvolving;
/**
- * A Gauge is a {@link org.apache.flink.metrics.Metric} that calculates a specific value at a point in time.
+ * A Gauge is a {@link Metric} that calculates a specific value at a point in time.
*/
@PublicEvolving
public abstract class Gauge<T> implements Metric {
+
/**
* Calculates and returns the measured value.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java
index 4036129..a3832ff 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/MetricGroup.java
@@ -15,31 +15,49 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics;
import org.apache.flink.annotation.PublicEvolving;
/**
- * A MetricGroup is a named container for {@link org.apache.flink.metrics.Metric}s and
- * {@link org.apache.flink.metrics.MetricGroup}s.
- * <p>
- * Instances of this class can be used to register new metrics with Flink and to create a nested hierarchy based on the
- * group names.
- * <p>
- * A MetricGroup is uniquely identified by it's place in the hierarchy and name.
+ * A MetricGroup is a named container for {@link Metric Metrics} and {@link MetricGroup MetricGroups}.
+ *
+ * <p>Instances of this class can be used to register new metrics with Flink and to create a nested
+ * hierarchy based on the group names.
+ *
+ * <p>A MetricGroup is uniquely identified by it's place in the hierarchy and name.
+ *
+ * <p>Metrics groups can be {@link #close() closed}. Upon closing, they de-register all metrics
+ * from any metrics reporter and any internal maps. Note that even closed metrics groups
+ * return Counters, Gauges, etc to the code, to prevent exceptions in the monitored code.
+ * These metrics simply do not get reported any more, when created on a closed group.
*/
@PublicEvolving
public interface MetricGroup {
+ // ------------------------------------------------------------------------
+ // Closing
+ // ------------------------------------------------------------------------
+
/**
- * Recursively unregisters all {@link org.apache.flink.metrics.Metric}s contained in this
- * {@link org.apache.flink.metrics.MetricGroup}
+ * Marks the group as closed.
+ * Recursively unregisters all {@link Metric Metrics} contained in this group.
+ *
+ * <p>Any metrics created after the call to this function will not be registered in
+ * the {@link MetricRegistry} and not be reported to any reporter (like JMX).
*/
void close();
- // -----------------------------------------------------------------------------------------------------------------
- // Metrics
- // -----------------------------------------------------------------------------------------------------------------
+ /**
+ * Checks whether this MetricGroup has been closed.
+ * @return True if the group has been closed, false is the group is still open.
+ */
+ boolean isClosed();
+
+ // ------------------------------------------------------------------------
+ // Metrics
+ // ------------------------------------------------------------------------
/**
* Creates and registers a new {@link org.apache.flink.metrics.Counter} with Flink.
@@ -77,12 +95,12 @@ public interface MetricGroup {
*/
<T> Gauge<T> gauge(String name, Gauge<T> gauge);
- // -----------------------------------------------------------------------------------------------------------------
+ // ------------------------------------------------------------------------
// Groups
- // -----------------------------------------------------------------------------------------------------------------
+ // ------------------------------------------------------------------------
/**
- * Creates a new {@link org.apache.flink.metrics.MetricGroup} and adds it to this groups sub-groups.
+ * Creates a new MetricGroup and adds it to this groups sub-groups.
*
* @param name name of the group
* @return the created group
@@ -90,7 +108,7 @@ public interface MetricGroup {
MetricGroup addGroup(int name);
/**
- * Creates a new {@link org.apache.flink.metrics.MetricGroup} and adds it to this groups sub-groups.
+ * Creates a new MetricGroup and adds it to this groups sub-groups.
*
* @param name name of the group
* @return the created group
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java b/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java
index 0e8b0d5..7e06217 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/MetricRegistry.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics;
import org.apache.flink.annotation.Internal;
@@ -24,6 +25,7 @@ import org.apache.flink.metrics.groups.Scope;
import org.apache.flink.metrics.reporter.JMXReporter;
import org.apache.flink.metrics.reporter.MetricReporter;
import org.apache.flink.metrics.reporter.Scheduled;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,15 +38,15 @@ import static org.apache.flink.metrics.groups.TaskManagerMetricGroup.DEFAULT_SCO
import static org.apache.flink.metrics.groups.TaskMetricGroup.DEFAULT_SCOPE_TASK;
/**
- * A MetricRegistry keeps track of all registered {@link org.apache.flink.metrics.Metric}s. It serves as the
- * connection between {@link org.apache.flink.metrics.MetricGroup}s and {@link org.apache.flink.metrics.reporter.MetricReporter}s.
+ * A MetricRegistry keeps track of all registered {@link Metric Metrics}. It serves as the
+ * connection between {@link MetricGroup MetricGroups} and {@link MetricReporter MetricReporters}.
*/
@Internal
public class MetricRegistry {
- private static final Logger LOG = LoggerFactory.getLogger(MetricRegistry.class);
- private final MetricReporter reporter;
- private java.util.Timer timer;
+ // ------------------------------------------------------------------------
+ // configuration keys
+ // ------------------------------------------------------------------------
public static final String KEY_METRICS_REPORTER_CLASS = "metrics.reporter.class";
public static final String KEY_METRICS_REPORTER_ARGUMENTS = "metrics.reporter.arguments";
@@ -55,78 +57,88 @@ public class MetricRegistry {
public static final String KEY_METRICS_SCOPE_NAMING_TASK = "metrics.scope.task";
public static final String KEY_METRICS_SCOPE_NAMING_OPERATOR = "metrics.scope.operator";
+ // ------------------------------------------------------------------------
+ // configuration keys
+ // ------------------------------------------------------------------------
+
+ private static final Logger LOG = LoggerFactory.getLogger(MetricRegistry.class);
+
+ private final MetricReporter reporter;
+ private final java.util.Timer timer;
+
private final Scope.ScopeFormat scopeConfig;
/**
- * Creates a new {@link MetricRegistry} and starts the configured reporter.
+ * Creates a new MetricRegistry and starts the configured reporter.
*/
public MetricRegistry(Configuration config) {
+ // first parse the scope formats, these are needed for all reporters
+
+ Scope.ScopeFormat scopeFormat;
try {
- String className = config.getString(KEY_METRICS_REPORTER_CLASS, null);
- if (className == null) {
- LOG.info("No reporter class name defined in flink-conf.yaml, defaulting to " + JMXReporter.class.getName() + ".");
- className = JMXReporter.class.getName();
- }
+ scopeFormat = createScopeConfig(config);
+ }
+ catch (Exception e) {
+ scopeFormat = createScopeConfig(new Configuration());
+ LOG.warn("Failed to parse scope format, using default scope formats", e);
+ }
+ this.scopeConfig = scopeFormat;
- this.scopeConfig = createScopeConfig(config);
-
- Configuration reporterConfig = createReporterConfig(config);
- Class<?> reporterClass = Class.forName(className);
- reporter = (MetricReporter) reporterClass.newInstance();
- reporter.open(reporterConfig);
-
- if (reporter instanceof Scheduled) {
- String[] interval = config.getString(KEY_METRICS_REPORTER_INTERVAL, "10 SECONDS").split(" ");
- long millis = TimeUnit.valueOf(interval[1]).toMillis(Long.parseLong(interval[0]));
- timer = new java.util.Timer(true);
- timer.schedule(new TimerTask() {
- @Override
- public void run() {
- ((Scheduled) reporter).report();
+ // second, instantiate any custom configured reporters
+
+ final String className = config.getString(KEY_METRICS_REPORTER_CLASS, null);
+ if (className == null) {
+ this.reporter = new JMXReporter();
+ this.timer = null;
+ }
+ else {
+ MetricReporter reporter;
+ java.util.Timer timer;
+
+ try {
+ String configuredPeriod = config.getString(KEY_METRICS_REPORTER_INTERVAL, null);
+ TimeUnit timeunit = TimeUnit.SECONDS;
+ long period = 10;
+
+ if (configuredPeriod != null) {
+ try {
+ String[] interval = configuredPeriod.split(" ");
+ period = Long.parseLong(interval[0]);
+ timeunit = TimeUnit.valueOf(interval[1]);
+ }
+ catch (Exception e) {
+ LOG.error("Cannot parse report interval from config: " + configuredPeriod +
+ " - please use values like '10 SECONDS' or '500 MILLISECONDS'. " +
+ "Using default reporting interval.");
}
- }, millis, millis);
+ }
+
+ Configuration reporterConfig = createReporterConfig(config, timeunit, period);
+
+ Class<?> reporterClass = Class.forName(className);
+ reporter = (MetricReporter) reporterClass.newInstance();
+ reporter.open(reporterConfig);
+
+ if (reporter instanceof Scheduled) {
+ LOG.info("Periodically reporting metrics in intervals of {} {}", period, timeunit.name());
+ long millis = timeunit.toMillis(period);
+
+ timer = new java.util.Timer("Periodic Metrics Reporter", true);
+ timer.schedule(new ReporterTask((Scheduled) reporter), millis, millis);
+ }
+ else {
+ timer = null;
+ }
}
- } catch (InstantiationException | ClassNotFoundException e) {
- throw new RuntimeException("Error while instantiating reporter.", e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException("Implementation error.", e);
- }
- }
-
- private static Configuration createReporterConfig(Configuration config) {
- String[] interval = config.getString(KEY_METRICS_REPORTER_INTERVAL, "10 SECONDS").split(" ");
-
- String[] arguments = config.getString(KEY_METRICS_REPORTER_ARGUMENTS, "").split(" ");
-
- Configuration reporterConfig = new Configuration();
- reporterConfig.setString("period", interval[0]);
- reporterConfig.setString("timeunit", interval[1]);
-
- if (arguments.length > 1) {
- for (int x = 0; x < arguments.length; x += 2) {
- reporterConfig.setString(arguments[x].replace("--", ""), arguments[x + 1]);
+ catch (Throwable t) {
+ reporter = new JMXReporter();
+ timer = null;
+ LOG.error("Could not instantiate custom metrics reporter. Defaulting to JMX metrics export.", t);
}
- }
- return reporterConfig;
- }
-
- private static Scope.ScopeFormat createScopeConfig(Configuration config) {
- String tmFormat = config.getString(KEY_METRICS_SCOPE_NAMING_TM, DEFAULT_SCOPE_TM);
- String jobFormat = config.getString(KEY_METRICS_SCOPE_NAMING_JOB, DEFAULT_SCOPE_JOB);
- String taskFormat = config.getString(KEY_METRICS_SCOPE_NAMING_TASK, DEFAULT_SCOPE_TASK);
- String operatorFormat = config.getString(KEY_METRICS_SCOPE_NAMING_OPERATOR, DEFAULT_SCOPE_OPERATOR);
-
- Scope.ScopeFormat format = new Scope.ScopeFormat();
- format.setTaskManagerFormat(tmFormat);
- format.setJobFormat(jobFormat);
- format.setTaskFormat(taskFormat);
- format.setOperatorFormat(operatorFormat);
- return format;
- }
-
- public Scope.ScopeFormat getScopeConfig() {
- return this.scopeConfig;
+ this.reporter = reporter;
+ this.timer = timer;
+ }
}
/**
@@ -137,10 +149,22 @@ public class MetricRegistry {
timer.cancel();
}
if (reporter != null) {
- reporter.close();
+ try {
+ reporter.close();
+ } catch (Throwable t) {
+ LOG.warn("Metrics reporter did not shut down cleanly", t);
+ }
}
}
+ public Scope.ScopeFormat getScopeConfig() {
+ return this.scopeConfig;
+ }
+
+ // ------------------------------------------------------------------------
+ // Metrics (de)registration
+ // ------------------------------------------------------------------------
+
/**
* Registers a new {@link org.apache.flink.metrics.Metric} with this registry.
*
@@ -150,8 +174,6 @@ public class MetricRegistry {
*/
public void register(Metric metric, String name, AbstractMetricGroup parent) {
String metricName = reporter.generateName(name, parent.generateScope());
-
-
this.reporter.notifyOfAddedMetric(metric, metricName);
}
@@ -167,4 +189,63 @@ public class MetricRegistry {
this.reporter.notifyOfRemovedMetric(metric, metricName);
}
+
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
+
+ private static Configuration createReporterConfig(Configuration config, TimeUnit timeunit, long period) {
+ Configuration reporterConfig = new Configuration();
+ reporterConfig.setLong("period", period);
+ reporterConfig.setString("timeunit", timeunit.name());
+
+ String[] arguments = config.getString(KEY_METRICS_REPORTER_ARGUMENTS, "").split(" ");
+ if (arguments.length > 1) {
+ for (int x = 0; x < arguments.length; x += 2) {
+ reporterConfig.setString(arguments[x].replace("--", ""), arguments[x + 1]);
+ }
+ }
+ return reporterConfig;
+ }
+
+ private static Scope.ScopeFormat createScopeConfig(Configuration config) {
+ String tmFormat = config.getString(KEY_METRICS_SCOPE_NAMING_TM, DEFAULT_SCOPE_TM);
+ String jobFormat = config.getString(KEY_METRICS_SCOPE_NAMING_JOB, DEFAULT_SCOPE_JOB);
+ String taskFormat = config.getString(KEY_METRICS_SCOPE_NAMING_TASK, DEFAULT_SCOPE_TASK);
+ String operatorFormat = config.getString(KEY_METRICS_SCOPE_NAMING_OPERATOR, DEFAULT_SCOPE_OPERATOR);
+
+
+ Scope.ScopeFormat format = new Scope.ScopeFormat();
+ format.setTaskManagerFormat(tmFormat);
+ format.setJobFormat(jobFormat);
+ format.setTaskFormat(taskFormat);
+ format.setOperatorFormat(operatorFormat);
+ return format;
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * This task is explicitly a static class, so that it does not hold any references to the enclosing
+ * MetricsRegistry instance.
+ *
+ * This is a subtle difference, but very important: With this static class, the enclosing class instance
+ * may become garbage-collectible, whereas with an anonymous inner class, the timer thread
+ * (which is a GC root) will hold a reference via the timer task and its enclosing instance pointer.
+ * Making the MetricsRegistry garbage collectible makes the java.util.Timer garbage collectible,
+ * which acts as a fail-safe to stop the timer thread and prevents resource leaks.
+ */
+ private static final class ReporterTask extends TimerTask {
+
+ private final Scheduled reporter;
+
+ private ReporterTask(Scheduled reporter) {
+ this.reporter = reporter;
+ }
+
+ @Override
+ public void run() {
+ reporter.report();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java
index 373ac09..cad241d 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/AbstractMetricGroup.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.groups;
import org.apache.flink.annotation.Internal;
@@ -23,44 +24,94 @@ import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.Metric;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.MetricRegistry;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
- * Abstract {@link org.apache.flink.metrics.MetricGroup} that contains key functionality for adding metrics and groups.
+ * Abstract {@link MetricGroup} that contains key functionality for adding metrics and groups.
+ *
+ * <p><b>IMPORTANT IMPLEMENTATION NOTE</b>
+ *
+ * <p>This class uses locks for adding and removing metrics objects. This is done to
+ * prevent resource leaks in the presence of concurrently closing a group and adding
+ * metrics and subgroups.
+ * Since closing groups recursively closes the subgroups, the lock acquisition order must
+ * be strictly from parent group to subgroup. If at any point, a subgroup holds its group
+ * lock and calls a parent method that also acquires the lock, it will create a deadlock
+ * condition.
*/
@Internal
public abstract class AbstractMetricGroup implements MetricGroup {
+
+ /** shared logger */
private static final Logger LOG = LoggerFactory.getLogger(MetricGroup.class);
+
+ private static final String METRIC_NAME_REGEX = "[a-zA-Z0-9]*";
+
+ /** The pattern that metric and group names have to match */
+ private static final Pattern METRIC_NAME_PATTERN = Pattern.compile(METRIC_NAME_REGEX);
+
+ // ------------------------------------------------------------------------
+
+ /** The registry that this metrics group belongs to */
protected final MetricRegistry registry;
- // all metrics that are directly contained in this group
+ /** All metrics that are directly contained in this group */
protected final Map<String, Metric> metrics = new HashMap<>();
- // all generic groups that are directly contained in this group
+
+ /** All metric subgroups of this group */
protected final Map<String, MetricGroup> groups = new HashMap<>();
+ /** Flag indicating whether this group has been closed */
+ private volatile boolean closed;
+
+ // ------------------------------------------------------------------------
+
public AbstractMetricGroup(MetricRegistry registry) {
- this.registry = registry;
+ this.registry = checkNotNull(registry);
}
+ // ------------------------------------------------------------------------
+ // Closing
+ // ------------------------------------------------------------------------
+
@Override
public void close() {
- for (MetricGroup group : groups.values()) {
- group.close();
- }
- this.groups.clear();
- for (Map.Entry<String, Metric> metric : metrics.entrySet()) {
- registry.unregister(metric.getValue(), metric.getKey(), this);
+ synchronized (this) {
+ if (!closed) {
+ closed = true;
+
+ // close all subgroups
+ for (MetricGroup group : groups.values()) {
+ group.close();
+ }
+ groups.clear();
+
+ // un-register all directly contained metrics
+ for (Map.Entry<String, Metric> metric : metrics.entrySet()) {
+ registry.unregister(metric.getValue(), metric.getKey(), this);
+ }
+ metrics.clear();
+ }
}
- this.metrics.clear();
+ }
+
+ @Override
+ public final boolean isClosed() {
+ return closed;
}
// -----------------------------------------------------------------------------------------------------------------
- // Scope
+ // Scope
// -----------------------------------------------------------------------------------------------------------------
/**
@@ -79,12 +130,12 @@ public abstract class AbstractMetricGroup implements MetricGroup {
public abstract List<String> generateScope(Scope.ScopeFormat format);
// -----------------------------------------------------------------------------------------------------------------
- // Metrics
+ // Metrics
// -----------------------------------------------------------------------------------------------------------------
@Override
public Counter counter(int name) {
- return counter("" + name);
+ return counter(String.valueOf(name));
}
@Override
@@ -96,7 +147,7 @@ public abstract class AbstractMetricGroup implements MetricGroup {
@Override
public <T> Gauge<T> gauge(int name, Gauge<T> gauge) {
- return gauge("" + name, gauge);
+ return gauge(String.valueOf(name), gauge);
}
@Override
@@ -105,41 +156,91 @@ public abstract class AbstractMetricGroup implements MetricGroup {
return gauge;
}
- protected MetricGroup addMetric(String name, Metric metric) {
- if (!name.matches("[a-zA-Z0-9]*")) {
- throw new IllegalArgumentException("Metric names may not contain special characters.");
- }
- if (metrics.containsKey(name)) {
- LOG.warn("Detected metric name collision. This group already contains a group for the given group name. " +
- this.generateScope().toString() + "." + name);
+ /**
+ * Adds the given metric to the group and registers it at the registry, if the group
+ * is not yet closed, and if no metric with the same name has been registered before.
+ *
+ * @param name the name to register the metric under
+ * @param metric the metric to register
+ */
+ protected void addMetric(String name, Metric metric) {
+ Matcher nameMatcher = METRIC_NAME_PATTERN.matcher(name);
+ if (!nameMatcher.matches()) {
+ throw new IllegalArgumentException("Metric names may not contain special characters or spaces. " +
+ "Allowed is: " + METRIC_NAME_REGEX);
}
- if (groups.containsKey(name)) {
- LOG.warn("Detected metric name collision. This group already contains a group for the given metric name." +
- this.generateScope().toString() + ")." + name);
+
+ // add the metric only if the group is still open
+ synchronized (this) {
+ if (!closed) {
+ // immediately put without a 'contains' check to optimize the common case (no collition)
+ // collisions are resolved later
+ Metric prior = metrics.put(name, metric);
+
+ // check for collisions with other metric names
+ if (prior == null) {
+ // no other metric with this name yet
+
+ if (groups.containsKey(name)) {
+ // we warn here, rather than failing, because metrics are tools that should not fail the
+ // program when used incorrectly
+ LOG.warn("Name collision: Adding a metric with the same name as a metric subgroup: '" +
+ name + "'. Metric might not get properly reported. (" + generateScope() + ')');
+ }
+
+ registry.register(metric, name, this);
+ }
+ else {
+ // we had a collision. put back the original value
+ metrics.put(name, prior);
+
+ // we warn here, rather than failing, because metrics are tools that should not fail the
+ // program when used incorrectly
+ LOG.warn("Name collision: Group already contains a Metric with the name '" +
+ name + "'. Metric will not be reported. (" + generateScope() + ')');
+ }
+ }
}
- metrics.put(name, metric);
- registry.register(metric, name, this);
- return this;
}
- // -----------------------------------------------------------------------------------------------------------------
- // Groups
- // -----------------------------------------------------------------------------------------------------------------
+ // ------------------------------------------------------------------------
+ // Groups
+ // ------------------------------------------------------------------------
@Override
public MetricGroup addGroup(int name) {
- return addGroup("" + name);
+ return addGroup(String.valueOf(name));
}
@Override
public MetricGroup addGroup(String name) {
- if (metrics.containsKey(name)) {
- LOG.warn("Detected metric name collision. This group already contains a metric for the given group name."
- + this.generateScope().toString() + "." + name);
- }
- if (!groups.containsKey(name)) {
- groups.put(name, new GenericMetricGroup(registry, this, name));
+ synchronized (this) {
+ if (!closed) {
+ // adding a group with the same name as a metric creates problems in many reporters/dashboards
+ // we warn here, rather than failing, because metrics are tools that should not fail the
+ // program when used incorrectly
+ if (metrics.containsKey(name)) {
+ LOG.warn("Name collision: Adding a metric subgroup with the same name as an existing metric: '" +
+ name + "'. Metric might not get properly reported. (" + generateScope() + ')');
+ }
+
+ MetricGroup newGroup = new GenericMetricGroup(registry, this, name);
+ MetricGroup prior = groups.put(name, newGroup);
+ if (prior == null) {
+ // no prior group with that name
+ return newGroup;
+ } else {
+ // had a prior group with that name, add the prior group back
+ groups.put(name, prior);
+ return prior;
+ }
+ }
+ else {
+ // return a non-registered group that is immediately closed already
+ GenericMetricGroup closedGroup = new GenericMetricGroup(registry, this, name);
+ closedGroup.close();
+ return closedGroup;
+ }
}
- return groups.get(name);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java
index 81851e2..c68cdc1 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/ComponentMetricGroup.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.groups;
import org.apache.flink.annotation.Internal;
@@ -28,17 +29,26 @@ import java.util.Map;
import static org.apache.flink.metrics.groups.Scope.SCOPE_WILDCARD;
/**
- * Abstract {@link org.apache.flink.metrics.MetricGroup} that contains key functionality for modifying the scope.
+ * Abstract {@link org.apache.flink.metrics.MetricGroup} for system components (e.g.,
+ * TaskManager, Job, Task, Operator).
+ *
+ * <p>The components metric groups contain functionality to define alias names for
+ * the component identifier. For example, while Tasks are registered under a Task Attempt ID,
+ * the metrics name scope may use the task name instead. Using these aliases makes
+ * Metric scope names stable across jobs, recovery attempts, etc.
*/
@Internal
public abstract class ComponentMetricGroup extends AbstractMetricGroup {
- private final ComponentMetricGroup parent;
- private final String format;
+ private final ComponentMetricGroup parent;
+ private final String format;
+
// Map: scope variable -> specific value
protected final Map<String, String> formats;
+ // ------------------------------------------------------------------------
+
/**
* Creates a new ComponentMetricGroup.
*
@@ -53,9 +63,45 @@ public abstract class ComponentMetricGroup extends AbstractMetricGroup {
this.format = scopeFormat;
}
+ /**
+ * Closes the component group by removing and closing all metrics and subgroups
+ * (inherited from {@link AbstractMetricGroup}), plus closing and removing all dedicated
+ * component subgroups.
+ */
+ @Override
+ public void close() {
+ synchronized (this) {
+ if (!isClosed()) {
+ // remove all metrics and generic subgroups
+ super.close();
+
+ // remove and close all subcomponent metrics
+ for (ComponentMetricGroup group : subComponents()) {
+ group.close();
+ }
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // sub components
+ // ------------------------------------------------------------------------
+
+ protected ComponentMetricGroup parent() {
+ return parent;
+ }
+
+ protected abstract Iterable<? extends ComponentMetricGroup> subComponents();
+
+ // ------------------------------------------------------------------------
+ // scope format
+ // ------------------------------------------------------------------------
+
+ protected abstract String getScopeFormat(Scope.ScopeFormat format);
+
@Override
public List<String> generateScope() {
- return this.generateScope(this.format);
+ return generateScope(format);
}
@Override
@@ -63,8 +109,6 @@ public abstract class ComponentMetricGroup extends AbstractMetricGroup {
return generateScope(getScopeFormat(format));
}
- protected abstract String getScopeFormat(Scope.ScopeFormat format);
-
private List<String> generateScope(String format) {
String[] components = Scope.split(format);
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java
index 5886312..eedb0fa 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/GenericMetricGroup.java
@@ -27,12 +27,13 @@ import java.util.List;
*/
@Internal
public class GenericMetricGroup extends AbstractMetricGroup {
+
private final AbstractMetricGroup parent;
private final String name;
protected GenericMetricGroup(MetricRegistry registry, AbstractMetricGroup parent, int name) {
- this(registry, parent, "" + name);
+ this(registry, parent, String.valueOf(name));
}
protected GenericMetricGroup(MetricRegistry registry, AbstractMetricGroup parent, String name) {
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java
index 68d91c4..dea5650 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/IOMetricGroup.java
@@ -26,12 +26,13 @@ import java.util.List;
* Special {@link org.apache.flink.metrics.MetricGroup} that contains shareable pre-defined IO-related metrics.
*/
public class IOMetricGroup extends AbstractMetricGroup {
+
private final TaskMetricGroup parent;
- private transient final Counter numBytesIn;
- private transient final Counter numBytesOut;
- private transient final Counter numRecordsIn;
- private transient final Counter numRecordsOut;
+ private final Counter numBytesIn;
+ private final Counter numBytesOut;
+ private final Counter numRecordsIn;
+ private final Counter numRecordsOut;
public IOMetricGroup(MetricRegistry registry, TaskMetricGroup parent) {
super(registry);
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java
index 35a01f8..f4f634a 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/JobMetricGroup.java
@@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.groups;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.MetricRegistry;
import org.apache.flink.util.AbstractID;
@@ -27,41 +27,85 @@ import java.util.HashMap;
import java.util.Map;
import static org.apache.flink.metrics.groups.TaskManagerMetricGroup.DEFAULT_SCOPE_TM;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
* Special {@link org.apache.flink.metrics.MetricGroup} representing a Job.
- * <p<
- * Contains extra logic for adding tasks.
+ *
+ * <p>Contains extra logic for adding tasks.
*/
@Internal
public class JobMetricGroup extends ComponentMetricGroup {
+
public static final String SCOPE_JOB_DESCRIPTOR = "job";
public static final String SCOPE_JOB_ID = Scope.format("job_id");
public static final String SCOPE_JOB_NAME = Scope.format("job_name");
public static final String DEFAULT_SCOPE_JOB_COMPONENT = Scope.concat(SCOPE_JOB_NAME);
public static final String DEFAULT_SCOPE_JOB = Scope.concat(DEFAULT_SCOPE_TM, DEFAULT_SCOPE_JOB_COMPONENT);
- private Map<AbstractID, TaskMetricGroup> tasks = new HashMap<>();
+ // ------------------------------------------------------------------------
+
+ /** Map from execution attempt ID (task identifier) to task metrics */
+ private final Map<AbstractID, TaskMetricGroup> tasks = new HashMap<>();
+
+ private final JobID jobId;
- public JobMetricGroup(MetricRegistry registry, TaskManagerMetricGroup taskManager, JobID id, String name) {
+ // ------------------------------------------------------------------------
+
+ public JobMetricGroup(MetricRegistry registry, TaskManagerMetricGroup taskManager, JobID jobId, String jobName) {
super(registry, taskManager, registry.getScopeConfig().getJobFormat());
- this.formats.put(SCOPE_JOB_ID, id.toString());
- this.formats.put(SCOPE_JOB_NAME, name);
+
+ this.jobId = checkNotNull(jobId);
+ this.formats.put(SCOPE_JOB_ID, jobId.toString());
+ this.formats.put(SCOPE_JOB_NAME, checkNotNull(jobName));
+ }
+
+ // ------------------------------------------------------------------------
+ // adding / removing tasks
+ // ------------------------------------------------------------------------
+
+ public TaskMetricGroup addTask(AbstractID vertexId, AbstractID executionId, int subtaskIndex, String name) {
+ checkNotNull(vertexId);
+ checkNotNull(executionId);
+ checkNotNull(name);
+
+ synchronized (this) {
+ if (!isClosed()) {
+ TaskMetricGroup task = new TaskMetricGroup(registry, this, vertexId, executionId, subtaskIndex, name);
+ tasks.put(executionId, task);
+ return task;
+ } else {
+ return null;
+ }
+ }
}
- public TaskMetricGroup addTask(AbstractID id, AbstractID attemptID, int subtaskIndex, String name) {
- TaskMetricGroup task = new TaskMetricGroup(this.registry, this, id, attemptID, subtaskIndex, name);
- tasks.put(id, task);
- return task;
+ public void removeTaskMetricGroup(AbstractID executionId) {
+ checkNotNull(executionId);
+
+ boolean removeFromParent = false;
+ synchronized (this) {
+ if (!isClosed() && tasks.remove(executionId) != null && tasks.isEmpty()) {
+ // this call removed the last task. close this group.
+ removeFromParent = true;
+ close();
+ }
+ }
+
+ // IMPORTANT: removing from the parent must happen while holding the this group's lock,
+ // because it would violate the "first parent then subgroup" lock acquisition order
+ if (removeFromParent) {
+ ((TaskManagerMetricGroup) parent()).removeJobMetricsGroup(jobId, this);
+ }
}
+ // ------------------------------------------------------------------------
+ // component group behavior
+ // ------------------------------------------------------------------------
+
@Override
- public void close() {
- super.close();
- for (MetricGroup group : tasks.values()) {
- group.close();
- }
- tasks.clear();
+ protected Iterable<? extends ComponentMetricGroup> subComponents() {
+ return tasks.values();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/NonRegisteringMetricsGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/NonRegisteringMetricsGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/NonRegisteringMetricsGroup.java
new file mode 100644
index 0000000..1bfcfe3
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/NonRegisteringMetricsGroup.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.MetricGroup;
+
+/**
+ * Metrics group that does not register any metrics.
+ */
+@Internal
+public class NonRegisteringMetricsGroup implements MetricGroup {
+
+ // ------------------------------------------------------------------------
+ // singleton
+ // ------------------------------------------------------------------------
+
+ private static final NonRegisteringMetricsGroup INSTANCE = new NonRegisteringMetricsGroup();
+
+ public static NonRegisteringMetricsGroup get() {
+ return INSTANCE;
+ }
+
+ /** Private constructor to prevent instantiation */
+ private NonRegisteringMetricsGroup() {}
+
+ // ------------------------------------------------------------------------
+ // metrics group
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void close() {}
+
+ @Override
+ public boolean isClosed() {
+ return false;
+ }
+
+ @Override
+ public Counter counter(int name) {
+ return new Counter();
+ }
+
+ @Override
+ public Counter counter(String name) {
+ return new Counter();
+ }
+
+ @Override
+ public <T> Gauge<T> gauge(int name, Gauge<T> gauge) {
+ return gauge;
+ }
+
+ @Override
+ public <T> Gauge<T> gauge(String name, Gauge<T> gauge) {
+ return gauge;
+ }
+
+
+ @Override
+ public MetricGroup addGroup(int name) {
+ return addGroup(String.valueOf(name));
+ }
+
+ @Override
+ public MetricGroup addGroup(String name) {
+ return new NonRegisteringMetricsGroup();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java
index 6475eec..390b55b 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/OperatorMetricGroup.java
@@ -20,6 +20,8 @@ package org.apache.flink.metrics.groups;
import org.apache.flink.annotation.Internal;
import org.apache.flink.metrics.MetricRegistry;
+import java.util.Collections;
+
import static org.apache.flink.metrics.groups.JobMetricGroup.DEFAULT_SCOPE_JOB;
/**
@@ -27,6 +29,7 @@ import static org.apache.flink.metrics.groups.JobMetricGroup.DEFAULT_SCOPE_JOB;
*/
@Internal
public class OperatorMetricGroup extends ComponentMetricGroup {
+
public static final String SCOPE_OPERATOR_DESCRIPTOR = "operator";
public static final String SCOPE_OPERATOR_NAME = Scope.format("operator_name");
public static final String SCOPE_OPERATOR_SUBTASK_INDEX = Scope.format("subtask_index");
@@ -35,12 +38,20 @@ public class OperatorMetricGroup extends ComponentMetricGroup {
protected OperatorMetricGroup(MetricRegistry registry, TaskMetricGroup task, String name, int subTaskIndex) {
super(registry, task, registry.getScopeConfig().getOperatorFormat());
+
this.formats.put(SCOPE_OPERATOR_NAME, name);
- this.formats.put(SCOPE_OPERATOR_SUBTASK_INDEX, "" + subTaskIndex);
+ this.formats.put(SCOPE_OPERATOR_SUBTASK_INDEX, String.valueOf(subTaskIndex));
}
+ // ------------------------------------------------------------------------
+
@Override
protected String getScopeFormat(Scope.ScopeFormat format) {
return format.getOperatorFormat();
}
+
+ @Override
+ protected Iterable<? extends ComponentMetricGroup> subComponents() {
+ return Collections.emptyList();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java
index 47bae37..83013e2 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/Scope.java
@@ -73,13 +73,12 @@ public class Scope {
* Simple container for component scope format strings.
*/
public static class ScopeFormat {
+
private String operatorFormat = OperatorMetricGroup.DEFAULT_SCOPE_OPERATOR;
private String taskFormat = TaskMetricGroup.DEFAULT_SCOPE_TASK;
private String jobFormat = JobMetricGroup.DEFAULT_SCOPE_JOB;
private String taskManagerFormat = TaskManagerMetricGroup.DEFAULT_SCOPE_TM;
-
- public ScopeFormat() {
- }
+
public ScopeFormat setOperatorFormat(String format) {
this.operatorFormat = format;
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java
index e199ca7..bfb9362 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskManagerMetricGroup.java
@@ -15,23 +15,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.groups;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.util.AbstractID;
import java.util.HashMap;
import java.util.Map;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Special {@link org.apache.flink.metrics.MetricGroup} representing a TaskManager.
- * <p<
- * Contains extra logic for adding jobs.
+ *
+ * <p>Contains extra logic for adding jobs with tasks, and removing jobs when they do
+ * not contain tasks any more
*/
@Internal
public class TaskManagerMetricGroup extends ComponentMetricGroup {
+
public static final String SCOPE_HOST_DESCRIPTOR = "host";
public static final String SCOPE_TM_DESCRIPTOR = "taskmanager";
public static final String SCOPE_TM_HOST = Scope.format("host");
@@ -39,32 +44,87 @@ public class TaskManagerMetricGroup extends ComponentMetricGroup {
public static final String DEFAULT_SCOPE_TM_COMPONENT = Scope.concat(SCOPE_TM_HOST, "taskmanager", SCOPE_TM_ID);
public static final String DEFAULT_SCOPE_TM = DEFAULT_SCOPE_TM_COMPONENT;
- private Map<JobID, JobMetricGroup> jobs = new HashMap<>();
+ // ------------------------------------------------------------------------
+
+ private final Map<JobID, JobMetricGroup> jobs = new HashMap<>();
- public TaskManagerMetricGroup(MetricRegistry registry, String host, String id) {
+ public TaskManagerMetricGroup(MetricRegistry registry, String host, String taskManagerId) {
super(registry, null, registry.getScopeConfig().getTaskManagerFormat());
- this.formats.put(SCOPE_TM_HOST, host);
- this.formats.put(SCOPE_TM_ID, id);
- }
- public JobMetricGroup addJob(JobID id, String name) {
- JobMetricGroup task = new JobMetricGroup(this.registry, this, id, name);
- jobs.put(id, task);
- return task;
+ this.formats.put(SCOPE_TM_HOST, checkNotNull(host));
+ this.formats.put(SCOPE_TM_ID, checkNotNull(taskManagerId));
}
- @Override
- public void close() {
- super.close();
- for (MetricGroup group : jobs.values()) {
- group.close();
+ // ------------------------------------------------------------------------
+ // job groups
+ // ------------------------------------------------------------------------
+
+ public TaskMetricGroup addTaskForJob(
+ JobID jobId,
+ String jobName,
+ AbstractID vertexID,
+ AbstractID executionId,
+ int subtaskIndex,
+ String taskName) {
+
+ // we cannot strictly lock both our map modification and the job group modification
+ // because it might lead to a deadlock
+ while (true) {
+ // get or create a jobs metric group
+ JobMetricGroup currentJobGroup;
+ synchronized (this) {
+ currentJobGroup = jobs.get(jobId);
+
+ if (currentJobGroup == null || currentJobGroup.isClosed()) {
+ currentJobGroup = new JobMetricGroup(registry, this, jobId, jobName);
+ jobs.put(jobId, currentJobGroup);
+ }
+ }
+
+ // try to add another task. this may fail if we found a pre-existing job metrics
+ // group and it is closed concurrently
+ TaskMetricGroup taskGroup = currentJobGroup.addTask(vertexID, executionId, subtaskIndex, taskName);
+ if (taskGroup != null) {
+ // successfully added the next task
+ return taskGroup;
+ }
+
+ // else fall through the loop
+ }
+ }
+
+ public void removeJobMetricsGroup(JobID jobId, JobMetricGroup group) {
+ if (jobId == null || group == null || !group.isClosed()) {
+ return;
+ }
+
+ synchronized (this) {
+ // optimistically remove the currently contained group, and check later if it was correct
+ JobMetricGroup containedGroup = jobs.remove(jobId);
+
+ // check if another group was actually contained, and restore that one
+ if (containedGroup != null && containedGroup != group) {
+ jobs.put(jobId, containedGroup);
+ }
}
- jobs.clear();
}
+ public int numRegisteredJobMetricGroups() {
+ return jobs.size();
+ }
+
+ // ------------------------------------------------------------------------
+ // component group behavior
+ // ------------------------------------------------------------------------
+
@Override
protected String getScopeFormat(Scope.ScopeFormat format) {
return format.getTaskManagerFormat();
}
+
+ @Override
+ protected Iterable<? extends ComponentMetricGroup> subComponents() {
+ return jobs.values();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java
index 4f8e010..3238534 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/groups/TaskMetricGroup.java
@@ -15,10 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.groups;
import org.apache.flink.annotation.Internal;
-import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.MetricRegistry;
import org.apache.flink.util.AbstractID;
@@ -26,14 +26,16 @@ import java.util.HashMap;
import java.util.Map;
import static org.apache.flink.metrics.groups.JobMetricGroup.DEFAULT_SCOPE_JOB;
+import static org.apache.flink.util.Preconditions.checkNotNull;
/**
- * Special {@link org.apache.flink.metrics.MetricGroup} representing a Task.
- * <p<
- * Contains extra logic for adding operators.
+ * Special {@link org.apache.flink.metrics.MetricGroup} representing a Flink runtime Task.
+ *
+ * <p>Contains extra logic for adding operators.
*/
@Internal
public class TaskMetricGroup extends ComponentMetricGroup {
+
public static final String SCOPE_TASK_DESCRIPTOR = "task";
public static final String SCOPE_TASK_ID = Scope.format("task_id");
public static final String SCOPE_TASK_NAME = Scope.format("task_name");
@@ -41,34 +43,56 @@ public class TaskMetricGroup extends ComponentMetricGroup {
public static final String SCOPE_TASK_SUBTASK_INDEX = Scope.format("subtask_index");
public static final String DEFAULT_SCOPE_TASK_COMPONENT = SCOPE_TASK_NAME;
public static final String DEFAULT_SCOPE_TASK = Scope.concat(DEFAULT_SCOPE_JOB, DEFAULT_SCOPE_TASK_COMPONENT);
+
+
+ private final Map<String, OperatorMetricGroup> operators = new HashMap<>();
+
+ private final IOMetricGroup ioMetrics;
+
+ private final AbstractID executionId;
+
private final int subtaskIndex;
- private Map<String, OperatorMetricGroup> operators = new HashMap<>();
- private IOMetricGroup ioMetrics;
+ protected TaskMetricGroup(
+ MetricRegistry registry,
+ JobMetricGroup parent,
+ AbstractID taskId,
+ AbstractID executionId,
+ int subtaskIndex,
+ String name) {
+
+ super(registry, parent, registry.getScopeConfig().getTaskFormat());
- protected TaskMetricGroup(MetricRegistry registry, JobMetricGroup job, AbstractID id, AbstractID attemptID, int subtaskIndex, String name) {
- super(registry, job, registry.getScopeConfig().getTaskFormat());
- this.formats.put(SCOPE_TASK_ID, id.toString());
- this.formats.put(SCOPE_TASK_ATTEMPT, attemptID.toString());
- this.formats.put(SCOPE_TASK_NAME, name);
- this.formats.put(SCOPE_TASK_SUBTASK_INDEX, "" + subtaskIndex);
+ this.executionId = executionId;
this.subtaskIndex = subtaskIndex;
this.ioMetrics = new IOMetricGroup(registry, this);
+
+ this.formats.put(SCOPE_TASK_ID, taskId.toString());
+ this.formats.put(SCOPE_TASK_ATTEMPT, executionId.toString());
+ this.formats.put(SCOPE_TASK_NAME, checkNotNull(name));
+ this.formats.put(SCOPE_TASK_SUBTASK_INDEX, String.valueOf(subtaskIndex));
}
- @Override
- public void close() {
- super.close();
- for (MetricGroup group : operators.values()) {
- group.close();
+ public OperatorMetricGroup addOperator(String name) {
+ OperatorMetricGroup operator = new OperatorMetricGroup(this.registry, this, name, this.subtaskIndex);
+
+ synchronized (this) {
+ OperatorMetricGroup previous = operators.put(name, operator);
+ if (previous == null) {
+ // no operator group so far
+ return operator;
+ } else {
+ // already had an operator group. restore that one.
+ operators.put(name, previous);
+ return previous;
+ }
}
- operators.clear();
}
- public OperatorMetricGroup addOperator(String name) {
- OperatorMetricGroup operator = new OperatorMetricGroup(this.registry, this, name, this.subtaskIndex);
- operators.put(name, operator);
- return operator;
+ @Override
+ public void close() {
+ super.close();
+ parent().removeTaskMetricGroup(executionId);
}
/**
@@ -81,7 +105,17 @@ public class TaskMetricGroup extends ComponentMetricGroup {
}
@Override
+ protected JobMetricGroup parent() {
+ return (JobMetricGroup) super.parent();
+ }
+
+ @Override
protected String getScopeFormat(Scope.ScopeFormat format) {
return format.getTaskFormat();
}
+
+ @Override
+ protected Iterable<? extends ComponentMetricGroup> subComponents() {
+ return operators.values();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java b/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java
index 43f09b2..1fe55df 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/reporter/AbstractReporter.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.metrics.reporter;
import org.apache.flink.hadoop.shaded.org.jboss.netty.util.internal.ConcurrentHashMap;
@@ -25,7 +26,8 @@ import org.apache.flink.metrics.Metric;
import java.util.Map;
public abstract class AbstractReporter implements MetricReporter {
- protected Map<String, Gauge> gauges = new ConcurrentHashMap<>();
+
+ protected Map<String, Gauge<?>> gauges = new ConcurrentHashMap<>();
protected Map<String, Counter> counters = new ConcurrentHashMap<>();
@Override
@@ -33,7 +35,7 @@ public abstract class AbstractReporter implements MetricReporter {
if (metric instanceof Counter) {
counters.put(name, (Counter) metric);
} else if (metric instanceof Gauge) {
- gauges.put(name, (Gauge) metric);
+ gauges.put(name, (Gauge<?>) metric);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java b/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java
index 0b2efe4..69fa11a 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/reporter/JMXReporter.java
@@ -120,7 +120,7 @@ public class JMXReporter implements MetricReporter {
fullName.append(value);
fullName.append(",");
}
- fullName.append("name=" + name);
+ fullName.append("name=").append(name);
return fullName.toString();
}
@@ -153,9 +153,10 @@ public class JMXReporter implements MetricReporter {
}
private static class JmxGauge extends AbstractBean implements JmxGaugeMBean {
- private final Gauge gauge;
+
+ private final Gauge<?> gauge;
- public JmxGauge(Gauge gauge) {
+ public JmxGauge(Gauge<?> gauge) {
this.gauge = gauge;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java b/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java
index 2bca606..f87f5d3 100644
--- a/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java
+++ b/flink-core/src/main/java/org/apache/flink/metrics/reporter/MetricReporter.java
@@ -31,6 +31,7 @@ import java.util.List;
*/
@PublicEvolving
public interface MetricReporter extends Reporter {
+
/**
* Configures this reporter. Since reporters are instantiated generically and hence parameter-less,
* this method is the place where the reporters set their basic fields based on configuration values.
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java
deleted file mode 100644
index f7502e5..0000000
--- a/flink-core/src/test/java/org/apache/flink/metrics/MetricGroupTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.metrics;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.metrics.groups.TaskManagerMetricGroup;
-import org.apache.flink.metrics.util.TestReporter;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class MetricGroupTest {
- /**
- * Verifies that group methods instantiate the correct metric with the given name.
- */
- @Test
- public void testMetricInstantiation() {
- Configuration config = new Configuration();
-
- config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter1.class.getName());
-
- MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
-
- root.counter("counter");
- Assert.assertTrue(TestReporter1.lastPassedMetric instanceof Counter);
- Assert.assertEquals("counter", TestReporter1.lastPassedName);
-
- root.gauge("gauge", new Gauge<Object>() {
- @Override
- public Object getValue() {
- return null;
- }
- });
- Assert.assertTrue(TestReporter1.lastPassedMetric instanceof Gauge);
- Assert.assertEquals("gauge", TestReporter1.lastPassedName);
- }
-
- protected static class TestReporter1 extends TestReporter {
- public static Metric lastPassedMetric;
- public static String lastPassedName;
-
- @Override
- public void notifyOfAddedMetric(Metric metric, String name) {
- lastPassedMetric = metric;
- lastPassedName = name;
- }
-
- @Override
- public void notifyOfRemovedMetric(Metric metric, String name) {
- }
- }
-
- /**
- * Verifies that metric names containing special characters are rejected.
- */
- @Test(expected = IllegalArgumentException.class)
- public void testInvalidMetricName() {
- Configuration config = new Configuration();
-
- MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
- root.counter("=)(/!");
- }
-
- /**
- * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
- */
- @Test
- public void testDuplicateGroupName() {
- Configuration config = new Configuration();
-
- MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
-
- MetricGroup group1 = root.addGroup("group");
- MetricGroup group2 = root.addGroup("group");
- MetricGroup group3 = root.addGroup("group");
- Assert.assertTrue(group1 == group2 && group2 == group3);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java
index 89483b3..cea11fb 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/JobGroupTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.metrics.groups;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.util.AbstractID;
import org.junit.Test;
import java.util.List;
@@ -30,10 +31,12 @@ public class JobGroupTest {
@Test
public void testGenerateScopeDefault() {
MetricRegistry registry = new MetricRegistry(new Configuration());
- JobMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job");
- List<String> scope = operator.generateScope();
+ TaskMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "host", "id")
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task");
+ JobMetricGroup jmGroup = tmGroup.parent();
+
+ List<String> scope = jmGroup.generateScope();
assertEquals(4, scope.size());
assertEquals("job", scope.get(3));
}
@@ -41,13 +44,15 @@ public class JobGroupTest {
@Test
public void testGenerateScopeWildcard() {
MetricRegistry registry = new MetricRegistry(new Configuration());
- JobMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job");
+
+ TaskMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "host", "id")
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task");
+ JobMetricGroup jmGroup = tmGroup.parent();
Scope.ScopeFormat format = new Scope.ScopeFormat();
format.setJobFormat(Scope.concat(Scope.SCOPE_WILDCARD, "superjob", JobMetricGroup.SCOPE_JOB_NAME));
- List<String> scope = operator.generateScope(format);
+ List<String> scope = jmGroup.generateScope(format);
assertEquals(5, scope.size());
assertEquals("superjob", scope.get(3));
assertEquals("job", scope.get(4));
@@ -56,13 +61,15 @@ public class JobGroupTest {
@Test
public void testGenerateScopeCustom() {
MetricRegistry registry = new MetricRegistry(new Configuration());
- JobMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job");
+
+ TaskMetricGroup tmGroup = new TaskManagerMetricGroup(registry, "host", "id")
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task");
+ JobMetricGroup jmGroup = tmGroup.parent();
Scope.ScopeFormat format = new Scope.ScopeFormat();
format.setJobFormat(Scope.concat(TaskManagerMetricGroup.SCOPE_TM_HOST, "superjob", JobMetricGroup.SCOPE_JOB_NAME));
- List<String> scope = operator.generateScope(format);
+ List<String> scope = jmGroup.generateScope(format);
assertEquals(3, scope.size());
assertEquals("host", scope.get(0));
assertEquals("superjob", scope.get(1));
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupRegistrationTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupRegistrationTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupRegistrationTest.java
new file mode 100644
index 0000000..cb962ea
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupRegistrationTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
+import org.apache.flink.metrics.util.TestReporter;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class MetricGroupRegistrationTest {
+ /**
+ * Verifies that group methods instantiate the correct metric with the given name.
+ */
+ @Test
+ public void testMetricInstantiation() {
+ Configuration config = new Configuration();
+ config.setString(MetricRegistry.KEY_METRICS_REPORTER_CLASS, TestReporter1.class.getName());
+
+ MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
+
+ Counter counter = root.counter("counter");
+ assertEquals(counter, TestReporter1.lastPassedMetric);
+ assertEquals("counter", TestReporter1.lastPassedName);
+
+ Gauge<Object> gauge = root.gauge("gauge", new Gauge<Object>() {
+ @Override
+ public Object getValue() {
+ return null;
+ }
+ });
+
+ Assert.assertEquals(gauge, TestReporter1.lastPassedMetric);
+ assertEquals("gauge", TestReporter1.lastPassedName);
+ }
+
+ public static class TestReporter1 extends TestReporter {
+
+ public static Metric lastPassedMetric;
+ public static String lastPassedName;
+
+ @Override
+ public void notifyOfAddedMetric(Metric metric, String name) {
+ lastPassedMetric = metric;
+ lastPassedName = name;
+ }
+
+ @Override
+ public void notifyOfRemovedMetric(Metric metric, String name) {}
+ }
+
+ /**
+ * Verifies that metric names containing special characters are rejected.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalidMetricName() {
+ Configuration config = new Configuration();
+
+ MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
+ root.counter("=)(/!");
+ }
+
+ /**
+ * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
+ */
+ @Test
+ public void testDuplicateGroupName() {
+ Configuration config = new Configuration();
+
+ MetricGroup root = new TaskManagerMetricGroup(new MetricRegistry(config), "host", "id");
+
+ MetricGroup group1 = root.addGroup("group");
+ MetricGroup group2 = root.addGroup("group");
+ MetricGroup group3 = root.addGroup("group");
+ Assert.assertTrue(group1 == group2 && group2 == group3);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupTest.java
new file mode 100644
index 0000000..31ab2a7
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/MetricGroupTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.metrics.groups;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.MetricRegistry;
+
+import org.apache.flink.metrics.util.DummyMetricGroup;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class MetricGroupTest {
+
+ private final MetricRegistry registry = new MetricRegistry(new Configuration());
+
+ private final MetricRegistry exceptionOnRegister = new ExceptionOnRegisterRegistry();
+
+ @Test
+ public void sameGroupOnNameCollision() {
+ GenericMetricGroup group = new GenericMetricGroup(registry, new DummyMetricGroup(registry), "somegroup");
+
+ String groupName = "sometestname";
+ MetricGroup subgroup1 = group.addGroup(groupName);
+ MetricGroup subgroup2 = group.addGroup(groupName);
+
+ assertNotNull(subgroup1);
+ assertNotNull(subgroup2);
+ assertTrue(subgroup1 == subgroup2);
+ }
+
+ @Test
+ public void closedGroupDoesNotRegisterMetrics() {
+ GenericMetricGroup group = new GenericMetricGroup(
+ exceptionOnRegister, new DummyMetricGroup(exceptionOnRegister), "testgroup");
+ assertFalse(group.isClosed());
+
+ group.close();
+ assertTrue(group.isClosed());
+
+ // these will fail is the registration is propagated
+ group.counter("testcounter");
+ group.gauge("testgauge", new Gauge<Object>() {
+ @Override
+ public Object getValue() { return null; }
+ });
+ }
+
+ @Test
+ public void closedGroupCreatesClosedGroups() {
+ GenericMetricGroup group = new GenericMetricGroup(exceptionOnRegister,
+ new DummyMetricGroup(exceptionOnRegister), "testgroup");
+ assertFalse(group.isClosed());
+
+ group.close();
+ assertTrue(group.isClosed());
+
+ MetricGroup subgroup = group.addGroup("test subgroup");
+ assertTrue(subgroup.isClosed());
+ }
+
+ @Test
+ public void tolerateMetricNameCollisions() {
+ final String name = "abctestname";
+ GenericMetricGroup group = new GenericMetricGroup(
+ registry, new DummyMetricGroup(registry), "testgroup");
+
+ assertNotNull(group.counter(name));
+ assertNotNull(group.counter(name));
+ }
+
+ @Test
+ public void tolerateMetricAndGroupNameCollisions() {
+ final String name = "abctestname";
+ GenericMetricGroup group = new GenericMetricGroup(registry, new DummyMetricGroup(registry), "testgroup");
+
+ assertNotNull(group.addGroup(name));
+ assertNotNull(group.counter(name));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void exceptionOnIllegalName() {
+ GenericMetricGroup group = new GenericMetricGroup(
+ exceptionOnRegister, new DummyMetricGroup(exceptionOnRegister), "testgroup");
+ group.counter("�berC�unter");
+ }
+
+ // ------------------------------------------------------------------------
+
+ private static class ExceptionOnRegisterRegistry extends MetricRegistry {
+
+ public ExceptionOnRegisterRegistry() {
+ super(new Configuration());
+ }
+
+ @Override
+ public void register(Metric metric, String name, AbstractMetricGroup parent) {
+ fail("Metric should never be registered");
+ }
+
+ @Override
+ public void unregister(Metric metric, String name, AbstractMetricGroup parent) {
+ fail("Metric should never be un-registered");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java
index 4f33d2a..6202834 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/OperatorGroupTest.java
@@ -31,9 +31,10 @@ public class OperatorGroupTest {
@Test
public void testGenerateScopeDefault() {
MetricRegistry registry = new MetricRegistry(new Configuration());
+
OperatorMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job")
- .addTask(new AbstractID(), new AbstractID(), 0, "task").addOperator("operator");
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task")
+ .addOperator("operator");
List<String> scope = operator.generateScope();
assertEquals(6, scope.size());
@@ -49,8 +50,8 @@ public class OperatorGroupTest {
public void testGenerateScopeWildcard() {
MetricRegistry registry = new MetricRegistry(new Configuration());
OperatorMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job")
- .addTask(new AbstractID(), new AbstractID(), 0, "task").addOperator("operator");
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task")
+ .addOperator("operator");
Scope.ScopeFormat format = new Scope.ScopeFormat();
format.setOperatorFormat(Scope.concat(Scope.SCOPE_WILDCARD, "op", OperatorMetricGroup.SCOPE_OPERATOR_NAME));
@@ -70,8 +71,8 @@ public class OperatorGroupTest {
public void testGenerateScopeCustom() {
MetricRegistry registry = new MetricRegistry(new Configuration());
OperatorMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job")
- .addTask(new AbstractID(), new AbstractID(), 0, "task").addOperator("operator");
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task")
+ .addOperator("operator");
Scope.ScopeFormat format = new Scope.ScopeFormat();
format.setOperatorFormat(Scope.concat("jobs", JobMetricGroup.SCOPE_JOB_NAME, "op", OperatorMetricGroup.SCOPE_OPERATOR_NAME));
http://git-wip-us.apache.org/repos/asf/flink/blob/707606ac/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java
index c49fdcd..69faee7 100644
--- a/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java
+++ b/flink-core/src/test/java/org/apache/flink/metrics/groups/TaskGroupTest.java
@@ -28,12 +28,17 @@ import java.util.List;
import static org.junit.Assert.assertEquals;
public class TaskGroupTest {
+
+ // ------------------------------------------------------------------------
+ // scope tests
+ // ------------------------------------------------------------------------
+
@Test
public void testGenerateScopeDefault() {
MetricRegistry registry = new MetricRegistry(new Configuration());
+
TaskMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job")
- .addTask(new AbstractID(), new AbstractID(), 0, "task");
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task");
List<String> scope = operator.generateScope();
assertEquals(5, scope.size());
@@ -44,8 +49,7 @@ public class TaskGroupTest {
public void testGenerateScopeWilcard() {
MetricRegistry registry = new MetricRegistry(new Configuration());
TaskMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job")
- .addTask(new AbstractID(), new AbstractID(), 0, "task");
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task");
Scope.ScopeFormat format = new Scope.ScopeFormat();
format.setTaskFormat(Scope.concat(Scope.SCOPE_WILDCARD, "supertask", TaskMetricGroup.SCOPE_TASK_NAME));
@@ -64,8 +68,7 @@ public class TaskGroupTest {
public void testGenerateScopeCustom() {
MetricRegistry registry = new MetricRegistry(new Configuration());
TaskMetricGroup operator = new TaskManagerMetricGroup(registry, "host", "id")
- .addJob(new JobID(), "job")
- .addTask(new AbstractID(), new AbstractID(), 0, "task");
+ .addTaskForJob(new JobID(), "job", new AbstractID(), new AbstractID(), 0, "task");
Scope.ScopeFormat format = new Scope.ScopeFormat();
format.setTaskFormat(Scope.concat(TaskManagerMetricGroup.SCOPE_TM_HOST, JobMetricGroup.SCOPE_JOB_NAME, "supertask", TaskMetricGroup.SCOPE_TASK_NAME));
[3/6] flink git commit: [FLINK-1502] [core] Add basic metric system
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index 3ee9a84..b3ad589 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -162,7 +162,7 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid = new ExecutionAttemptID();
final SerializedValue<ExecutionConfig> executionConfig = ExecutionConfigTest.getSerializedConfig();
- final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, vid, eid, executionConfig,
+ final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, "TestJob", vid, eid, executionConfig,
"TestTask", 2, 7, 0, new Configuration(), new Configuration(),
TestInvokableCorrect.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -263,14 +263,14 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid1 = new ExecutionAttemptID();
final ExecutionAttemptID eid2 = new ExecutionAttemptID();
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, vid1, eid1,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, "TestJob1", vid1, eid1,
ExecutionConfigTest.getSerializedConfig(), "TestTask1", 1, 5, 0,
new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, vid2, eid2,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, "TestJob2", vid2, eid2,
ExecutionConfigTest.getSerializedConfig(), "TestTask2", 2, 7, 0,
new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -399,13 +399,13 @@ public class TaskManagerTest extends TestLogger {
final SerializedValue<ExecutionConfig> executionConfig = ExecutionConfigTest.getSerializedConfig();
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, vid1, eid1, executionConfig,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid1, "TestJob", vid1, eid1, executionConfig,
"TestTask1", 1, 5, 0, new Configuration(), new Configuration(), StoppableInvokable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, vid2, eid2, executionConfig,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid2, "TestJob", vid2, eid2, executionConfig,
"TestTask2", 2, 7, 0, new Configuration(), new Configuration(), TestInvokableBlockingCancelable.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
@@ -525,14 +525,14 @@ public class TaskManagerTest extends TestLogger {
final ExecutionAttemptID eid1 = new ExecutionAttemptID();
final ExecutionAttemptID eid2 = new ExecutionAttemptID();
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, vid1, eid1,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, "TestJob", vid1, eid1,
ExecutionConfigTest.getSerializedConfig(), "Sender", 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, vid2, eid2,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, "TestJob", vid2, eid2,
ExecutionConfigTest.getSerializedConfig(), "Receiver", 2, 7, 0,
new Configuration(), new Configuration(), Tasks.Receiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -626,13 +626,13 @@ public class TaskManagerTest extends TestLogger {
}
);
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, vid1, eid1,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, "TestJob", vid1, eid1,
ExecutionConfigTest.getSerializedConfig(), "Sender", 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
irpdd, Collections.<InputGateDeploymentDescriptor>emptyList(), new ArrayList<BlobKey>(),
Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, vid2, eid2,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, "TestJob", vid2, eid2,
ExecutionConfigTest.getSerializedConfig(), "Receiver", 2, 7, 0,
new Configuration(), new Configuration(), Tasks.Receiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -767,13 +767,13 @@ public class TaskManagerTest extends TestLogger {
}
);
- final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, vid1, eid1,
+ final TaskDeploymentDescriptor tdd1 = new TaskDeploymentDescriptor(jid, "TestJob", vid1, eid1,
ExecutionConfigTest.getSerializedConfig(), "Sender", 0, 1, 0,
new Configuration(), new Configuration(), Tasks.Sender.class.getName(),
irpdd, Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), Collections.<URL>emptyList(), 0);
- final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, vid2, eid2,
+ final TaskDeploymentDescriptor tdd2 = new TaskDeploymentDescriptor(jid, "TestJob", vid2, eid2,
ExecutionConfigTest.getSerializedConfig(), "Receiver", 2, 7, 0,
new Configuration(), new Configuration(), Tasks.BlockingReceiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -912,7 +912,7 @@ public class TaskManagerTest extends TestLogger {
new InputGateDeploymentDescriptor(resultId, 0, icdd);
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- jid, vid, eid,
+ jid, "TestJob", vid, eid,
ExecutionConfigTest.getSerializedConfig(), "Receiver", 0, 1, 0,
new Configuration(), new Configuration(),
Tasks.AgnosticReceiver.class.getName(),
@@ -1007,7 +1007,7 @@ public class TaskManagerTest extends TestLogger {
new InputGateDeploymentDescriptor(resultId, 0, icdd);
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- jid, vid, eid, ExecutionConfigTest.getSerializedConfig(), "Receiver", 0, 1, 0,
+ jid, "TestJob", vid, eid, ExecutionConfigTest.getSerializedConfig(), "Receiver", 0, 1, 0,
new Configuration(), new Configuration(),
Tasks.AgnosticReceiver.class.getName(),
Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
@@ -1081,6 +1081,7 @@ public class TaskManagerTest extends TestLogger {
// Single blocking task
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
new JobID(),
+ "Job",
new JobVertexID(),
new ExecutionAttemptID(),
ExecutionConfigTest.getSerializedConfig(),
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
index 99e037d..7660893 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskStopTest.java
@@ -21,6 +21,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
import org.apache.flink.runtime.execution.ExecutionState;
@@ -69,7 +70,7 @@ public class TaskStopTest {
task = new Task(tddMock, mock(MemoryManager.class), mock(IOManager.class), mock(NetworkEnvironment.class),
mock(BroadcastVariableManager.class), mock(ActorGateway.class), mock(ActorGateway.class),
mock(FiniteDuration.class), mock(LibraryCacheManager.class), mock(FileCache.class),
- mock(TaskManagerRuntimeInfo.class));
+ mock(TaskManagerRuntimeInfo.class), mock(TaskMetricGroup.class));
Field f = task.getClass().getDeclaredField("invokable");
f.setAccessible(true);
f.set(task, taskMock);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index 06f393f..f237c87 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -22,6 +22,7 @@ import com.google.common.collect.Maps;
import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
@@ -622,12 +623,13 @@ public class TaskTest {
new FiniteDuration(60, TimeUnit.SECONDS),
libCache,
mock(FileCache.class),
- new TaskManagerRuntimeInfo("localhost", new Configuration()));
+ new TaskManagerRuntimeInfo("localhost", new Configuration()),
+ mock(TaskMetricGroup.class));
}
private TaskDeploymentDescriptor createTaskDeploymentDescriptor(Class<? extends AbstractInvokable> invokable) {
return new TaskDeploymentDescriptor(
- new JobID(), new JobVertexID(), new ExecutionAttemptID(),
+ new JobID(), "Test Job", new JobVertexID(), new ExecutionAttemptID(),
ExecutionConfigTest.getSerializedConfig(),
"Test Task", 0, 1, 0,
new Configuration(), new Configuration(),
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
index c9b1d38..783b3e2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
@@ -53,6 +53,7 @@ public class StreamConfig implements Serializable {
private static final String CHAINED_OUTPUTS = "chainedOutputs";
private static final String CHAINED_TASK_CONFIG = "chainedTaskConfig_";
private static final String IS_CHAINED_VERTEX = "isChainedSubtask";
+ private static final String CHAIN_INDEX = "chainIndex";
private static final String VERTEX_NAME = "vertexID";
private static final String ITERATION_ID = "iterationId";
private static final String OUTPUT_SELECTOR_WRAPPER = "outputSelectorWrapper";
@@ -387,6 +388,14 @@ public class StreamConfig implements Serializable {
throw new StreamTaskException("Could not instantiate configuration.", e);
}
}
+
+ public void setChainIndex(int index) {
+ this.config.setInteger(CHAIN_INDEX, index);
+ }
+
+ public int getChainIndex() {
+ return this.config.getInteger(CHAIN_INDEX, 0);
+ }
// ------------------------------------------------------------------------
// State backend
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 9adabae..71cc7f2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -167,14 +167,15 @@ public class StreamingJobGraphGenerator {
*/
private void setChaining(Map<Integer, byte[]> hashes) {
for (Integer sourceNodeId : streamGraph.getSourceIDs()) {
- createChain(sourceNodeId, sourceNodeId, hashes);
+ createChain(sourceNodeId, sourceNodeId, hashes, 0);
}
}
private List<StreamEdge> createChain(
Integer startNodeId,
Integer currentNodeId,
- Map<Integer, byte[]> hashes) {
+ Map<Integer, byte[]> hashes,
+ int chainIndex) {
if (!builtVertices.contains(startNodeId)) {
@@ -192,12 +193,12 @@ public class StreamingJobGraphGenerator {
}
for (StreamEdge chainable : chainableOutputs) {
- transitiveOutEdges.addAll(createChain(startNodeId, chainable.getTargetId(), hashes));
+ transitiveOutEdges.addAll(createChain(startNodeId, chainable.getTargetId(), hashes, chainIndex + 1));
}
for (StreamEdge nonChainable : nonChainableOutputs) {
transitiveOutEdges.add(nonChainable);
- createChain(nonChainable.getTargetId(), nonChainable.getTargetId(), hashes);
+ createChain(nonChainable.getTargetId(), nonChainable.getTargetId(), hashes, 0);
}
chainedNames.put(currentNodeId, createChainedName(currentNodeId, chainableOutputs));
@@ -211,6 +212,7 @@ public class StreamingJobGraphGenerator {
if (currentNodeId.equals(startNodeId)) {
config.setChainStart();
+ config.setChainIndex(0);
config.setOutEdgesInOrder(transitiveOutEdges);
config.setOutEdges(streamGraph.getStreamNode(currentNodeId).getOutEdges());
@@ -227,6 +229,7 @@ public class StreamingJobGraphGenerator {
if (chainedConfs == null) {
chainedConfigs.put(startNodeId, new HashMap<Integer, StreamConfig>());
}
+ config.setChainIndex(chainIndex);
chainedConfigs.get(startNodeId).put(currentNodeId, config);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 9673f87..326a42f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.state.StateDescriptor;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.base.VoidSerializer;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.runtime.state.KvStateSnapshot;
import org.apache.flink.runtime.state.AbstractStateBackend;
@@ -91,6 +92,7 @@ public abstract class AbstractStreamOperator<OUT>
/** The state backend that stores the state and checkpoints for this task */
private AbstractStateBackend stateBackend = null;
+ protected MetricGroup metrics;
// ------------------------------------------------------------------------
// Life Cycle
@@ -101,6 +103,9 @@ public abstract class AbstractStreamOperator<OUT>
this.container = containingTask;
this.config = config;
this.output = output;
+ String operatorName = containingTask.getEnvironment().getTaskInfo().getTaskName().split("->")[config.getChainIndex()].trim();
+
+ this.metrics = container.getEnvironment().getMetricGroup().addOperator(operatorName);
this.runtimeContext = new StreamingRuntimeContext(this, container.getEnvironment(), container.getAccumulatorMap());
stateKeySelector1 = config.getStatePartitioner(0, getUserCodeClassloader());
@@ -116,6 +121,10 @@ public abstract class AbstractStreamOperator<OUT>
throw new RuntimeException("Could not initialize state backend. ", e);
}
}
+
+ public MetricGroup getMetricGroup() {
+ return metrics;
+ }
/**
* This method is called immediately before any elements are processed, it should contain the
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index ca28a6a..69e920f 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.api.operators;
import java.io.Serializable;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.StreamTask;
@@ -149,4 +150,6 @@ public interface StreamOperator<OUT> extends Serializable {
ChainingStrategy getChainingStrategy();
void setChainingStrategy(ChainingStrategy strategy);
+
+ MetricGroup getMetricGroup();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
index 4500ee7..5fef3c7 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamingRuntimeContext.java
@@ -64,7 +64,8 @@ public class StreamingRuntimeContext extends AbstractRuntimeUDFContext {
env.getUserClassLoader(),
operator.getExecutionConfig(),
accumulators,
- env.getDistributedCacheEntries());
+ env.getDistributedCacheEntries(),
+ operator.getMetricGroup());
this.operator = operator;
this.taskEnvironment = env;
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
index 8c0b63f..1dde85b 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.event.AbstractEvent;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
@@ -202,6 +203,17 @@ public class StreamInputProcessor<IN> {
deserializer.setReporter(reporter);
}
}
+
+ /**
+ * Sets the metric group for this StreamInputProcessor.
+ *
+ * @param metrics metric group
+ */
+ public void setMetricGroup(IOMetricGroup metrics) {
+ for (RecordDeserializer<?> deserializer : recordDeserializers) {
+ deserializer.instantiateMetrics(metrics);
+ }
+ }
public void cleanup() throws IOException {
// clear the buffers first. this part should not ever fail
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
index 4adea3b..07ada23 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
@@ -20,6 +20,7 @@ package org.apache.flink.streaming.runtime.io;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.event.AbstractEvent;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
@@ -278,6 +279,17 @@ public class StreamTwoInputProcessor<IN1, IN2> {
deserializer.setReporter(reporter);
}
}
+
+ /**
+ * Sets the metric group for this StreamTwoInputProcessor.
+ *
+ * @param metrics metric group
+ */
+ public void setMetricGroup(IOMetricGroup metrics) {
+ for (RecordDeserializer<?> deserializer : recordDeserializers) {
+ deserializer.instantiateMetrics(metrics);
+ }
+ }
public void cleanup() throws IOException {
// clear the buffers first. this part should not ever fail
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
index a1ed828..938d8c1 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
@@ -52,6 +52,7 @@ public class OneInputStreamTask<IN, OUT> extends StreamTask<OUT, OneInputStreamO
AccumulatorRegistry registry = getEnvironment().getAccumulatorRegistry();
AccumulatorRegistry.Reporter reporter = registry.getReadWriteReporter();
inputProcessor.setReporter(reporter);
+ inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index f3d3482..90abea4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -286,6 +286,7 @@ public class OperatorChain<OUT> {
StreamRecordWriter<SerializationDelegate<StreamRecord<T>>> output =
new StreamRecordWriter<>(bufferWriter, outputPartitioner, upStreamConfig.getBufferTimeout());
output.setReporter(reporter);
+ output.setMetricGroup(taskEnvironment.getMetricGroup().getIOMetricGroup());
return new RecordWriterOutput<T>(output, outSerializer, withTimestamps);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
index ebc7789..c3305eb 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
@@ -77,6 +77,7 @@ public class TwoInputStreamTask<IN1, IN2, OUT> extends StreamTask<OUT, TwoInputS
AccumulatorRegistry registry = getEnvironment().getAccumulatorRegistry();
AccumulatorRegistry.Reporter reporter = registry.getReadWriteReporter();
this.inputProcessor.setReporter(reporter);
+ inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup());
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
index dff1184..812507f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AccumulatingAlignedProcessingTimeWindowOperatorTest.java
@@ -29,6 +29,7 @@ import org.apache.flink.api.common.typeutils.base.StringSerializer;
import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.util.DummyTaskMetricGroup;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.streaming.api.functions.windowing.RichWindowFunction;
import org.apache.flink.streaming.api.functions.windowing.WindowFunction;
@@ -794,6 +795,7 @@ public class AccumulatingAlignedProcessingTimeWindowOperatorTest {
final Environment env = mock(Environment.class);
when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 0, 1, 0));
when(env.getUserClassLoader()).thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader());
+ when(env.getMetricGroup()).thenReturn(new DummyTaskMetricGroup());
when(task.getEnvironment()).thenReturn(env);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java
index 3429215..c89ac50 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/windowing/AggregatingAlignedProcessingTimeWindowOperatorTest.java
@@ -34,6 +34,7 @@ import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TupleTypeInfo;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.util.DummyTaskMetricGroup;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.Output;
@@ -966,6 +967,7 @@ public class AggregatingAlignedProcessingTimeWindowOperatorTest {
final Environment env = mock(Environment.class);
when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 0, 1, 0));
when(env.getUserClassLoader()).thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader());
+ when(env.getMetricGroup()).thenReturn(new DummyTaskMetricGroup());
when(task.getEnvironment()).thenReturn(env);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
index f91353e..c62c881 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamMockEnvironment.java
@@ -26,6 +26,8 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.core.memory.MemorySegmentFactory;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
+import org.apache.flink.metrics.util.DummyTaskMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.event.AbstractEvent;
@@ -303,5 +305,10 @@ public class StreamMockEnvironment implements Environment {
public TaskManagerRuntimeInfo getTaskManagerInfo() {
return new TaskManagerRuntimeInfo("localhost", new UnmodifiableConfiguration(new Configuration()));
}
+
+ @Override
+ public TaskMetricGroup getMetricGroup() {
+ return new DummyTaskMetricGroup();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
index ed8bf01..deda82f 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java
@@ -23,6 +23,7 @@ import akka.actor.ActorRef;
import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
@@ -134,7 +135,7 @@ public class StreamTaskTest {
when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- new JobID(), new JobVertexID(), new ExecutionAttemptID(),
+ new JobID(), "Job Name", new JobVertexID(), new ExecutionAttemptID(),
ExecutionConfigTest.getSerializedConfig(),
"Test Task", 0, 1, 0,
new Configuration(),
@@ -157,7 +158,8 @@ public class StreamTaskTest {
new FiniteDuration(60, TimeUnit.SECONDS),
libCache,
mock(FileCache.class),
- new TaskManagerRuntimeInfo("localhost", new Configuration()));
+ new TaskManagerRuntimeInfo("localhost", new Configuration()),
+ mock(TaskMetricGroup.class));
}
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1ba237f..c97815a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,6 +74,7 @@ under the License.
<module>flink-quickstart</module>
<module>flink-contrib</module>
<module>flink-dist</module>
+ <module>flink-metrics</module>
</modules>
<properties>
@@ -103,6 +104,7 @@ under the License.
<zookeeper.version>3.4.6</zookeeper.version>
<curator.version>2.8.0</curator.version>
<jackson.version>2.7.4</jackson.version>
+ <metrics.version>3.1.0</metrics.version>
</properties>
<dependencies>
[4/6] flink git commit: [FLINK-1502] [core] Add basic metric system
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-ganglia/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-ganglia/pom.xml b/flink-metrics/flink-metrics-ganglia/pom.xml
new file mode 100644
index 0000000..c4f51da
--- /dev/null
+++ b/flink-metrics/flink-metrics-ganglia/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-ganglia</artifactId>
+ <name>flink-metrics-ganglia</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics-dropwizard</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>info.ganglia.gmetric4j</groupId>
+ <artifactId>gmetric4j</artifactId>
+ <version>1.0.7</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-ganglia</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java b/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
new file mode 100644
index 0000000..a1dafc9
--- /dev/null
+++ b/flink-metrics/flink-metrics-ganglia/src/main/java/org/apache/flink/metrics/graphite/GangliaReporter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.graphite;
+
+import com.codahale.metrics.ScheduledReporter;
+import info.ganglia.gmetric4j.gmetric.GMetric;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.dropwizard.ScheduledDropwizardReporter;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public class GangliaReporter extends ScheduledDropwizardReporter {
+ public static final String ARG_DMAX = "dmax";
+ public static final String ARG_TMAX = "tmax";
+ public static final String ARG_TTL = "ttl";
+ public static final String ARG_MODE_ADDRESSING = "addressingMode";
+
+ @Override
+ public ScheduledReporter getReporter(Configuration config) {
+
+ try {
+ String host = config.getString(ARG_HOST, null);
+ int port = config.getInteger(ARG_PORT, -1);
+ if (host == null || host.length() == 0 || port < 1) {
+ throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
+ }
+ String addressingMode = config.getString(ARG_MODE_ADDRESSING, "MULTICAST");
+ int ttl = config.getInteger(ARG_TTL, -1);
+ GMetric gMetric = new GMetric(host, port, GMetric.UDPAddressingMode.valueOf(addressingMode), ttl);
+
+ String prefix = config.getString(ARG_PREFIX, null);
+ String conversionRate = config.getString(ARG_CONVERSION_RATE, null);
+ String conversionDuration = config.getString(ARG_CONVERSION_DURATION, null);
+ int dMax = config.getInteger(ARG_DMAX, 0);
+ int tMax = config.getInteger(ARG_TMAX, 60);
+
+ com.codahale.metrics.ganglia.GangliaReporter.Builder builder =
+ com.codahale.metrics.ganglia.GangliaReporter.forRegistry(registry);
+
+ if (prefix != null) {
+ builder.prefixedWith(prefix);
+ }
+ if (conversionRate != null) {
+ builder.convertRatesTo(TimeUnit.valueOf(conversionRate));
+ }
+ if (conversionDuration != null) {
+ builder.convertDurationsTo(TimeUnit.valueOf(conversionDuration));
+ }
+ builder.withDMax(dMax);
+ builder.withTMax(tMax);
+
+ return builder.build(gMetric);
+ } catch (IOException e) {
+ throw new RuntimeException("Error while instantiating GangliaReporter.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-graphite/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-graphite/pom.xml b/flink-metrics/flink-metrics-graphite/pom.xml
new file mode 100644
index 0000000..45fb018
--- /dev/null
+++ b/flink-metrics/flink-metrics-graphite/pom.xml
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-graphite</artifactId>
+ <name>flink-metrics-graphite</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics-dropwizard</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-graphite</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java b/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
new file mode 100644
index 0000000..b28d7a4
--- /dev/null
+++ b/flink-metrics/flink-metrics-graphite/src/main/java/org/apache/flink/metrics/graphite/GraphiteReporter.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.graphite;
+
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.graphite.Graphite;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.dropwizard.ScheduledDropwizardReporter;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class GraphiteReporter extends ScheduledDropwizardReporter {
+ @Override
+ public ScheduledReporter getReporter(Configuration config) {
+ String host = config.getString(ARG_HOST, null);
+ int port = config.getInteger(ARG_PORT, -1);
+
+ if (host == null || host.length() == 0 || port < 1) {
+ throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
+ }
+
+ String prefix = config.getString(ARG_PREFIX, null);
+ String conversionRate = config.getString(ARG_CONVERSION_RATE, null);
+ String conversionDuration = config.getString(ARG_CONVERSION_DURATION, null);
+
+ com.codahale.metrics.graphite.GraphiteReporter.Builder builder =
+ com.codahale.metrics.graphite.GraphiteReporter.forRegistry(registry);
+
+ if (prefix != null) {
+ builder.prefixedWith(prefix);
+ }
+
+ if (conversionRate != null) {
+ builder.convertRatesTo(TimeUnit.valueOf(conversionRate));
+ }
+
+ if (conversionDuration != null) {
+ builder.convertDurationsTo(TimeUnit.valueOf(conversionDuration));
+ }
+
+ return builder.build(new Graphite(host, port));
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : scope) {
+ sb.append(s.replace(".", "_").replace("\"", ""));
+ sb.append(".");
+ }
+ sb.append(name);
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-statsd/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-statsd/pom.xml b/flink-metrics/flink-metrics-statsd/pom.xml
new file mode 100644
index 0000000..5d63908
--- /dev/null
+++ b/flink-metrics/flink-metrics-statsd/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-metrics</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics-statsd</artifactId>
+ <name>flink-metrics-statsd</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
----------------------------------------------------------------------
diff --git a/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java b/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
new file mode 100644
index 0000000..288b8b8
--- /dev/null
+++ b/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.metrics.statsd;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.reporter.AbstractReporter;
+import org.apache.flink.metrics.reporter.Scheduled;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.DatagramPacket;
+import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
+import java.net.SocketException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Largely based on the StatsDReporter class by ReadyTalk
+ * https://github.com/ReadyTalk/metrics-statsd/blob/master/metrics3-statsd/src/main/java/com/readytalk/metrics/StatsDReporter.java
+ *
+ * Ported since it was not present in maven central.
+ */
+public class StatsDReporter extends AbstractReporter implements Scheduled {
+ private static final Logger LOG = LoggerFactory.getLogger(StatsDReporter.class);
+
+ public static final String ARG_HOST = "host";
+ public static final String ARG_PORT = "port";
+ public static final String ARG_CONVERSION_RATE = "rateConversion";
+ public static final String ARG_CONVERSION_DURATION = "durationConversion";
+
+ private DatagramSocket socket;
+ private InetSocketAddress address;
+
+ private double durationFactor;
+ private double rateFactor;
+
+ @Override
+ public void open(Configuration config) {
+ String host = config.getString(ARG_HOST, null);
+ int port = config.getInteger(ARG_PORT, -1);
+
+ if (host == null || host.length() == 0 || port < 1) {
+ throw new IllegalArgumentException("Invalid host/port configuration. Host: " + host + " Port: " + port);
+ }
+
+ String conversionRate = config.getString(ARG_CONVERSION_RATE, "SECONDS");
+ String conversionDuration = config.getString(ARG_CONVERSION_DURATION, "MILLISECONDS");
+
+ this.address = new InetSocketAddress(host, port);
+ this.rateFactor = TimeUnit.valueOf(conversionRate).toSeconds(1);
+ this.durationFactor = 1.0 / TimeUnit.valueOf(conversionDuration).toNanos(1);
+ try {
+ this.socket = new DatagramSocket(0);
+ } catch (SocketException e) {
+ throw new RuntimeException("Failure while creating socket. ", e);
+ }
+ }
+
+ @Override
+ public void close() {
+ if (socket != null && !socket.isClosed()) {
+ socket.close();
+ }
+ }
+
+ @Override
+ public String generateName(String name, List<String> scope) {
+ StringBuilder sb = new StringBuilder();
+ for (String s : scope) {
+ sb.append(s);
+ sb.append('.');
+ }
+ sb.append(name);
+ return sb.toString();
+ }
+
+ public void send(final String name, final double value) {
+ send(name, "" + value);
+ }
+
+ public void send(final String name, final String value) {
+ try {
+ String formatted = String.format("%s:%s|g", name, value);
+ byte[] data = formatted.getBytes();
+ socket.send(new DatagramPacket(data, data.length, this.address));
+ } catch (IOException e) {
+ LOG.error("unable to send packet to statsd at '{}:{}'", address.getHostName(), address.getPort());
+ }
+ }
+
+ @Override
+ public void report() {
+ for (Map.Entry<String, Gauge> entry : gauges.entrySet()) {
+ reportGauge(entry.getKey(), entry.getValue());
+ }
+
+ for (Map.Entry<String, Counter> entry : counters.entrySet()) {
+ reportCounter(entry.getKey(), entry.getValue());
+ }
+ }
+
+ private void reportCounter(final String name, final Counter counter) {
+ send(name, counter.getCount());
+ }
+
+ private void reportGauge(final String name, final Gauge<?> gauge) {
+ final String value = gauge.getValue().toString();
+ if (value != null) {
+ send((name), value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/flink-metrics/pom.xml b/flink-metrics/pom.xml
new file mode 100644
index 0000000..542f49c
--- /dev/null
+++ b/flink-metrics/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>1.1-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-metrics</artifactId>
+ <name>flink-metrics</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>flink-metrics-dropwizard</module>
+ <module>flink-metrics-ganglia</module>
+ <module>flink-metrics-graphite</module>
+ <module>flink-metrics-statsd</module>
+ </modules>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 13c4fa5..8007428 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -34,10 +34,6 @@ under the License.
<packaging>jar</packaging>
- <properties>
- <metrics.version>3.1.0</metrics.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
index 2b1c224..d3ebc95 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
@@ -45,6 +45,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
/** The ID of the job the tasks belongs to. */
private final JobID jobID;
+ private final String jobName;
/** The task's job vertex ID. */
private final JobVertexID vertexID;
@@ -99,6 +100,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
*/
public TaskDeploymentDescriptor(
JobID jobID,
+ String jobName,
JobVertexID vertexID,
ExecutionAttemptID executionId,
SerializedValue<ExecutionConfig> serializedExecutionConfig,
@@ -123,6 +125,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
checkArgument(attemptNumber >= 0);
this.jobID = checkNotNull(jobID);
+ this.jobName = checkNotNull(jobName);
this.vertexID = checkNotNull(vertexID);
this.executionId = checkNotNull(executionId);
this.serializedExecutionConfig = checkNotNull(serializedExecutionConfig);
@@ -144,6 +147,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
public TaskDeploymentDescriptor(
JobID jobID,
+ String jobName,
JobVertexID vertexID,
ExecutionAttemptID executionId,
SerializedValue<ExecutionConfig> serializedExecutionConfig,
@@ -162,6 +166,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
this(
jobID,
+ jobName,
vertexID,
executionId,
serializedExecutionConfig,
@@ -195,6 +200,8 @@ public final class TaskDeploymentDescriptor implements Serializable {
public JobID getJobID() {
return jobID;
}
+
+ public String getJobName() { return jobName; }
/**
* Returns the task's execution vertex ID.
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
index a10c463..121936c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -89,6 +90,13 @@ public interface Environment {
TaskManagerRuntimeInfo getTaskManagerInfo();
/**
+ * Returns the task specific metric group.
+ *
+ * @return The MetricGroup of this task.
+ */
+ TaskMetricGroup getMetricGroup();
+
+ /**
* Returns the job-wide configuration object that was attached to the JobGraph.
*
* @return The job-wide configuration
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index cbc47a4..a85f32a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -673,6 +673,7 @@ public class ExecutionVertex implements Serializable {
return new TaskDeploymentDescriptor(
getJobId(),
+ getExecutionGraph().getJobName(),
getJobvertexId(),
executionId,
serializedConfig,
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
index 7aa57a8..a784f54 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.io.network.api.reader;
import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer;
import org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult;
@@ -122,4 +123,11 @@ abstract class AbstractRecordReader<T extends IOReadableWritable> extends Abstra
deserializer.setReporter(reporter);
}
}
+
+ @Override
+ public void setMetricGroup(IOMetricGroup metrics) {
+ for (RecordDeserializer<?> deserializer : recordDeserializers) {
+ deserializer.instantiateMetrics(metrics);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
index debb352..e5f5930 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.io.network.api.reader;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
@@ -53,4 +54,8 @@ public final class BufferReader extends AbstractReader {
public void setReporter(AccumulatorRegistry.Reporter reporter) {
}
+
+ @Override
+ public void setMetricGroup(IOMetricGroup metrics) {
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
index a1d705f..192a9ab 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.io.network.api.reader;
import java.io.IOException;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.event.TaskEvent;
import org.apache.flink.runtime.util.event.EventListener;
@@ -57,4 +58,11 @@ public interface ReaderBase {
*/
void setReporter(AccumulatorRegistry.Reporter reporter);
+ /**
+ * Setter for the metric group.
+ *
+ * @param metrics metric group to set
+ */
+ void setMetricGroup(IOMetricGroup metrics);
+
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
index cdd8731..1c17476 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
@@ -21,6 +21,8 @@ package org.apache.flink.runtime.io.network.api.serialization;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.util.DataInputDeserializer;
@@ -47,6 +49,9 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
private Buffer currentBuffer;
private AccumulatorRegistry.Reporter reporter;
+
+ private transient Counter numRecordsIn;
+ private transient Counter numBytesIn;
public AdaptiveSpanningRecordDeserializer() {
this.nonSpanningWrapper = new NonSpanningWrapper();
@@ -96,6 +101,9 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
if (reporter != null) {
reporter.reportNumBytesIn(len);
}
+ if (numBytesIn != null) {
+ numBytesIn.inc(len);
+ }
if (len <= nonSpanningRemaining - 4) {
// we can get a full record from here
@@ -104,6 +112,9 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
if (reporter != null) {
reporter.reportNumRecordsIn(1);
}
+ if (numRecordsIn != null) {
+ numRecordsIn.inc();
+ }
return (this.nonSpanningWrapper.remaining() == 0) ?
DeserializationResult.LAST_RECORD_FROM_BUFFER :
@@ -131,6 +142,9 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
if (reporter != null) {
reporter.reportNumRecordsIn(1);
}
+ if (numRecordsIn != null) {
+ numRecordsIn.inc();
+ }
// move the remainder to the non-spanning wrapper
// this does not copy it, only sets the memory segment
@@ -165,6 +179,12 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
this.spanningWrapper.setReporter(reporter);
}
+ @Override
+ public void instantiateMetrics(IOMetricGroup metrics) {
+ numBytesIn = metrics.getBytesInCounter();
+ numRecordsIn = metrics.getRecordsInCounter();
+ }
+
// -----------------------------------------------------------------------------------------------------------------
private static final class NonSpanningWrapper implements DataInputView {
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
index e4c7890..2f0c1ac 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.io.network.buffer.Buffer;
@@ -70,4 +71,11 @@ public interface RecordDeserializer<T extends IOReadableWritable> {
* Setter for the reporter, e.g. for the number of records emitted and the number of bytes read.
*/
void setReporter(AccumulatorRegistry.Reporter reporter);
+
+ /**
+ * Instantiates all metrics.
+ *
+ * @param metrics metric group
+ */
+ void instantiateMetrics(IOMetricGroup metrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
index e9f339a..9e7dfc1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
@@ -22,6 +22,7 @@ package org.apache.flink.runtime.io.network.api.serialization;
import java.io.IOException;
import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.io.network.buffer.Buffer;
@@ -69,4 +70,11 @@ public interface RecordSerializer<T extends IOReadableWritable> {
* Setter for the reporter, e.g. for the number of records emitted and the number of bytes read.
*/
void setReporter(AccumulatorRegistry.Reporter reporter);
+
+ /**
+ * Insantiates all metrics.
+ *
+ * @param metrics metric group
+ */
+ void instantiateMetrics(IOMetricGroup metrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
index f163e05..6495650 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
@@ -24,6 +24,8 @@ import java.nio.ByteOrder;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.util.DataOutputSerializer;
@@ -53,6 +55,9 @@ public class SpanningRecordSerializer<T extends IOReadableWritable> implements R
private AccumulatorRegistry.Reporter reporter;
+ private transient Counter numRecordsOut;
+ private transient Counter numBytesOut;
+
public SpanningRecordSerializer() {
this.serializationBuffer = new DataOutputSerializer(128);
@@ -85,6 +90,14 @@ public class SpanningRecordSerializer<T extends IOReadableWritable> implements R
reporter.reportNumBytesOut(len);
reporter.reportNumRecordsOut(1);
}
+
+ if (numBytesOut != null) {
+ numBytesOut.inc(len);
+ }
+
+ if (numRecordsOut != null) {
+ numRecordsOut.inc();
+ }
this.dataBuffer = this.serializationBuffer.wrapAsByteBuffer();
@@ -187,4 +200,10 @@ public class SpanningRecordSerializer<T extends IOReadableWritable> implements R
public void setReporter(AccumulatorRegistry.Reporter reporter) {
this.reporter = reporter;
}
+
+ @Override
+ public void instantiateMetrics(IOMetricGroup metrics) {
+ numBytesOut = metrics.getBytesOutCounter();
+ numRecordsOut = metrics.getRecordsOutCounter();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
index 6a4692e..49f7584 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
@@ -24,6 +24,8 @@ import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.util.DataInputDeserializer;
@@ -63,6 +65,9 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
private AccumulatorRegistry.Reporter reporter;
+ private transient Counter numRecordsIn;
+ private transient Counter numBytesIn;
+
public SpillingAdaptiveSpanningRecordDeserializer() {
String tempDirString = GlobalConfiguration.getString(
@@ -117,6 +122,9 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
if (reporter != null) {
reporter.reportNumBytesIn(len);
}
+ if (numBytesIn != null) {
+ numBytesIn.inc(len);
+ }
if (len <= nonSpanningRemaining - 4) {
// we can get a full record from here
@@ -126,6 +134,9 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
if (reporter != null) {
reporter.reportNumRecordsIn(1);
}
+ if (numRecordsIn != null) {
+ numRecordsIn.inc();
+ }
int remaining = this.nonSpanningWrapper.remaining();
if (remaining > 0) {
@@ -165,6 +176,9 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
if (reporter != null) {
reporter.reportNumRecordsIn(1);
}
+ if (numRecordsIn != null) {
+ numRecordsIn.inc();
+ }
// move the remainder to the non-spanning wrapper
// this does not copy it, only sets the memory segment
@@ -196,6 +210,13 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
this.spanningWrapper.setReporter(reporter);
}
+ @Override
+ public void instantiateMetrics(IOMetricGroup metrics) {
+ numBytesIn = metrics.getBytesInCounter();
+ numRecordsIn = metrics.getRecordsInCounter();
+ }
+
+
// -----------------------------------------------------------------------------------------------------------------
private static final class NonSpanningWrapper implements DataInputView {
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
index c534aa2..f93cdfc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.io.network.api.writer;
import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.event.AbstractEvent;
import org.apache.flink.runtime.io.network.api.serialization.RecordSerializer;
@@ -210,6 +211,16 @@ public class RecordWriter<T extends IOReadableWritable> {
}
/**
+ * Sets the metric group for this RecordWriter.
+ * @param metrics
+ */
+ public void setMetricGroup(IOMetricGroup metrics) {
+ for(RecordSerializer<?> serializer : serializers) {
+ serializer.instantiateMetrics(metrics);
+ }
+ }
+
+ /**
* Writes the buffer to the {@link ResultPartitionWriter}.
*
* <p> The buffer is cleared from the serializer state after a call to this method.
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativeTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativeTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativeTask.java
index 434f7d4..047fd7e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativeTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativeTask.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.operators.BatchTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,10 +170,10 @@ public abstract class AbstractIterativeTask<S extends Function, OT> extends Batc
}
@Override
- public DistributedRuntimeUDFContext createRuntimeContext() {
+ public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
Environment env = getEnvironment();
return new IterativeRuntimeUdfContext(env.getTaskInfo(), getUserCodeClassLoader(),
- getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap);
+ getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics);
}
// --------------------------------------------------------------------------------------------
@@ -362,8 +363,8 @@ public abstract class AbstractIterativeTask<S extends Function, OT> extends Batc
private class IterativeRuntimeUdfContext extends DistributedRuntimeUDFContext implements IterationRuntimeContext {
public IterativeRuntimeUdfContext(TaskInfo taskInfo, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
- Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?,?>> accumulatorMap) {
- super(taskInfo, userCodeClassLoader, executionConfig, cpTasks, accumulatorMap);
+ Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?,?>> accumulatorMap, MetricGroup metrics) {
+ super(taskInfo, userCodeClassLoader, executionConfig, cpTasks, accumulatorMap, metrics);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
index 51e00b3..546193c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.IOReadableWritable;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.broadcast.BroadcastVariableMaterialization;
import org.apache.flink.runtime.execution.CancelTaskException;
@@ -212,6 +213,7 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
* The accumulator map used in the RuntimeContext.
*/
protected Map<String, Accumulator<?,?>> accumulatorMap;
+ private MetricGroup metrics;
// --------------------------------------------------------------------------------------------
// Task Interface
@@ -237,6 +239,10 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
final Class<? extends Driver<S, OT>> driverClass = this.config.getDriver();
this.driver = InstantiationUtil.instantiate(driverClass, Driver.class);
+ String headName = getEnvironment().getTaskInfo().getTaskName().split("->")[0].trim();
+ this.metrics = getEnvironment().getMetricGroup()
+ .addOperator(headName.startsWith("CHAIN") ? headName.substring(6) : headName);
+
// initialize the readers.
// this does not yet trigger any stream consuming or processing.
initInputReaders();
@@ -256,7 +262,7 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
LOG.debug(formatLogString("Start task code."));
}
- this.runtimeUdfContext = createRuntimeContext();
+ this.runtimeUdfContext = createRuntimeContext(metrics);
// whatever happens in this scope, make sure that the local strategies are cleaned up!
// note that the initialization of the local strategies is in the try-finally block as well,
@@ -666,6 +672,7 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
}
inputReaders[i].setReporter(reporter);
+ inputReaders[i].setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup());
currentReaderOffset += groupSize;
}
@@ -1005,11 +1012,11 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
this.getExecutionConfig(), reporter, this.accumulatorMap);
}
- public DistributedRuntimeUDFContext createRuntimeContext() {
+ public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
Environment env = getEnvironment();
return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
- getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap);
+ getExecutionConfig(), env.getDistributedCacheEntries(), this.accumulatorMap, metrics);
}
// --------------------------------------------------------------------------------------------
@@ -1057,6 +1064,11 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
}
@Override
+ public MetricGroup getMetricGroup() {
+ return metrics;
+ }
+
+ @Override
public <X> MutableObjectIterator<X> getInput(int index) {
if (index < 0 || index > this.driver.getNumberOfInputs()) {
throw new IndexOutOfBoundsException();
@@ -1226,6 +1238,7 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
// setup live accumulator counters
recordWriter.setReporter(reporter);
+ recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());
writers.add(recordWriter);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
index 21e8784..39bf23f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
@@ -344,6 +344,8 @@ public class DataSinkTask<IT> extends AbstractInvokable {
final AccumulatorRegistry.Reporter reporter = accumulatorRegistry.getReadWriteReporter();
inputReader.setReporter(reporter);
+
+ inputReader.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup());
this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
@SuppressWarnings({ "rawtypes" })
@@ -375,6 +377,7 @@ public class DataSinkTask<IT> extends AbstractInvokable {
Environment env = getEnvironment();
return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
- getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap());
+ getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(),
+ getEnvironment().getMetricGroup().addOperator(getEnvironment().getTaskInfo().getTaskName()));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
index 0c525ea..819b84f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
@@ -358,7 +358,10 @@ public class DataSourceTask<OT> extends AbstractInvokable {
public DistributedRuntimeUDFContext createRuntimeContext() {
Environment env = getEnvironment();
+ String sourceName = getEnvironment().getTaskInfo().getTaskName().split("->")[0].trim();
+ sourceName = sourceName.startsWith("CHAIN") ? sourceName.substring(6) : sourceName;
return new DistributedRuntimeUDFContext(env.getTaskInfo(), getUserCodeClassLoader(),
- getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap());
+ getExecutionConfig(), env.getDistributedCacheEntries(), env.getAccumulatorRegistry().getUserMap(),
+ getEnvironment().getMetricGroup().addOperator(sourceName));
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
index 220527d..d6825ac 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/GroupReduceDriver.java
@@ -134,4 +134,4 @@ public class GroupReduceDriver<IT, OT> implements Driver<GroupReduceFunction<IT,
public void cancel() {
this.running = false;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/TaskContext.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/TaskContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/TaskContext.java
index fd5d238..df22528 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/TaskContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/TaskContext.java
@@ -22,6 +22,7 @@ package org.apache.flink.runtime.operators;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeutils.TypeComparator;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memory.MemoryManager;
@@ -67,4 +68,6 @@ public interface TaskContext<S, OT> {
AbstractInvokable getOwningNepheleTask();
String formatLogString(String message);
+
+ MetricGroup getMetricGroup();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
index 2fb52cd..407716f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.operators.BatchTask;
@@ -50,6 +51,8 @@ public abstract class ChainedDriver<IT, OT> implements Collector<IT> {
protected ExecutionConfig executionConfig;
protected boolean objectReuseEnabled = false;
+
+ protected MetricGroup metrics;
public void setup(TaskConfig config, String taskName, Collector<OT> outputCollector,
@@ -60,14 +63,15 @@ public abstract class ChainedDriver<IT, OT> implements Collector<IT> {
this.taskName = taskName;
this.outputCollector = outputCollector;
this.userCodeClassLoader = userCodeClassLoader;
+ this.metrics = parent.getEnvironment().getMetricGroup().addOperator(taskName);
Environment env = parent.getEnvironment();
if (parent instanceof BatchTask) {
- this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext();
+ this.udfContext = ((BatchTask<?, ?>) parent).createRuntimeContext(metrics);
} else {
this.udfContext = new DistributedRuntimeUDFContext(env.getTaskInfo(), userCodeClassLoader,
- parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap
+ parent.getExecutionConfig(), env.getDistributedCacheEntries(), accumulatorMap, metrics
);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
index b5ac4d7..293d34f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.functions.util.AbstractRuntimeUDFContext;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.broadcast.BroadcastVariableMaterialization;
import org.apache.flink.runtime.broadcast.InitializationTypeConflictException;
@@ -43,8 +44,8 @@ public class DistributedRuntimeUDFContext extends AbstractRuntimeUDFContext {
private final HashMap<String, BroadcastVariableMaterialization<?, ?>> broadcastVars = new HashMap<String, BroadcastVariableMaterialization<?, ?>>();
public DistributedRuntimeUDFContext(TaskInfo taskInfo, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig,
- Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?,?>> accumulators) {
- super(taskInfo, userCodeClassLoader, executionConfig, accumulators, cpTasks);
+ Map<String, Future<Path>> cpTasks, Map<String, Accumulator<?,?>> accumulators, MetricGroup metrics) {
+ super(taskInfo, userCodeClassLoader, executionConfig, accumulators, cpTasks, metrics);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
index 51e7e34..1f93a0d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.execution.Environment;
@@ -75,6 +76,7 @@ public class RuntimeEnvironment implements Environment {
private final AccumulatorRegistry accumulatorRegistry;
private final TaskManagerRuntimeInfo taskManagerInfo;
+ private final TaskMetricGroup metrics;
// ------------------------------------------------------------------------
@@ -96,7 +98,8 @@ public class RuntimeEnvironment implements Environment {
ResultPartitionWriter[] writers,
InputGate[] inputGates,
ActorGateway jobManager,
- TaskManagerRuntimeInfo taskManagerInfo) {
+ TaskManagerRuntimeInfo taskManagerInfo,
+ TaskMetricGroup metrics) {
this.jobId = checkNotNull(jobId);
this.jobVertexId = checkNotNull(jobVertexId);
@@ -116,6 +119,7 @@ public class RuntimeEnvironment implements Environment {
this.inputGates = checkNotNull(inputGates);
this.jobManager = checkNotNull(jobManager);
this.taskManagerInfo = checkNotNull(taskManagerInfo);
+ this.metrics = metrics;
}
// ------------------------------------------------------------------------
@@ -161,6 +165,11 @@ public class RuntimeEnvironment implements Environment {
}
@Override
+ public TaskMetricGroup getMetricGroup() {
+ return metrics;
+ }
+
+ @Override
public ClassLoader getUserClassLoader() {
return userCodeClassLoader;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 251673f..bc3ba0d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.cache.DistributedCache;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
@@ -201,6 +202,8 @@ public class Task implements Runnable {
/** atomic flag that makes sure the invokable is canceled exactly once upon error */
private final AtomicBoolean invokableHasBeenCanceled;
+ private final TaskMetricGroup metrics;
+
/** The invokable of this task, if initialized */
private volatile AbstractInvokable invokable;
@@ -239,7 +242,8 @@ public class Task implements Runnable {
FiniteDuration actorAskTimeout,
LibraryCacheManager libraryCache,
FileCache fileCache,
- TaskManagerRuntimeInfo taskManagerConfig)
+ TaskManagerRuntimeInfo taskManagerConfig,
+ TaskMetricGroup metricGroup)
{
this.taskInfo = checkNotNull(tdd.getTaskInfo());
this.jobId = checkNotNull(tdd.getJobID());
@@ -274,6 +278,7 @@ public class Task implements Runnable {
this.taskManagerConfig = checkNotNull(taskManagerConfig);
this.executionListenerActors = new CopyOnWriteArrayList<ActorGateway>();
+ this.metrics = metricGroup;
// create the reader and writer structures
@@ -518,7 +523,7 @@ public class Task implements Runnable {
userCodeClassLoader, memoryManager, ioManager,
broadcastVariableManager, accumulatorRegistry,
splitProvider, distributedCacheEntries,
- writers, inputGates, jobManager, taskManagerConfig);
+ writers, inputGates, jobManager, taskManagerConfig, metrics);
// let the task code create its readers and writers
invokable.setEnvironment(env);
@@ -683,6 +688,9 @@ public class Task implements Runnable {
// remove all of the tasks library resources
libraryCache.unregisterTask(jobId, executionId);
+
+ //Uncomment before Merging!!!
+ //metrics.close();
// remove all files in the distributed cache
removeCachedFiles(distributedCacheEntries, fileCache);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 1c7815c..beb012c 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -18,13 +18,14 @@
package org.apache.flink.runtime.taskmanager
-import java.io.{FileInputStream, File, IOException}
+import java.io.{File, FileInputStream, IOException}
import java.lang.management.{ManagementFactory, OperatingSystemMXBean}
import java.lang.reflect.Method
import java.net.{InetAddress, InetSocketAddress}
import java.util
import java.util.UUID
import java.util.concurrent.TimeUnit
+import javax.management.ObjectName
import _root_.akka.actor._
import _root_.akka.pattern.ask
@@ -36,12 +37,14 @@ import com.fasterxml.jackson.databind.ObjectMapper
import grizzled.slf4j.Logger
import org.apache.flink.configuration._
import org.apache.flink.core.fs.FileSystem
-import org.apache.flink.core.memory.{HybridMemorySegment, HeapMemorySegment, MemorySegmentFactory, MemoryType}
+import org.apache.flink.core.memory.{HeapMemorySegment, HybridMemorySegment, MemorySegmentFactory, MemoryType}
+import org.apache.flink.metrics.groups.TaskManagerMetricGroup
+import org.apache.flink.metrics.{MetricGroup, Gauge => FlinkGauge, MetricRegistry => FlinkMetricRegistry}
import org.apache.flink.runtime.accumulators.AccumulatorSnapshot
import org.apache.flink.runtime.clusterframework.messages.StopCluster
import org.apache.flink.runtime.clusterframework.types.ResourceID
import org.apache.flink.runtime.akka.AkkaUtils
-import org.apache.flink.runtime.blob.{BlobClient, BlobCache, BlobService}
+import org.apache.flink.runtime.blob.{BlobCache, BlobClient, BlobService}
import org.apache.flink.runtime.broadcast.BroadcastVariableManager
import org.apache.flink.runtime.deployment.{InputChannelDeploymentDescriptor, TaskDeploymentDescriptor}
import org.apache.flink.runtime.execution.ExecutionState
@@ -58,7 +61,7 @@ import org.apache.flink.runtime.leaderretrieval.{LeaderRetrievalListener, Leader
import org.apache.flink.runtime.memory.MemoryManager
import org.apache.flink.runtime.messages.Messages._
import org.apache.flink.runtime.messages.RegistrationMessages._
-import org.apache.flink.runtime.messages.StackTraceSampleMessages.{SampleTaskStackTrace, ResponseStackTraceSampleFailure, ResponseStackTraceSampleSuccess, StackTraceSampleMessages, TriggerStackTraceSample}
+import org.apache.flink.runtime.messages.StackTraceSampleMessages.{ResponseStackTraceSampleFailure, ResponseStackTraceSampleSuccess, SampleTaskStackTrace, StackTraceSampleMessages, TriggerStackTraceSample}
import org.apache.flink.runtime.messages.TaskManagerMessages._
import org.apache.flink.runtime.messages.TaskMessages._
import org.apache.flink.runtime.messages.checkpoint.{AbstractCheckpointMessage, NotifyCheckpointComplete, TriggerCheckpoint}
@@ -152,6 +155,9 @@ class TaskManager(
/** Registry of metrics periodically transmitted to the JobManager */
private val metricRegistry = TaskManager.createMetricsRegistry()
+ private var metricsRegistry : FlinkMetricRegistry = null
+ private var taskManagerMetricGroup : TaskManagerMetricGroup = null
+
/** Metric serialization */
private val metricRegistryMapper: ObjectMapper = new ObjectMapper()
.registerModule(
@@ -257,6 +263,13 @@ class TaskManager(
} catch {
case t: Exception => log.error("FileCache did not shutdown properly.", t)
}
+
+ try {
+ //enable this before merging
+ //metricsRegistry.shutdown()
+ } catch {
+ case t: Exception => log.error("MetricRegistry did not shutdown properly.", t)
+ }
log.info(s"Task manager ${self.path} is completely shut down.")
}
@@ -924,6 +937,14 @@ class TaskManager(
else {
libraryCacheManager = Some(new FallbackLibraryCacheManager)
}
+
+ metricsRegistry = new FlinkMetricRegistry(this.config.configuration);
+
+ taskManagerMetricGroup =
+ new TaskManagerMetricGroup(metricsRegistry, this.runtimeInfo.getHostname, id.toString)
+
+ TaskManager.instantiateStatusMetrics(taskManagerMetricGroup)
+
// watch job manager to detect when it dies
context.watch(jobManager)
@@ -1056,6 +1077,17 @@ class TaskManager(
val jobManagerGateway = new AkkaActorGateway(jobManagerActor, leaderSessionID.orNull)
val selfGateway = new AkkaActorGateway(self, leaderSessionID.orNull)
+ var jobName = tdd.getJobName
+ if (tdd.getJobName.length == 0) {
+ jobName = tdd.getJobID.toString()
+ } else {
+ jobName = tdd.getJobName
+ }
+
+ val taskMetricGroup = taskManagerMetricGroup
+ .addJob(tdd.getJobID, jobName)
+ .addTask(tdd.getVertexID, tdd.getExecutionId, tdd.getIndexInSubtaskGroup, tdd.getTaskName)
+
val task = new Task(
tdd,
memoryManager,
@@ -1067,7 +1099,8 @@ class TaskManager(
config.timeout,
libCache,
fileCache,
- runtimeInfo)
+ runtimeInfo,
+ taskMetricGroup)
log.info(s"Received task ${task.getTaskInfo.getTaskNameWithSubtasks()}")
@@ -2234,4 +2267,130 @@ object TaskManager {
})
metricRegistry
}
+
+ private def instantiateStatusMetrics(taskManagerMetricGroup: MetricGroup) : Unit = {
+ val jvm = taskManagerMetricGroup
+ .addGroup("Status")
+ .addGroup("JVM")
+
+ instantiateClassLoaderMetrics(jvm.addGroup("ClassLoader"))
+ instantiateGarbageCollectorMetrics(jvm.addGroup("GarbageCollector"))
+ instantiateMemoryMetrics(jvm.addGroup("Memory"))
+ instantiateThreadMetrics(jvm.addGroup("Threads"))
+ instantiateCPUMetrics(jvm.addGroup("CPU"))
+ }
+
+ private def instantiateClassLoaderMetrics(metrics: MetricGroup) {
+ val mxBean = ManagementFactory.getClassLoadingMXBean
+
+ metrics
+ .gauge("ClassesLoaded", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getTotalLoadedClassCount
+ })
+ metrics.gauge("ClassesUnloaded", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getUnloadedClassCount
+ })
+ }
+
+ private def instantiateGarbageCollectorMetrics(metrics: MetricGroup) {
+ val garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans
+
+ for (garbageCollector <- garbageCollectors) {
+ val gcGroup = metrics.addGroup("\"" + garbageCollector.getName + "\"")
+ gcGroup.gauge("Count", new FlinkGauge[Long] {
+ override def getValue: Long = garbageCollector.getCollectionCount
+ })
+ gcGroup.gauge("Time", new FlinkGauge[Long] {
+ override def getValue: Long = garbageCollector.getCollectionTime
+ })
+ }
+ }
+
+ private def instantiateMemoryMetrics(metrics: MetricGroup) {
+ val mxBean = ManagementFactory.getMemoryMXBean
+ val heap = metrics.addGroup("Heap")
+ heap.gauge("Used", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getHeapMemoryUsage.getUsed
+ })
+ heap.gauge("Committed", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getHeapMemoryUsage.getCommitted
+ })
+ heap.gauge("Max", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getHeapMemoryUsage.getMax
+ })
+
+ val nonHeap = metrics.addGroup("NonHeap")
+ nonHeap.gauge("Used", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getNonHeapMemoryUsage.getUsed
+ })
+ nonHeap.gauge("Committed", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getNonHeapMemoryUsage.getCommitted
+ })
+ nonHeap.gauge("Max", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getNonHeapMemoryUsage.getMax
+ })
+
+ val con = ManagementFactory.getPlatformMBeanServer;
+
+ val directObjectName = new ObjectName("java.nio:type=BufferPool,name=direct")
+
+ val direct = metrics.addGroup("Direct")
+ direct.gauge("Count", new FlinkGauge[Long] {
+ override def getValue: Long = con
+ .getAttribute(directObjectName, "Count").asInstanceOf[Long]
+ })
+ direct.gauge("MemoryUsed", new FlinkGauge[Long] {
+ override def getValue: Long = con
+ .getAttribute(directObjectName, "MemoryUsed").asInstanceOf[Long]
+ })
+ direct.gauge("TotalCapacity", new FlinkGauge[Long] {
+ override def getValue: Long = con
+ .getAttribute(directObjectName, "TotalCapacity").asInstanceOf[Long]
+ })
+
+ val mappedObjectName = new ObjectName("java.nio:type=BufferPool,name=direct")
+
+ val mapped = metrics.addGroup("Mapped")
+ mapped.gauge("Count", new FlinkGauge[Long] {
+ override def getValue: Long = con
+ .getAttribute(mappedObjectName, "Count").asInstanceOf[Long]
+ })
+ mapped.gauge("MemoryUsed", new FlinkGauge[Long] {
+ override def getValue: Long = con
+ .getAttribute(mappedObjectName, "MemoryUsed").asInstanceOf[Long]
+ })
+ mapped.gauge("TotalCapacity", new FlinkGauge[Long] {
+ override def getValue: Long = con
+ .getAttribute(mappedObjectName, "TotalCapacity").asInstanceOf[Long]
+ })
+ }
+
+ private def instantiateThreadMetrics(metrics: MetricGroup): Unit = {
+ val mxBean = ManagementFactory.getThreadMXBean
+
+ metrics
+ .gauge("Count", new FlinkGauge[Int] {
+ override def getValue: Int = mxBean.getThreadCount
+ })
+ }
+
+ private def instantiateCPUMetrics(metrics: MetricGroup): Unit = {
+ try {
+ val mxBean = ManagementFactory.getOperatingSystemMXBean
+ .asInstanceOf[com.sun.management.OperatingSystemMXBean]
+
+ metrics
+ .gauge("Load", new FlinkGauge[Double] {
+ override def getValue: Double = mxBean.getProcessCpuLoad
+ })
+ metrics.gauge("Time", new FlinkGauge[Long] {
+ override def getValue: Long = mxBean.getProcessCpuTime
+ })
+ }
+ catch {
+ case t: Throwable =>
+ LOG.warn("Cannot access com.sun.management.OperatingSystemMXBean.getProcessCpuLoad()" +
+ " - CPU load metrics will not be available.")
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
index 36744a9..f0bf694 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptorTest.java
@@ -46,6 +46,7 @@ public class TaskDeploymentDescriptorTest {
final JobID jobID = new JobID();
final JobVertexID vertexID = new JobVertexID();
final ExecutionAttemptID execId = new ExecutionAttemptID();
+ final String jobName = "job name";
final String taskName = "task name";
final int indexInSubtaskGroup = 0;
final int currentNumberOfSubtasks = 1;
@@ -59,7 +60,7 @@ public class TaskDeploymentDescriptorTest {
final List<URL> requiredClasspaths = new ArrayList<URL>(0);
final SerializedValue<ExecutionConfig> executionConfig = ExecutionConfigTest.getSerializedConfig();
- final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(jobID, vertexID, execId,
+ final TaskDeploymentDescriptor orig = new TaskDeploymentDescriptor(jobID, jobName, vertexID, execId,
executionConfig, taskName, indexInSubtaskGroup, currentNumberOfSubtasks, attemptNumber,
jobConfiguration, taskConfiguration, invokableClass.getName(), producedResults, inputGates,
requiredJars, requiredClasspaths, 47);
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
index 6853722..9724a80 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.io.network.api.reader;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.metrics.groups.IOMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.event.TaskEvent;
import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
@@ -189,5 +190,9 @@ public class AbstractReaderTest {
public void setReporter(AccumulatorRegistry.Reporter reporter) {
}
+
+ @Override
+ public void setMetricGroup(IOMetricGroup metrics) {
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
index 9be957a..0300a07 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/drivers/TestTaskContext.java
@@ -25,6 +25,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.api.java.typeutils.runtime.RuntimeSerializerFactory;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.memory.MemoryType;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memory.MemoryManager;
@@ -221,4 +223,9 @@ public class TestTaskContext<S, T> implements TaskContext<S, T> {
public String formatLogString(String message) {
return message;
}
+
+ @Override
+ public MetricGroup getMetricGroup() {
+ return new DummyMetricGroup();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
index 7043a63..2c3dcf1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/BinaryOperatorTestBase.java
@@ -26,6 +26,8 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.api.java.typeutils.runtime.RuntimeSerializerFactory;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
@@ -366,6 +368,11 @@ public class BinaryOperatorTestBase<S extends Function, IN, OUT> extends TestLog
return "Driver Tester: " + message;
}
+ @Override
+ public MetricGroup getMetricGroup() {
+ return new DummyMetricGroup();
+ }
+
// --------------------------------------------------------------------------------------------
@After
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
index ab58cea..6381733 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DriverTestBase.java
@@ -24,6 +24,8 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.util.DummyMetricGroup;
import org.apache.flink.runtime.operators.Driver;
import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo;
import org.apache.flink.util.TestLogger;
@@ -363,6 +365,11 @@ public class DriverTestBase<S extends Function> extends TestLogger implements Ta
return "Driver Tester: " + message;
}
+ @Override
+ public MetricGroup getMetricGroup() {
+ return new DummyMetricGroup();
+ }
+
// --------------------------------------------------------------------------------------------
@After
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
index 233dace..9b54383 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/DummyEnvironment.java
@@ -27,6 +27,8 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.TaskInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
+import org.apache.flink.metrics.util.DummyTaskMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.execution.Environment;
@@ -82,6 +84,11 @@ public class DummyEnvironment implements Environment {
}
@Override
+ public TaskMetricGroup getMetricGroup() {
+ return new DummyTaskMetricGroup();
+ }
+
+ @Override
public Configuration getJobConfiguration() {
return null;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
index d29b206..31fd08c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
@@ -24,6 +24,8 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.UnmodifiableConfiguration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.core.memory.MemorySegmentFactory;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
+import org.apache.flink.metrics.util.DummyTaskMetricGroup;
import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.execution.Environment;
@@ -210,6 +212,11 @@ public class MockEnvironment implements Environment {
}
@Override
+ public TaskMetricGroup getMetricGroup() {
+ return new DummyTaskMetricGroup();
+ }
+
+ @Override
public InputSplitProvider getInputSplitProvider() {
return this.inputSplitProvider;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
index 886c881..77e18c6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/UnaryOperatorTestBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.api.java.typeutils.runtime.RuntimeSerializerFactory;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
@@ -358,6 +359,11 @@ public class UnaryOperatorTestBase<S extends Function, IN, OUT> extends TestLogg
return "Driver Tester: " + message;
}
+ @Override
+ public MetricGroup getMetricGroup() {
+ return null;
+ }
+
// --------------------------------------------------------------------------------------------
@After
http://git-wip-us.apache.org/repos/asf/flink/blob/003ce18e/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
index 1b463bc..ae05ae9 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskAsyncCallTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.taskmanager;
import org.apache.flink.api.common.ExecutionConfigTest;
import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.groups.TaskMetricGroup;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
@@ -148,7 +149,7 @@ public class TaskAsyncCallTest {
when(networkEnvironment.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(
- new JobID(), new JobVertexID(), new ExecutionAttemptID(),
+ new JobID(), "Job Name", new JobVertexID(), new ExecutionAttemptID(),
ExecutionConfigTest.getSerializedConfig(),
"Test Task", 0, 1, 0,
new Configuration(), new Configuration(),
@@ -170,7 +171,8 @@ public class TaskAsyncCallTest {
new FiniteDuration(60, TimeUnit.SECONDS),
libCache,
mock(FileCache.class),
- new TaskManagerRuntimeInfo("localhost", new Configuration()));
+ new TaskManagerRuntimeInfo("localhost", new Configuration()),
+ mock(TaskMetricGroup.class));
}
public static class CheckpointsInOrderInvokable extends AbstractInvokable implements StatefulTask<StateHandle<Serializable>> {