You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2016/10/31 15:08:29 UTC

[1/5] flink git commit: [FLINK-4972] Fix CoordinatorShutdownTest

Repository: flink
Updated Branches:
  refs/heads/master 094b747a3 -> ba2d007e5


[FLINK-4972] Fix CoordinatorShutdownTest


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/99f1dc3e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/99f1dc3e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/99f1dc3e

Branch: refs/heads/master
Commit: 99f1dc3e24ec529852ce38bcb9c46ffaf749333d
Parents: 094b747
Author: zentol <ch...@apache.org>
Authored: Mon Oct 31 14:15:38 2016 +0100
Committer: zentol <ch...@apache.org>
Committed: Mon Oct 31 15:12:01 2016 +0100

----------------------------------------------------------------------
 .../checkpoint/CoordinatorShutdownTest.java     | 79 +++++++++++++++++---
 1 file changed, 68 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/99f1dc3e/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
index 0b2f4f3..777ba9b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
@@ -26,9 +26,9 @@ import org.apache.flink.runtime.instance.ActorGateway;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
 import org.apache.flink.runtime.jobgraph.tasks.ExternalizedCheckpointSettings;
 import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
-import org.apache.flink.runtime.jobmanager.Tasks;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
 import org.apache.flink.runtime.testingUtils.TestingUtils;
@@ -51,14 +51,15 @@ public class CoordinatorShutdownTest {
 	public void testCoordinatorShutsDownOnFailure() {
 		LocalFlinkMiniCluster cluster = null;
 		try {
-			Configuration noTaskManagerConfig = new Configuration();
-			noTaskManagerConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 0);
-			cluster = new LocalFlinkMiniCluster(noTaskManagerConfig, true);
+			Configuration config = new Configuration();
+			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
+			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
+			cluster = new LocalFlinkMiniCluster(config, true);
 			cluster.start();
 			
 			// build a test graph with snapshotting enabled
 			JobVertex vertex = new JobVertex("Test Vertex");
-			vertex.setInvokableClass(Tasks.NoOpInvokable.class);
+			vertex.setInvokableClass(FailingBlockingInvokable.class);
 			List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
 			
 			JobGraph testGraph = new JobGraph("test job", vertex);
@@ -72,11 +73,11 @@ public class CoordinatorShutdownTest {
 					testGraph,
 					ListeningBehaviour.EXECUTION_RESULT);
 			
-			// submit is successful, but then the job dies because no TaskManager / slot is available
+			// submit is successful, but then the job blocks due to the invokable
 			Future<Object> submitFuture = jmGateway.ask(submitMessage, timeout);
 			Await.result(submitFuture, timeout);
 
-			// get the execution graph and make sure the coordinator is properly shut down
+			// get the execution graph and store the ExecutionGraph reference
 			Future<Object> jobRequestFuture = jmGateway.ask(
 					new JobManagerMessages.RequestJob(testGraph.getJobID()),
 					timeout);
@@ -84,8 +85,12 @@ public class CoordinatorShutdownTest {
 			ExecutionGraph graph = (ExecutionGraph)((JobManagerMessages.JobFound) Await.result(jobRequestFuture, timeout)).executionGraph();
 			
 			assertNotNull(graph);
+
+			FailingBlockingInvokable.unblock();
+
 			graph.waitUntilFinished();
 			
+			// verify that the coordinator was shut down
 			CheckpointCoordinator coord = graph.getCheckpointCoordinator();
 			assertTrue(coord == null || coord.isShutdown());
 		}
@@ -105,12 +110,15 @@ public class CoordinatorShutdownTest {
 	public void testCoordinatorShutsDownOnSuccess() {
 		LocalFlinkMiniCluster cluster = null;
 		try {
-			cluster = new LocalFlinkMiniCluster(new Configuration(), true);
+			Configuration config = new Configuration();
+			config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
+			config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
+			cluster = new LocalFlinkMiniCluster(config, true);
 			cluster.start();
 			
 			// build a test graph with snapshotting enabled
 			JobVertex vertex = new JobVertex("Test Vertex");
-			vertex.setInvokableClass(Tasks.NoOpInvokable.class);
+			vertex.setInvokableClass(BlockingInvokable.class);
 			List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
 
 			JobGraph testGraph = new JobGraph("test job", vertex);
@@ -124,11 +132,11 @@ public class CoordinatorShutdownTest {
 					testGraph,
 					ListeningBehaviour.EXECUTION_RESULT);
 
-			// submit is successful, but then the job dies because no TaskManager / slot is available
+			// submit is successful, but then the job blocks due to the invokable
 			Future<Object> submitFuture = jmGateway.ask(submitMessage, timeout);
 			Await.result(submitFuture, timeout);
 
-			// get the execution graph and make sure the coordinator is properly shut down
+			// get the execution graph and store the ExecutionGraph reference
 			Future<Object> jobRequestFuture = jmGateway.ask(
 					new JobManagerMessages.RequestJob(testGraph.getJobID()),
 					timeout);
@@ -136,8 +144,12 @@ public class CoordinatorShutdownTest {
 			ExecutionGraph graph = (ExecutionGraph)((JobManagerMessages.JobFound) Await.result(jobRequestFuture, timeout)).executionGraph();
 
 			assertNotNull(graph);
+
+			BlockingInvokable.unblock();
+			
 			graph.waitUntilFinished();
 
+			// verify that the coordinator was shut down
 			CheckpointCoordinator coord = graph.getCheckpointCoordinator();
 			assertTrue(coord == null || coord.isShutdown());
 		}
@@ -152,4 +164,49 @@ public class CoordinatorShutdownTest {
 			}
 		}
 	}
+
+	public static class BlockingInvokable extends AbstractInvokable {
+		private static boolean blocking = true;
+		private static final Object lock = new Object();
+
+		@Override
+		public void invoke() throws Exception {
+			while (blocking) {
+				synchronized (lock) {
+					lock.wait();
+				}
+			}
+		}
+
+		public static void unblock() {
+			blocking = false;
+
+			synchronized (lock) {
+				lock.notifyAll();
+			}
+		}
+	}
+
+	public static class FailingBlockingInvokable extends AbstractInvokable {
+		private static boolean blocking = true;
+		private static final Object lock = new Object();
+
+		@Override
+		public void invoke() throws Exception {
+			while (blocking) {
+				synchronized (lock) {
+					lock.wait();
+				}
+			}
+			throw new RuntimeException("This exception is expected.");
+		}
+
+		public static void unblock() {
+			blocking = false;
+
+			synchronized (lock) {
+				lock.notifyAll();
+			}
+		}
+	}
 }


[4/5] flink git commit: [FLINK-4733] Port Task IO metrics

Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
index e89559d..39d3453 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
@@ -27,7 +27,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.InputSplit;
 import org.apache.flink.metrics.Counter;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.CancelTaskException;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
@@ -284,11 +283,8 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 		this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
 		this.eventualOutputs = new ArrayList<RecordWriter<?>>();
 
-		final AccumulatorRegistry accumulatorRegistry = getEnvironment().getAccumulatorRegistry();
-		final AccumulatorRegistry.Reporter reporter = accumulatorRegistry.getReadWriteReporter();
-
 		this.output = BatchTask.initOutputs(this, cl, this.config, this.chainedTasks, this.eventualOutputs,
-				getExecutionConfig(), reporter, getEnvironment().getAccumulatorRegistry().getUserMap());
+				getExecutionConfig(), getEnvironment().getAccumulatorRegistry().getUserMap());
 	}
 
 	// ------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 7ce9b0b..3a5c96f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -414,6 +414,10 @@ public class Task implements Runnable, TaskActions {
 		return accumulatorRegistry;
 	}
 
+	public TaskMetricGroup getMetricGroup() {
+		return metrics;
+	}
+
 	public Thread getExecutingThread() {
 		return executingThread;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
index 60aadf5..7bea5d3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
@@ -22,6 +22,7 @@ import org.apache.flink.runtime.accumulators.AccumulatorSnapshot;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.executiongraph.IOMetrics;
 import org.apache.flink.runtime.util.SerializedThrowable;
 
 /**
@@ -48,6 +49,7 @@ public class TaskExecutionState implements java.io.Serializable {
 
 	/** Serialized flink and user-defined accumulators */
 	private final AccumulatorSnapshot accumulators;
+	private final IOMetrics ioMetrics;
 
 	/**
 	 * Creates a new task execution state update, with no attached exception and no accumulators.
@@ -60,7 +62,7 @@ public class TaskExecutionState implements java.io.Serializable {
 	 *        the execution state to be reported
 	 */
 	public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId, ExecutionState executionState) {
-		this(jobID, executionId, executionState, null, null);
+		this(jobID, executionId, executionState, null, null, null);
 	}
 
 	/**
@@ -75,7 +77,7 @@ public class TaskExecutionState implements java.io.Serializable {
 	 */
 	public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
 							ExecutionState executionState, Throwable error) {
-		this(jobID, executionId, executionState, error, null);
+		this(jobID, executionId, executionState, error, null, null);
 	}
 
 	/**
@@ -95,7 +97,7 @@ public class TaskExecutionState implements java.io.Serializable {
 	 */
 	public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
 			ExecutionState executionState, Throwable error,
-			AccumulatorSnapshot accumulators) {
+			AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {
 
 		if (jobID == null || executionId == null || executionState == null) {
 			throw new NullPointerException();
@@ -110,6 +112,7 @@ public class TaskExecutionState implements java.io.Serializable {
 			this.throwable = null;
 		}
 		this.accumulators = accumulators;
+		this.ioMetrics = ioMetrics;
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -164,6 +167,10 @@ public class TaskExecutionState implements java.io.Serializable {
 		return accumulators;
 	}
 
+	public IOMetrics getIOMetrics() {
+		return ioMetrics;
+	}
+
 	// --------------------------------------------------------------------------------------------
 
 	@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 7608b87..ce763e7 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -23,7 +23,6 @@ import java.lang.management.ManagementFactory
 import java.net.{InetAddress, InetSocketAddress}
 import java.util
 import java.util.UUID
-import java.util.concurrent.TimeUnit
 
 import _root_.akka.actor._
 import _root_.akka.pattern.ask
@@ -1296,7 +1295,8 @@ class TaskManager(
             task.getExecutionId,
             task.getExecutionState,
             task.getFailureCause,
-            accumulators)
+            accumulators,
+            task.getMetricGroup.getIOMetricGroup.createSnapshot())
         )
       )
     }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
index aac2d00..ff50573 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
@@ -25,7 +25,6 @@ import org.apache.flink.api.common.accumulators.LongCounter;
 import org.apache.flink.api.common.restartstrategy.RestartStrategies;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.testutils.CommonTestUtils;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.checkpoint.CompletedCheckpoint;
@@ -66,8 +65,6 @@ public class ArchivedExecutionGraphTest {
 	private static JobVertexID v1ID = new JobVertexID();
 	private static JobVertexID v2ID = new JobVertexID();
 
-	private static ExecutionAttemptID executionWithAccumulatorsID;
-
 	private static ExecutionGraph runtimeGraph;
 
 	@BeforeClass
@@ -119,15 +116,10 @@ public class ArchivedExecutionGraphTest {
 			null,
 			new TestCheckpointStatsTracker());
 
-		Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators = new HashMap<>();
-		flinkAccumulators.put(AccumulatorRegistry.Metric.NUM_BYTES_IN, new LongCounter(32));
-
 		Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<>();
 		userAccumulators.put("userAcc", new LongCounter(64));
 
 		Execution executionWithAccumulators = runtimeGraph.getJobVertex(v1ID).getTaskVertices()[0].getCurrentExecutionAttempt();
-		executionWithAccumulators.setAccumulators(flinkAccumulators, userAccumulators);
-		executionWithAccumulatorsID = executionWithAccumulators.getAttemptId();
 
 		runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
 	}
@@ -200,7 +192,6 @@ public class ArchivedExecutionGraphTest {
 		// -------------------------------------------------------------------------------------------------------------
 		compareStringifiedAccumulators(runtimeGraph.getAccumulatorResultsStringified(), archivedGraph.getAccumulatorResultsStringified());
 		compareSerializedAccumulators(runtimeGraph.getAccumulatorsSerialized(), archivedGraph.getAccumulatorsSerialized());
-		compareFlinkAccumulators(runtimeGraph.getFlinkAccumulators().get(executionWithAccumulatorsID), archivedGraph.getFlinkAccumulators().get(executionWithAccumulatorsID));
 
 		// -------------------------------------------------------------------------------------------------------------
 		// JobVertices
@@ -250,7 +241,6 @@ public class ArchivedExecutionGraphTest {
 		compareOperatorCheckpointStats(runtimeJobVertex.getCheckpointStats().get(), archivedJobVertex.getCheckpointStats().get());
 
 		compareStringifiedAccumulators(runtimeJobVertex.getAggregatedUserAccumulatorsStringified(), archivedJobVertex.getAggregatedUserAccumulatorsStringified());
-		compareFlinkAccumulators(runtimeJobVertex.getAggregatedMetricAccumulators(), archivedJobVertex.getAggregatedMetricAccumulators());
 
 		AccessExecutionVertex[] runtimeExecutionVertices = runtimeJobVertex.getTaskVertices();
 		AccessExecutionVertex[] archivedExecutionVertices = archivedJobVertex.getTaskVertices();
@@ -294,7 +284,6 @@ public class ArchivedExecutionGraphTest {
 		assertEquals(runtimeExecution.getStateTimestamp(ExecutionState.CANCELED), archivedExecution.getStateTimestamp(ExecutionState.CANCELED));
 		assertEquals(runtimeExecution.getStateTimestamp(ExecutionState.FAILED), archivedExecution.getStateTimestamp(ExecutionState.FAILED));
 		compareStringifiedAccumulators(runtimeExecution.getUserAccumulatorsStringified(), archivedExecution.getUserAccumulatorsStringified());
-		compareFlinkAccumulators(runtimeExecution.getFlinkAccumulators(), archivedExecution.getFlinkAccumulators());
 		assertEquals(runtimeExecution.getParallelSubtaskIndex(), archivedExecution.getParallelSubtaskIndex());
 	}
 
@@ -321,18 +310,6 @@ public class ArchivedExecutionGraphTest {
 		}
 	}
 
-	private static void compareFlinkAccumulators(Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> runtimeAccs, Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> archivedAccs) {
-		assertEquals(runtimeAccs == null, archivedAccs == null);
-		if (runtimeAccs != null && archivedAccs != null) {
-			assertEquals(runtimeAccs.size(), archivedAccs.size());
-			for (Map.Entry<AccumulatorRegistry.Metric, Accumulator<?, ?>> runtimeAcc : runtimeAccs.entrySet()) {
-				Accumulator<?, ?> archivedAcc = archivedAccs.get(runtimeAcc.getKey());
-
-				assertEquals(runtimeAcc.getValue().getLocalValue(), archivedAcc.getLocalValue());
-			}
-		}
-	}
-
 	private static void compareOperatorCheckpointStats(OperatorCheckpointStats runtimeStats, OperatorCheckpointStats archivedStats) {
 		assertEquals(runtimeStats.getNumberOfSubTasks(), archivedStats.getNumberOfSubTasks());
 		assertEquals(runtimeStats.getCheckpointId(), archivedStats.getCheckpointId());

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
index 2d0ae41..2a287af 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphDeploymentTest.java
@@ -27,15 +27,11 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
-import org.apache.flink.runtime.accumulators.AccumulatorSnapshot;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
 import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
@@ -50,7 +46,6 @@ import org.apache.flink.runtime.jobgraph.JobVertex;
 import org.apache.flink.runtime.jobgraph.DistributionPattern;
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
 import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
 import org.apache.flink.runtime.operators.BatchTask;
 import org.apache.flink.runtime.taskmanager.TaskExecutionState;
@@ -335,7 +330,7 @@ public class ExecutionGraphDeploymentTest {
 
 		ExecutionAttemptID attemptID = eg.getJobVertex(v1.getID()).getTaskVertices()[0].getCurrentExecutionAttempt().getAttemptId();
 		eg.updateState(new TaskExecutionState(jobId, attemptID, ExecutionState.RUNNING));
-		eg.updateState(new TaskExecutionState(jobId, attemptID, ExecutionState.FINISHED, null, new AccumulatorSnapshot(jobId, attemptID, new HashMap<AccumulatorRegistry.Metric, Accumulator<?, ?>>(), new HashMap<String, Accumulator<?, ?>>())));
+		eg.updateState(new TaskExecutionState(jobId, attemptID, ExecutionState.FINISHED, null));
 
 		assertEquals(JobStatus.FAILED, eg.getState());
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
index 6853722..c6c0645 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/reader/AbstractReaderTest.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.io.network.api.reader;
 
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.event.TaskEvent;
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
 import org.apache.flink.runtime.io.network.api.EndOfSuperstepEvent;
@@ -184,10 +183,5 @@ public class AbstractReaderTest {
 		protected MockReader(InputGate inputGate) {
 			super(inputGate);
 		}
-
-		@Override
-		public void setReporter(AccumulatorRegistry.Reporter reporter) {
-
-		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
index d775869..a1f7c2d 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerLike.scala
@@ -226,10 +226,9 @@ trait TestingJobManagerLike extends FlinkActor {
         case (jobID, (updated, actors)) if updated =>
           currentJobs.get(jobID) match {
             case Some((graph, jobInfo)) =>
-              val flinkAccumulators = graph.getFlinkAccumulators
               val userAccumulators = graph.aggregateUserAccumulators
               actors foreach {
-                 actor => actor ! UpdatedAccumulators(jobID, flinkAccumulators, userAccumulators)
+                 actor => actor ! UpdatedAccumulators(jobID, userAccumulators)
               }
             case None =>
           }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
index 72cf58b..48c2bfb 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
@@ -23,7 +23,6 @@ import java.util.Map
 import akka.actor.ActorRef
 import org.apache.flink.api.common.JobID
 import org.apache.flink.api.common.accumulators.Accumulator
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry
 import org.apache.flink.runtime.checkpoint.savepoint.Savepoint
 import org.apache.flink.runtime.executiongraph.{AccessExecutionGraph, ExecutionAttemptID, ExecutionGraph}
 import org.apache.flink.runtime.instance.ActorGateway
@@ -73,7 +72,6 @@ object TestingJobManagerMessages {
    * Reports updated accumulators back to the listener.
    */
   case class UpdatedAccumulators(jobID: JobID,
-    flinkAccumulators: Map[ExecutionAttemptID, Map[AccumulatorRegistry.Metric, Accumulator[_,_]]],
     userAccumulators: Map[String, Accumulator[_,_]])
 
   /** Notifies the sender when the [[TestingJobManager]] has been elected as the leader

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
index 5a27564..714317d 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java
@@ -27,7 +27,6 @@ import org.apache.flink.metrics.Gauge;
 import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
 import org.apache.flink.runtime.metrics.groups.OperatorMetricGroup;
 import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.event.AbstractEvent;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
@@ -204,12 +203,6 @@ public class StreamInputProcessor<IN> {
 			}
 		}
 	}
-	
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-		for (RecordDeserializer<?> deserializer : recordDeserializers) {
-			deserializer.setReporter(reporter);
-		}
-	}
 
 	/**
 	 * Sets the metric group for this StreamInputProcessor.

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
index 075d9e0..5f7ffe4 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java
@@ -23,7 +23,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.metrics.Gauge;
 import org.apache.flink.runtime.jobgraph.tasks.StatefulTask;
 import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.event.AbstractEvent;
 import org.apache.flink.runtime.io.disk.iomanager.IOManager;
 import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
@@ -269,12 +268,6 @@ public class StreamTwoInputProcessor<IN1, IN2> {
 			}
 		}
 	}
-	
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-		for (RecordDeserializer<?> deserializer : recordDeserializers) {
-			deserializer.setReporter(reporter);
-		}
-	}
 
 	/**
 	 * Sets the metric group for this StreamTwoInputProcessor.

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
index 0f8f4a4..68d3064 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.runtime.tasks;
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
 import org.apache.flink.streaming.api.graph.StreamConfig;
 import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
@@ -48,9 +47,6 @@ public class OneInputStreamTask<IN, OUT> extends StreamTask<OUT, OneInputStreamO
 					getEnvironment().getIOManager());
 
 			// make sure that stream tasks report their I/O statistics
-			AccumulatorRegistry registry = getEnvironment().getAccumulatorRegistry();
-			AccumulatorRegistry.Reporter reporter = registry.getReadWriteReporter();
-			inputProcessor.setReporter(reporter);
 			inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup());
 		}
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
index 2a4a065..30bc377 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OperatorChain.java
@@ -28,7 +28,6 @@ import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.metrics.Counter;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
 import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
@@ -74,7 +73,7 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> {
 
 	private final OP headOperator;
 
-	public OperatorChain(StreamTask<OUT, OP> containingTask, AccumulatorRegistry.Reporter reporter) {
+	public OperatorChain(StreamTask<OUT, OP> containingTask) {
 		
 		final ClassLoader userCodeClassloader = containingTask.getUserCodeClassLoader();
 		final StreamConfig configuration = containingTask.getConfiguration();
@@ -99,7 +98,7 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> {
 				
 				RecordWriterOutput<?> streamOutput = createStreamOutput(
 						outEdge, chainedConfigs.get(outEdge.getSourceId()), i,
-						containingTask.getEnvironment(), reporter, containingTask.getName());
+						containingTask.getEnvironment(), containingTask.getName());
 	
 				this.streamOutputs[i] = streamOutput;
 				streamOutputMap.put(outEdge, streamOutput);
@@ -306,7 +305,7 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> {
 	private static <T> RecordWriterOutput<T> createStreamOutput(
 			StreamEdge edge, StreamConfig upStreamConfig, int outputIndex,
 			Environment taskEnvironment,
-			AccumulatorRegistry.Reporter reporter, String taskName)
+			String taskName)
 	{
 		TypeSerializer<T> outSerializer = upStreamConfig.getTypeSerializerOut(taskEnvironment.getUserClassLoader());
 
@@ -319,7 +318,6 @@ public class OperatorChain<OUT, OP extends StreamOperator<OUT>> {
 
 		StreamRecordWriter<SerializationDelegate<StreamRecord<T>>> output = 
 				new StreamRecordWriter<>(bufferWriter, outputPartitioner, upStreamConfig.getBufferTimeout());
-		output.setReporter(reporter);
 		output.setMetricGroup(taskEnvironment.getMetricGroup().getIOMetricGroup());
 		
 		return new RecordWriterOutput<>(output, outSerializer);

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index cd63102..fa7d1b0 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -227,7 +227,7 @@ public abstract class StreamTask<OUT, OP extends StreamOperator<OUT>>
 				timerService = new SystemProcessingTimeService(this, getCheckpointLock(), timerThreadFactory);
 			}
 
-			operatorChain = new OperatorChain<>(this, getEnvironment().getAccumulatorRegistry().getReadWriteReporter());
+			operatorChain = new OperatorChain<>(this);
 			headOperator = operatorChain.getHeadOperator();
 
 			getEnvironment().getMetricGroup().gauge("lastCheckpointSize", new Gauge<Long>() {

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
index d695781..978c9f2 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java
@@ -22,7 +22,6 @@ import java.util.List;
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
 import org.apache.flink.streaming.api.graph.StreamConfig;
 import org.apache.flink.streaming.api.graph.StreamEdge;
@@ -73,9 +72,6 @@ public class TwoInputStreamTask<IN1, IN2, OUT> extends StreamTask<OUT, TwoInputS
 				getEnvironment().getIOManager());
 
 		// make sure that stream tasks report their I/O statistics
-		AccumulatorRegistry registry = getEnvironment().getAccumulatorRegistry();
-		AccumulatorRegistry.Reporter reporter = registry.getReadWriteReporter();
-		this.inputProcessor.setReporter(reporter);
 		inputProcessor.setMetricGroup(getEnvironment().getMetricGroup().getIOMetricGroup());
 	}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamOperatorChainingTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamOperatorChainingTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamOperatorChainingTest.java
index 32e8ea9..c95a85e 100644
--- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamOperatorChainingTest.java
+++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/operators/StreamOperatorChainingTest.java
@@ -20,7 +20,6 @@ package org.apache.flink.streaming.runtime.operators;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.jobgraph.JobGraph;
 import org.apache.flink.runtime.jobgraph.JobVertex;
@@ -149,9 +148,7 @@ public class StreamOperatorChainingTest {
 		StreamTask<Integer, StreamMap<Integer, Integer>> mockTask =
 				createMockTask(streamConfig, chainedVertex.getName());
 
-		OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = new OperatorChain<>(
-				mockTask,
-				mock(AccumulatorRegistry.Reporter.class));
+		OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = new OperatorChain<>(mockTask);
 
 		headOperator.setup(mockTask, streamConfig, operatorChain.getChainEntryPoint());
 
@@ -291,9 +288,7 @@ public class StreamOperatorChainingTest {
 		StreamTask<Integer, StreamMap<Integer, Integer>> mockTask =
 				createMockTask(streamConfig, chainedVertex.getName());
 
-		OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = new OperatorChain<>(
-				mockTask,
-				mock(AccumulatorRegistry.Reporter.class));
+		OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = new OperatorChain<>(mockTask);
 
 		headOperator.setup(mockTask, streamConfig, operatorChain.getChainEntryPoint());
 

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java
index 624bfff..c56fa91 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorLiveITCase.java
@@ -29,7 +29,6 @@ import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.Plan;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.accumulators.IntCounter;
-import org.apache.flink.api.common.accumulators.LongCounter;
 import org.apache.flink.api.common.functions.RichFlatMapFunction;
 import org.apache.flink.api.common.io.OutputFormat;
 import org.apache.flink.api.java.DataSet;
@@ -41,7 +40,6 @@ import org.apache.flink.optimizer.DataStatistics;
 import org.apache.flink.optimizer.Optimizer;
 import org.apache.flink.optimizer.plan.OptimizedPlan;
 import org.apache.flink.optimizer.plantranslate.JobGraphGenerator;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.akka.AkkaUtils;
 import org.apache.flink.runtime.akka.ListeningBehaviour;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -203,32 +201,14 @@ public class AccumulatorLiveITCase {
 
 
 			TestingJobManagerMessages.UpdatedAccumulators msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
-			Map<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>> flinkAccumulators = msg.flinkAccumulators();
 			Map<String, Accumulator<?, ?>> userAccumulators = msg.userAccumulators();
 
 			ExecutionAttemptID mapperTaskID = null;
 
-			// find out the first task's execution attempt id
-			for (Map.Entry<ExecutionAttemptID, ?> entry : flinkAccumulators.entrySet()) {
-				if (entry.getValue() != null) {
-					mapperTaskID = entry.getKey();
-					break;
-				}
-			}
-
 			ExecutionAttemptID sinkTaskID = null;
 
-			// find the second's task id
-			for (ExecutionAttemptID key : flinkAccumulators.keySet()) {
-				if (key != mapperTaskID) {
-					sinkTaskID = key;
-					break;
-				}
-			}
-
 			/* Check for accumulator values */
-			if(checkUserAccumulators(0, userAccumulators) &&
-					checkFlinkAccumulators(mapperTaskID, 0, 0, 0, 0, flinkAccumulators)) {
+			if(checkUserAccumulators(0, userAccumulators)) {
 				LOG.info("Passed initial check for map task.");
 			} else {
 				fail("Wrong accumulator results when map task begins execution.");
@@ -242,17 +222,13 @@ public class AccumulatorLiveITCase {
 
 				// receive message
 				msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
-				flinkAccumulators = msg.flinkAccumulators();
 				userAccumulators = msg.userAccumulators();
 
-				LOG.info("{}", flinkAccumulators);
 				LOG.info("{}", userAccumulators);
 
-				if (checkUserAccumulators(expectedAccVal, userAccumulators) &&
-						checkFlinkAccumulators(mapperTaskID, 0, i, 0, i * 4, flinkAccumulators)) {
+				if (checkUserAccumulators(expectedAccVal, userAccumulators)) {
 					LOG.info("Passed round #" + i);
-				} else if (checkUserAccumulators(expectedAccVal, userAccumulators) &&
-						checkFlinkAccumulators(sinkTaskID, 0, i, 0, i * 4, flinkAccumulators)) {
+				} else if (checkUserAccumulators(expectedAccVal, userAccumulators)) {
 					// we determined the wrong task id and need to switch the two here
 					ExecutionAttemptID temp = mapperTaskID;
 					mapperTaskID = sinkTaskID;
@@ -264,11 +240,9 @@ public class AccumulatorLiveITCase {
 			}
 
 			msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
-			flinkAccumulators = msg.flinkAccumulators();
 			userAccumulators = msg.userAccumulators();
 
-			if(checkUserAccumulators(expectedAccVal, userAccumulators) &&
-					checkFlinkAccumulators(sinkTaskID, 0, 0, 0, 0, flinkAccumulators)) {
+			if(checkUserAccumulators(expectedAccVal, userAccumulators)) {
 				LOG.info("Passed initial check for sink task.");
 			} else {
 				fail("Wrong accumulator results when sink task begins execution.");
@@ -280,14 +254,11 @@ public class AccumulatorLiveITCase {
 				// receive message
 				msg = (TestingJobManagerMessages.UpdatedAccumulators) receiveOne(TIMEOUT);
 
-				flinkAccumulators = msg.flinkAccumulators();
 				userAccumulators = msg.userAccumulators();
 
-				LOG.info("{}", flinkAccumulators);
 				LOG.info("{}", userAccumulators);
 
-				if (checkUserAccumulators(expectedAccVal, userAccumulators) &&
-						checkFlinkAccumulators(sinkTaskID, i, 0, i * 4, 0, flinkAccumulators)) {
+				if (checkUserAccumulators(expectedAccVal, userAccumulators)) {
 					LOG.info("Passed round #" + i);
 				} else {
 					fail("Failed in round #" + i);
@@ -305,49 +276,6 @@ public class AccumulatorLiveITCase {
 		return accumulatorMap.containsKey(ACCUMULATOR_NAME) && expected == ((IntCounter)accumulatorMap.get(ACCUMULATOR_NAME)).getLocalValue();
 	}
 
-	private static boolean checkFlinkAccumulators(ExecutionAttemptID taskKey, int expectedRecordsIn, int expectedRecordsOut, int expectedBytesIn, int expectedBytesOut,
-												  Map<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?,?>>> accumulatorMap) {
-		LOG.info("checking flink accumulators");
-
-		Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> taskMap = accumulatorMap.get(taskKey);
-		assertTrue(accumulatorMap.size() > 0);
-
-		for (Map.Entry<AccumulatorRegistry.Metric, Accumulator<?, ?>> entry : taskMap.entrySet()) {
-			switch (entry.getKey()) {
-				/**
-				 * The following two cases are for the DataSource and Map task
-				 */
-				case NUM_RECORDS_OUT:
-					if(((LongCounter) entry.getValue()).getLocalValue() < expectedRecordsOut) {
-						return false;
-					}
-					break;
-				case NUM_BYTES_OUT:
-					if (((LongCounter) entry.getValue()).getLocalValue() < expectedBytesOut) {
-						return false;
-					}
-					break;
-				/**
-				 * The following two cases are for the DataSink task
-				 */
-				case NUM_RECORDS_IN:
-					if (((LongCounter) entry.getValue()).getLocalValue() < expectedRecordsIn) {
-						return false;
-					}
-					break;
-				case NUM_BYTES_IN:
-					if (((LongCounter) entry.getValue()).getLocalValue() < expectedBytesIn) {
-						return false;
-					}
-					break;
-				default:
-					fail("Unknown accumulator found.");
-			}
-		}
-		return true;
-	}
-
-
 	/**
 	 * UDF that notifies when it changes the accumulator values
 	 */


[2/5] flink git commit: [FLINK-4733] Reuse operator IO metrics for task

Posted by ch...@apache.org.
[FLINK-4733] Reuse operator IO metrics for task


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1f4f6f92
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1f4f6f92
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1f4f6f92

Branch: refs/heads/master
Commit: 1f4f6f928cb71fcbcba8d5aca731e738352d1175
Parents: 99f1dc3
Author: zentol <ch...@apache.org>
Authored: Mon Oct 31 14:08:26 2016 +0100
Committer: zentol <ch...@apache.org>
Committed: Mon Oct 31 15:12:02 2016 +0100

----------------------------------------------------------------------
 .../flink/runtime/executiongraph/IOMetrics.java | 99 ++++++++++++++++++++
 .../metrics/groups/OperatorIOMetricGroup.java   | 17 ++++
 .../metrics/groups/TaskIOMetricGroup.java       | 67 ++++++++++++-
 .../flink/runtime/operators/BatchTask.java      |  8 ++
 .../flink/runtime/operators/DataSinkTask.java   |  2 +
 .../flink/runtime/operators/DataSourceTask.java |  4 +
 .../operators/chaining/ChainedDriver.java       |  4 +
 .../metrics/groups/TaskIOMetricGroupTest.java   | 61 ++++++++++++
 .../flink/streaming/api/graph/StreamConfig.java |  9 ++
 .../api/graph/StreamingJobGraphGenerator.java   |  3 +
 .../api/operators/AbstractStreamOperator.java   |  6 ++
 11 files changed, 278 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IOMetrics.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IOMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IOMetrics.java
new file mode 100644
index 0000000..15c54b4
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/IOMetrics.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.executiongraph;
+
+import org.apache.flink.metrics.Meter;
+
+import java.io.Serializable;
+
+/**
+ * An instance of this class represents a snapshot of the io-related metrics of a single task.
+ */
+public class IOMetrics implements Serializable {
+	private static final long serialVersionUID = -7208093607556457183L;
+	private final long numRecordsIn;
+	private final long numRecordsOut;
+
+	private final double numRecordsInPerSecond;
+	private final double numRecordsOutPerSecond;
+
+	private final long numBytesInLocal;
+	private final long numBytesInRemote;
+	private final long numBytesOut;
+
+	private final double numBytesInLocalPerSecond;
+	private final double numBytesInRemotePerSecond;
+	private final double numBytesOutPerSecond;
+
+	public IOMetrics(Meter recordsIn, Meter recordsOut, Meter bytesLocalIn, Meter bytesRemoteIn, Meter bytesOut) {
+		this.numRecordsIn = recordsIn.getCount();
+		this.numRecordsInPerSecond = recordsIn.getRate();
+		this.numRecordsOut = recordsOut.getCount();
+		this.numRecordsOutPerSecond = recordsOut.getRate();
+		this.numBytesInLocal = bytesLocalIn.getCount();
+		this.numBytesInLocalPerSecond = bytesLocalIn.getRate();
+		this.numBytesInRemote = bytesRemoteIn.getCount();
+		this.numBytesInRemotePerSecond = bytesRemoteIn.getRate();
+		this.numBytesOut = bytesOut.getCount();
+		this.numBytesOutPerSecond = bytesOut.getRate();
+	}
+
+	public long getNumRecordsIn() {
+		return numRecordsIn;
+	}
+
+	public long getNumRecordsOut() {
+		return numRecordsOut;
+	}
+
+	public long getNumBytesInLocal() {
+		return numBytesInLocal;
+	}
+
+	public long getNumBytesInRemote() {
+		return numBytesInRemote;
+	}
+
+	public long getNumBytesInTotal() {
+		return numBytesInLocal + numBytesInRemote;
+	}
+
+	public long getNumBytesOut() {
+		return numBytesOut;
+	}
+
+	public double getNumRecordsInPerSecond() {
+		return numRecordsInPerSecond;
+	}
+
+	public double getNumRecordsOutPerSecond() {
+		return numRecordsOutPerSecond;
+	}
+
+	public double getNumBytesInLocalPerSecond() {
+		return numBytesInLocalPerSecond;
+	}
+
+	public double getNumBytesInRemotePerSecond() {
+		return numBytesInRemotePerSecond;
+	}
+
+	public double getNumBytesOutPerSecond() {
+		return numBytesOutPerSecond;
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/OperatorIOMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/OperatorIOMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/OperatorIOMetricGroup.java
index 32611fd..2e321fe 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/OperatorIOMetricGroup.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/OperatorIOMetricGroup.java
@@ -56,4 +56,21 @@ public class OperatorIOMetricGroup extends ProxyMetricGroup<OperatorMetricGroup>
 	public Meter getNumRecordsOutRate() {
 		return numRecordsOutRate;
 	}
+
+	/**
+	 * Causes the containing task to use this operators input record counter.
+	 */
+	public void reuseInputMetricsForTask() {
+		TaskIOMetricGroup taskIO = parentMetricGroup.parent().getIOMetricGroup();
+		taskIO.reuseRecordsInputCounter(this.numRecordsIn);
+		
+	}
+
+	/**
+	 * Causes the containing task to use this operators output record counter.
+	 */
+	public void reuseOutputMetricsForTask() {
+		TaskIOMetricGroup taskIO = parentMetricGroup.parent().getIOMetricGroup();
+		taskIO.reuseRecordsOutputCounter(this.numRecordsOut);
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroup.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroup.java
index b2884ec..4e32563 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroup.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroup.java
@@ -23,9 +23,14 @@ import org.apache.flink.metrics.Gauge;
 import org.apache.flink.metrics.Meter;
 import org.apache.flink.metrics.MeterView;
 import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.metrics.SimpleCounter;
 import org.apache.flink.runtime.io.network.partition.ResultPartition;
 import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
 import org.apache.flink.runtime.taskmanager.Task;
+import org.apache.flink.runtime.executiongraph.IOMetrics;
+
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Metric group that contains shareable pre-defined IO-related metrics. The metrics registration is
@@ -36,10 +41,14 @@ public class TaskIOMetricGroup extends ProxyMetricGroup<TaskMetricGroup> {
 	private final Counter numBytesOut;
 	private final Counter numBytesInLocal;
 	private final Counter numBytesInRemote;
+	private final SumCounter numRecordsIn;
+	private final SumCounter numRecordsOut;
 
 	private final Meter numBytesInRateLocal;
 	private final Meter numBytesInRateRemote;
 	private final Meter numBytesOutRate;
+	private final Meter numRecordsInRate;
+	private final Meter numRecordsOutRate;
 
 	private final MetricGroup buffers;
 
@@ -52,10 +61,21 @@ public class TaskIOMetricGroup extends ProxyMetricGroup<TaskMetricGroup> {
 		this.numBytesOutRate = meter("numBytesOutPerSecond", new MeterView(numBytesOut, 60));
 		this.numBytesInRateLocal = meter("numBytesInLocalPerSecond", new MeterView(numBytesInLocal, 60));
 		this.numBytesInRateRemote = meter("numBytesInRemotePerSecond", new MeterView(numBytesInRemote, 60));
+		this.numRecordsIn = counter("numRecordsIn", new SumCounter());
+		this.numRecordsOut = counter("numRecordsOut", new SumCounter());
+		this.numRecordsInRate = meter("numRecordsInPerSecond", new MeterView(numRecordsIn, 60));
+		this.numRecordsOutRate = meter("numRecordsOutPerSecond", new MeterView(numRecordsOut, 60));
 
 		this.buffers = addGroup("buffers");
 	}
 
+	public IOMetrics createSnapshot() {
+		return new IOMetrics(numRecordsInRate, numRecordsOutRate, numBytesInRateLocal, numBytesInRateRemote, numBytesOutRate);
+	}
+
+	// ============================================================================================
+	// Getters
+	// ============================================================================================
 	public Counter getNumBytesOutCounter() {
 		return numBytesOut;
 	}
@@ -68,11 +88,19 @@ public class TaskIOMetricGroup extends ProxyMetricGroup<TaskMetricGroup> {
 		return numBytesInRemote;
 	}
 
-	public Meter getNumBytesInRateLocalMeter() {
+	public Counter getNumRecordsInCounter() {
+		return numRecordsIn;
+	}
+
+	public Counter getNumRecordsOutCounter() {
+		return numRecordsOut;
+	}
+
+	public Meter getNumBytesInLocalRateMeter() {
 		return numBytesInRateLocal;
 	}
 
-	public Meter getNumBytesInRateRemoteMeter() {
+	public Meter getNumBytesInRemoteRateMeter() {
 		return numBytesInRateRemote;
 	}
 
@@ -134,6 +162,41 @@ public class TaskIOMetricGroup extends ProxyMetricGroup<TaskMetricGroup> {
 		}
 	}
 
+	// ============================================================================================
+	// Metric Reuse
+	// ============================================================================================
+	public void reuseRecordsInputCounter(Counter numRecordsInCounter) {
+		this.numRecordsIn.addCounter(numRecordsInCounter);
+	}
+
+	public void reuseRecordsOutputCounter(Counter numRecordsOutCounter) {
+		this.numRecordsOut.addCounter(numRecordsOutCounter);
+	}
+
+	/**
+	 * A {@link SimpleCounter} that can contain other {@link Counter}s. A call to {@link SumCounter#getCount()} returns
+	 * the sum of this counters and all contained counters.
+	 */
+	private static class SumCounter extends SimpleCounter {
+		private final List<Counter> internalCounters = new ArrayList<>();
+
+		SumCounter() {
+		}
+
+		public void addCounter(Counter toAdd) {
+			internalCounters.add(toAdd);
+		}
+
+		@Override
+		public long getCount() {
+			long sum = super.getCount();
+			for (Counter counter : internalCounters) {
+				sum += counter.getCount();
+			}
+			return sum;
+		}
+	}
+
 	/**
 	 * Input buffer pool usage gauge of a task
 	 */

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
index 354dbac..e896639 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
@@ -243,6 +243,10 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 		String headName =  getEnvironment().getTaskInfo().getTaskName().split("->")[0].trim();
 		this.metrics = getEnvironment().getMetricGroup()
 			.addOperator(headName.startsWith("CHAIN") ? headName.substring(6) : headName);
+		this.metrics.getIOMetricGroup().reuseInputMetricsForTask();
+		if (config.getNumberOfChainedStubs() == 0) {
+			this.metrics.getIOMetricGroup().reuseOutputMetricsForTask();
+		}
 
 		// initialize the readers.
 		// this does not yet trigger any stream consuming or processing.
@@ -1306,6 +1310,10 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 				ct.setup(chainedStubConf, taskName, previous, containingTask, cl, executionConfig, accumulatorMap);
 				chainedTasksTarget.add(0, ct);
 
+				if (i == numChained - 1) {
+					ct.getIOMetrics().reuseOutputMetricsForTask();
+				}
+
 				previous = ct;
 			}
 			// the collector of the first in the chain is the collector for the task

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
index 4626b69..eb7999c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
@@ -109,6 +109,8 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 
 		RuntimeContext ctx = createRuntimeContext();
 		final Counter numRecordsIn = ((OperatorMetricGroup) ctx.getMetricGroup()).getIOMetricGroup().getNumRecordsInCounter();
+		((OperatorMetricGroup) ctx.getMetricGroup()).getIOMetricGroup().reuseInputMetricsForTask();
+		((OperatorMetricGroup) ctx.getMetricGroup()).getIOMetricGroup().reuseOutputMetricsForTask();
 		
 		if(RichOutputFormat.class.isAssignableFrom(this.format.getClass())){
 			((RichOutputFormat) this.format).setRuntimeContext(ctx);

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
index 4dc3ef5..e89559d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
@@ -103,7 +103,11 @@ public class DataSourceTask<OT> extends AbstractInvokable {
 
 		RuntimeContext ctx = createRuntimeContext();
 		Counter completedSplitsCounter = ctx.getMetricGroup().counter("numSplitsProcessed");
+		((OperatorMetricGroup) ctx.getMetricGroup()).getIOMetricGroup().reuseInputMetricsForTask();
 		Counter numRecordsOut = ((OperatorMetricGroup) ctx.getMetricGroup()).getIOMetricGroup().getNumRecordsOutCounter();
+		if (this.config.getNumberOfChainedStubs() == 0) {
+			((OperatorMetricGroup) ctx.getMetricGroup()).getIOMetricGroup().reuseOutputMetricsForTask();
+		}
 
 		if (RichInputFormat.class.isAssignableFrom(this.format.getClass())) {
 			((RichInputFormat) this.format).setRuntimeContext(ctx);

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
index cf62dfa..442a53c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
@@ -25,6 +25,7 @@ import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.metrics.groups.OperatorIOMetricGroup;
 import org.apache.flink.runtime.metrics.groups.OperatorMetricGroup;
 import org.apache.flink.runtime.operators.BatchTask;
 import org.apache.flink.runtime.operators.util.DistributedRuntimeUDFContext;
@@ -104,6 +105,9 @@ public abstract class ChainedDriver<IT, OT> implements Collector<IT> {
 	@Override
 	public abstract void collect(IT record);
 
+	public OperatorIOMetricGroup getIOMetrics() {
+		return this.metrics.getIOMetricGroup();
+	}
 	
 	protected RuntimeContext getUdfRuntimeContext() {
 		return this.udfContext;

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroupTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroupTest.java
new file mode 100644
index 0000000..564a518
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/TaskIOMetricGroupTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.runtime.metrics.groups;
+
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.SimpleCounter;
+import org.apache.flink.runtime.executiongraph.IOMetrics;
+import org.apache.flink.runtime.operators.testutils.UnregisteredTaskMetricsGroup;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class TaskIOMetricGroupTest {
+	@Test
+	public void testTaskIOMetricGroup() {
+		TaskMetricGroup task = new UnregisteredTaskMetricsGroup();
+		TaskIOMetricGroup taskIO = task.getIOMetricGroup();
+
+		// test counter forwarding
+		assertNotNull(taskIO.getNumRecordsInCounter());
+		assertNotNull(taskIO.getNumRecordsOutCounter());
+
+		Counter c1 = new SimpleCounter();
+		c1.inc(32L);
+		Counter c2 = new SimpleCounter();
+		c2.inc(64L);
+
+		taskIO.reuseRecordsInputCounter(c1);
+		taskIO.reuseRecordsOutputCounter(c2);
+		assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
+		assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());
+
+		// test IOMetrics instantiation
+		taskIO.getNumBytesInLocalCounter().inc(100L);
+		taskIO.getNumBytesInRemoteCounter().inc(150L);
+		taskIO.getNumBytesOutCounter().inc(250L);
+		
+		IOMetrics io = taskIO.createSnapshot();
+		assertEquals(32L, io.getNumRecordsIn());
+		assertEquals(64L, io.getNumRecordsOut());
+		assertEquals(100L, io.getNumBytesInLocal());
+		assertEquals(150L, io.getNumBytesInRemote());
+		assertEquals(250L, io.getNumBytesOut());
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
index 2d38fb9..dd4c55c 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java
@@ -69,6 +69,7 @@ public class StreamConfig implements Serializable {
 	private static final String OUT_STREAM_EDGES = "outStreamEdges";
 	private static final String IN_STREAM_EDGES = "inStreamEdges";
 	private static final String OPERATOR_NAME = "operatorName";
+	private static final String CHAIN_END = "chainEnd";
 
 	private static final String CHECKPOINTING_ENABLED = "checkpointing";
 	private static final String CHECKPOINT_MODE = "checkpointMode";
@@ -478,6 +479,14 @@ public class StreamConfig implements Serializable {
 		return config.getBoolean(IS_CHAINED_VERTEX, false);
 	}
 
+	public void setChainEnd() {
+		config.setBoolean(CHAIN_END, true);
+	}
+
+	public boolean isChainEnd() {
+		return config.getBoolean(CHAIN_END, false);
+	}
+
 	@Override
 	public String toString() {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 87fd7eb..8f9da8a 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -231,6 +231,9 @@ public class StreamingJobGraphGenerator {
 				config.setChainIndex(chainIndex);
 				config.setOperatorName(streamGraph.getStreamNode(currentNodeId).getOperatorName());
 				chainedConfigs.get(startNodeId).put(currentNodeId, config);
+				if (chainableOutputs.isEmpty()) {
+					config.setChainEnd();
+				}
 			}
 
 			return transitiveOutEdges;

http://git-wip-us.apache.org/repos/asf/flink/blob/1f4f6f92/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
index 5f0dd85..a659866 100644
--- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
+++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/AbstractStreamOperator.java
@@ -163,6 +163,12 @@ public abstract class AbstractStreamOperator<OUT>
 		
 		this.metrics = container.getEnvironment().getMetricGroup().addOperator(config.getOperatorName());
 		this.output = new CountingOutput(output, ((OperatorMetricGroup) this.metrics).getIOMetricGroup().getNumRecordsOutCounter());
+		if (config.isChainStart()) {
+			((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseInputMetricsForTask();
+		}
+		if (config.isChainEnd()) {
+			((OperatorMetricGroup) this.metrics).getIOMetricGroup().reuseOutputMetricsForTask();
+		}
 		Configuration taskManagerConfig = container.getEnvironment().getTaskManagerInfo().getConfiguration();
 		int historySize = taskManagerConfig.getInteger(ConfigConstants.METRICS_LATENCY_HISTORY_SIZE, ConfigConstants.DEFAULT_METRICS_LATENCY_HISTORY_SIZE);
 		if (historySize <= 0) {


[3/5] flink git commit: [FLINK-4733] Port TaskManager metrics

Posted by ch...@apache.org.
[FLINK-4733] Port TaskManager metrics


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cf4f3644
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cf4f3644
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cf4f3644

Branch: refs/heads/master
Commit: cf4f3644c0eda42c4c872daa175d095117788086
Parents: 1f4f6f9
Author: zentol <ch...@apache.org>
Authored: Mon Oct 31 14:13:01 2016 +0100
Committer: zentol <ch...@apache.org>
Committed: Mon Oct 31 15:12:03 2016 +0100

----------------------------------------------------------------------
 .../runtime/webmonitor/WebRuntimeMonitor.java   |  4 +-
 .../handlers/TaskManagersHandler.java           | 62 +++++++++++++--
 .../runtime/webmonitor/metrics/MetricStore.java | 11 +++
 .../taskmanager/taskmanager.metrics.jade        | 79 ++++++++-----------
 .../taskmanager/taskmanager.metrics.html        | 82 ++++++++------------
 flink-runtime/pom.xml                           | 17 ++--
 .../apache/flink/runtime/instance/Instance.java | 10 ---
 .../flink/runtime/instance/InstanceManager.java |  3 +-
 .../flink/runtime/jobmanager/JobManager.scala   |  4 +-
 .../runtime/messages/TaskManagerMessages.scala  |  7 +-
 .../flink/runtime/taskmanager/TaskManager.scala | 66 +---------------
 .../runtime/instance/InstanceManagerTest.java   | 12 +--
 12 files changed, 153 insertions(+), 204 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
index e907124..a0afba2 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
@@ -287,8 +287,8 @@ public class WebRuntimeMonitor implements WebMonitor {
 			.GET("/jobs/:jobid/checkpoints", handler(new JobCheckpointsHandler(currentGraphs)))
 			.GET("/jobs/:jobid/metrics", handler(new JobMetricsHandler(metricFetcher)))
 
-			.GET("/taskmanagers", handler(new TaskManagersHandler(DEFAULT_REQUEST_TIMEOUT)))
-			.GET("/taskmanagers/:" + TaskManagersHandler.TASK_MANAGER_ID_KEY + "/metrics", handler(new TaskManagersHandler(DEFAULT_REQUEST_TIMEOUT)))
+			.GET("/taskmanagers", handler(new TaskManagersHandler(DEFAULT_REQUEST_TIMEOUT, metricFetcher)))
+			.GET("/taskmanagers/:" + TaskManagersHandler.TASK_MANAGER_ID_KEY + "/metrics", handler(new TaskManagersHandler(DEFAULT_REQUEST_TIMEOUT, metricFetcher)))
 			.GET("/taskmanagers/:" + TaskManagersHandler.TASK_MANAGER_ID_KEY + "/log", 
 				new TaskManagerLogHandler(retriever, context, jobManagerAddressPromise.future(), timeout,
 					TaskManagerLogHandler.FileMode.LOG, config, enableSSL))

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/TaskManagersHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/TaskManagersHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/TaskManagersHandler.java
index c20d4fe..42815ae 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/TaskManagersHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/TaskManagersHandler.java
@@ -25,6 +25,8 @@ import org.apache.flink.runtime.instance.InstanceID;
 import org.apache.flink.runtime.messages.JobManagerMessages;
 import org.apache.flink.runtime.messages.JobManagerMessages.RegisteredTaskManagers;
 import org.apache.flink.runtime.messages.JobManagerMessages.TaskManagerInstance;
+import org.apache.flink.runtime.webmonitor.metrics.MetricFetcher;
+import org.apache.flink.runtime.webmonitor.metrics.MetricStore;
 import org.apache.flink.util.StringUtils;
 import scala.concurrent.Await;
 import scala.concurrent.Future;
@@ -42,9 +44,12 @@ public class TaskManagersHandler extends AbstractJsonRequestHandler  {
 	public static final String TASK_MANAGER_ID_KEY = "taskmanagerid";
 	
 	private final FiniteDuration timeout;
+
+	private final MetricFetcher fetcher;
 	
-	public TaskManagersHandler(FiniteDuration timeout) {
+	public TaskManagersHandler(FiniteDuration timeout, MetricFetcher fetcher) {
 		this.timeout = requireNonNull(timeout);
+		this.fetcher = fetcher;
 	}
 
 	@Override
@@ -95,10 +100,57 @@ public class TaskManagersHandler extends AbstractJsonRequestHandler  {
 
 					// only send metrics when only one task manager requests them.
 					if (pathParams.containsKey(TASK_MANAGER_ID_KEY)) {
-						byte[] report = instance.getLastMetricsReport();
-						if (report != null) {
-							gen.writeFieldName("metrics");
-							gen.writeRawValue(new String(report, "utf-8"));
+						fetcher.update();
+						MetricStore.TaskManagerMetricStore metrics = fetcher.getMetricStore().getTaskManagerMetricStore(instance.getId().toString());
+						if (metrics != null) {
+							gen.writeObjectFieldStart("metrics");
+							long heapUsed = Long.valueOf( metrics.getMetric("Status.JVM.Memory.Heap.Used", "0"));
+							long heapCommitted = Long.valueOf( metrics.getMetric("Status.JVM.Memory.Heap.Committed", "0"));
+							long heapTotal = Long.valueOf( metrics.getMetric("Status.JVM.Memory.Heap.Max", "0"));
+
+							gen.writeNumberField("heapCommitted", heapCommitted);
+							gen.writeNumberField("heapUsed", heapUsed);
+							gen.writeNumberField("heapMax", heapTotal);
+
+							long nonHeapUsed = Long.valueOf( metrics.getMetric("Status.JVM.Memory.NonHeap.Used", "0"));
+							long nonHeapCommitted = Long.valueOf( metrics.getMetric("Status.JVM.Memory.NonHeap.Committed", "0"));
+							long nonHeapTotal = Long.valueOf( metrics.getMetric("Status.JVM.Memory.NonHeap.Max", "0"));
+
+							gen.writeNumberField("nonHeapCommitted", nonHeapCommitted);
+							gen.writeNumberField("nonHeapUsed", nonHeapUsed);
+							gen.writeNumberField("nonHeapMax", nonHeapTotal);
+
+							gen.writeNumberField("totalCommitted", heapCommitted + nonHeapCommitted);
+							gen.writeNumberField("totalUsed", heapUsed + nonHeapUsed);
+							gen.writeNumberField("totalMax", heapTotal + nonHeapTotal);
+
+							gen.writeStringField("directCount", metrics.getMetric("Status.JVM.Memory.Direct.Count", "0"));
+							gen.writeStringField("directUsed", metrics.getMetric("Status.JVM.Memory.Direct.MemoryUsed", "0"));
+							gen.writeStringField("directMax", metrics.getMetric("Status.JVM.Memory.Direct.TotalCapacity", "0"));
+
+							gen.writeStringField("mappedCount", metrics.getMetric("Status.JVM.Memory.Mapped.Count", "0"));
+							gen.writeStringField("mappedUsed", metrics.getMetric("Status.JVM.Memory.Mapped.MemoryUsed", "0"));
+							gen.writeStringField("mappedMax", metrics.getMetric("Status.JVM.Memory.Mapped.TotalCapacity", "0"));
+
+							gen.writeStringField("memorySegmentsAvailable", metrics.getMetric("Status.Network.AvailableMemorySegments", "0"));
+							gen.writeStringField("memorySegmentsTotal", metrics.getMetric("Status.Network.TotalMemorySegments", "0"));
+
+							gen.writeArrayFieldStart("garbageCollectors");
+
+							for (String gcName : metrics.garbageCollectorNames) {
+								String count = metrics.getMetric("Status.JVM.GarbageCollector." + gcName + ".Count", null);
+								String time = metrics.getMetric("Status.JVM.GarbageCollector." + gcName + ".Time", null);
+								if (count != null  && time != null) {
+									gen.writeStartObject();
+									gen.writeStringField("name", gcName);
+									gen.writeStringField("count", count);
+									gen.writeStringField("time", time);
+									gen.writeEndObject();
+								}
+							}
+
+							gen.writeEndArray();
+							gen.writeEndObject();
 						}
 					}
 

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/metrics/MetricStore.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/metrics/MetricStore.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/metrics/MetricStore.java
index 989145b..51b3b4d 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/metrics/MetricStore.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/metrics/MetricStore.java
@@ -23,7 +23,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import static org.apache.flink.runtime.metrics.dump.MetricDump.METRIC_CATEGORY_COUNTER;
 import static org.apache.flink.runtime.metrics.dump.MetricDump.METRIC_CATEGORY_GAUGE;
@@ -77,6 +79,10 @@ public class MetricStore {
 						tm = new TaskManagerMetricStore();
 						taskManagers.put(tmID, tm);
 					}
+					if (name.contains("GarbageCollector")) {
+						String gcName = name.substring("Status.JVM.GarbageCollector.".length(), name.lastIndexOf('.'));
+						tm.addGarbageCollectorName(gcName);
+					}
 					addMetric(tm.metrics, name, metric);
 					break;
 				case INFO_CATEGORY_JOB:
@@ -260,6 +266,11 @@ public class MetricStore {
 	 * Sub-structure containing metrics of a single TaskManager.
 	 */
 	public static class TaskManagerMetricStore extends ComponentMetricStore {
+		public final Set<String> garbageCollectorNames = new HashSet<>();
+		
+		public void addGarbageCollectorName(String name) {
+			garbageCollectorNames.add(name);
+		}
 	}
 
 	/**

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime-web/web-dashboard/app/partials/taskmanager/taskmanager.metrics.jade
----------------------------------------------------------------------
diff --git a/flink-runtime-web/web-dashboard/app/partials/taskmanager/taskmanager.metrics.jade b/flink-runtime-web/web-dashboard/app/partials/taskmanager/taskmanager.metrics.jade
index c546d74..7920178 100644
--- a/flink-runtime-web/web-dashboard/app/partials/taskmanager/taskmanager.metrics.jade
+++ b/flink-runtime-web/web-dashboard/app/partials/taskmanager/taskmanager.metrics.jade
@@ -50,19 +50,19 @@ div(ng-if="metrics.id")
     tbody
       tr
         td Heap
-        td {{metrics.metrics.gauges['memory.heap.committed'].value | humanizeBytes}}
-        td {{metrics.metrics.gauges['memory.heap.init'].value | humanizeBytes}}
-        td {{metrics.metrics.gauges['memory.heap.max'].value | humanizeBytes}}
+        td {{ metrics.metrics.heapCommitted | humanizeBytes  }}
+        td {{ metrics.metrics.heapUsed | humanizeBytes  }}
+        td {{ metrics.metrics.heapMax | humanizeBytes  }}
       tr
         td Non-Heap
-        td {{metrics.metrics.gauges['memory.non-heap.committed'].value | humanizeBytes}}
-        td {{metrics.metrics.gauges['memory.non-heap.init'].value | humanizeBytes}}
-        td {{metrics.metrics.gauges['memory.non-heap.max'].value | humanizeBytes}}
+        td {{ metrics.metrics.nonHeapCommitted | humanizeBytes  }}
+        td {{ metrics.metrics.nonHeapUsed | humanizeBytes  }}
+        td {{ metrics.metrics.nonHeapMax | humanizeBytes  }}
       tr
         td Total
-        td {{metrics.metrics.gauges['memory.total.committed'].value | humanizeBytes}}
-        td {{metrics.metrics.gauges['memory.total.init'].value | humanizeBytes}}
-        td {{metrics.metrics.gauges['memory.total.max'].value | humanizeBytes}}
+        td {{ metrics.metrics.totalCommitted | humanizeBytes  }}
+        td {{ metrics.metrics.totalUsed | humanizeBytes  }}
+        td {{ metrics.metrics.totalMax | humanizeBytes  }}
 
   h2 Outside JVM
   table.table.table-properties
@@ -75,54 +75,41 @@ div(ng-if="metrics.id")
     tbody
       tr
         td Direct
-        td {{ metrics.metrics.gauges['direct-memory.direct.count'].value }}
-        td {{ metrics.metrics.gauges['direct-memory.direct.used'].value | humanizeBytes }}
-        td {{ metrics.metrics.gauges['direct-memory.direct.capacity'].value | humanizeBytes }}
+        td {{ metrics.metrics.directCount }}
+        td {{ metrics.metrics.directUsed }}
+        td {{ metrics.metrics.directTotal }}
       tr
         td Mapped
-        td {{ metrics.metrics.gauges['direct-memory.mapped.count'].value }}
-        td {{ metrics.metrics.gauges['direct-memory.mapped.used'].value | humanizeBytes }}
-        td {{ metrics.metrics.gauges['direct-memory.mapped.capacity'].value | humanizeBytes }}
+        td {{ metrics.metrics.mappedCount }}
+        td {{ metrics.metrics.mappedUsed }}
+        td {{ metrics.metrics.mappedMax }}
 
-  h1 Garbage Collection
+  h1 Network
+
+  h2 MemorySegments
   table.table.table-properties
     thead
       tr
-        th Collector
+        th Type
         th Count
-        th Time
     tbody
       tr
-        td PS-MarkSweep
-        td(table-property value="metrics.metrics.gauges['gc.PS-MarkSweep.count'].value")
-        td(table-property value="metrics.metrics.gauges['gc.PS-MarkSweep.time'].value | humanizeDuration")
+        td Available
+        td {{ metrics.metrics.memorySegmentsAvailable }}
       tr
-        td PS-Scavenge
-        td(table-property value="metrics.metrics.gauges['gc.PS-Scavenge.count'].value")
-        td(table-property value="metrics.metrics.gauges['gc.PS-Scavenge.time'].value | humanizeDuration")
+        td Total
+        td {{ metrics.metrics.memorySegmentsTotal }}
+
 
-  h1 Other Memory Pools
+  h1 Garbage Collection
   table.table.table-properties
     thead
       tr
-        th Pool
-        td Relative Usage
-    tbody
-      tr
-        td Code Cache
-        td(table-property value="metrics.metrics.gauges['memory.pools.Code-Cache.usage'].value | number:2")
-      tr
-        td Compressed Class Space
-        td(table-property value="metrics.metrics.gauges['memory.pools.Compressed-Class-Space.usage'].value | number:2")
-      tr
-        td Metaspace
-        td(table-property value="metrics.metrics.gauges['memory.pools.Metaspace.usage'].value | number:2")
-      tr
-        td PS Eden Space
-        td(table-property value="metrics.metrics.gauges['memory.pools.PS-Eden-Space.usage'].value | number:2")
-      tr
-        td PS Old Gen
-        td(table-property value="metrics.metrics.gauges['memory.pools.PS-Old-Gen.usage'].value | number:2")
-      tr
-        td PS Survivor Space
-        td(table-property value="metrics.metrics.gauges['memory.pools.PS-Survivor-Space.usage'].value | number:2")
+        th Collector
+        th Count
+        th Time
+    tbody(ng-repeat="g in metrics.metrics.garbageCollectors")
+      tr  
+        td {{ g.name }}
+        td {{ g.count }}
+        td {{ g.time }}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime-web/web-dashboard/web/partials/taskmanager/taskmanager.metrics.html
----------------------------------------------------------------------
diff --git a/flink-runtime-web/web-dashboard/web/partials/taskmanager/taskmanager.metrics.html b/flink-runtime-web/web-dashboard/web/partials/taskmanager/taskmanager.metrics.html
index e1d6670..43687cd 100644
--- a/flink-runtime-web/web-dashboard/web/partials/taskmanager/taskmanager.metrics.html
+++ b/flink-runtime-web/web-dashboard/web/partials/taskmanager/taskmanager.metrics.html
@@ -57,21 +57,21 @@ limitations under the License.
     <tbody>
       <tr>
         <td>Heap</td>
-        <td>{{metrics.metrics.gauges['memory.heap.committed'].value | humanizeBytes}}</td>
-        <td>{{metrics.metrics.gauges['memory.heap.init'].value | humanizeBytes}}</td>
-        <td>{{metrics.metrics.gauges['memory.heap.max'].value | humanizeBytes}}</td>
+        <td>{{ metrics.metrics.heapCommitted | humanizeBytes  }}</td>
+        <td>{{ metrics.metrics.heapUsed | humanizeBytes  }}</td>
+        <td>{{ metrics.metrics.heapMax | humanizeBytes  }}</td>
       </tr>
       <tr>
         <td>Non-Heap</td>
-        <td>{{metrics.metrics.gauges['memory.non-heap.committed'].value | humanizeBytes}}</td>
-        <td>{{metrics.metrics.gauges['memory.non-heap.init'].value | humanizeBytes}}</td>
-        <td>{{metrics.metrics.gauges['memory.non-heap.max'].value | humanizeBytes}}</td>
+        <td>{{ metrics.metrics.nonHeapCommitted | humanizeBytes  }}</td>
+        <td>{{ metrics.metrics.nonHeapUsed | humanizeBytes  }}</td>
+        <td>{{ metrics.metrics.nonHeapMax | humanizeBytes  }}</td>
       </tr>
       <tr>
         <td>Total</td>
-        <td>{{metrics.metrics.gauges['memory.total.committed'].value | humanizeBytes}}</td>
-        <td>{{metrics.metrics.gauges['memory.total.init'].value | humanizeBytes}}</td>
-        <td>{{metrics.metrics.gauges['memory.total.max'].value | humanizeBytes}}</td>
+        <td>{{ metrics.metrics.totalCommitted | humanizeBytes  }}</td>
+        <td>{{ metrics.metrics.totalUsed | humanizeBytes  }}</td>
+        <td>{{ metrics.metrics.totalMax | humanizeBytes  }}</td>
       </tr>
     </tbody>
   </table>
@@ -88,72 +88,52 @@ limitations under the License.
     <tbody>
       <tr>
         <td>Direct</td>
-        <td>{{ metrics.metrics.gauges['direct-memory.direct.count'].value }}</td>
-        <td>{{ metrics.metrics.gauges['direct-memory.direct.used'].value | humanizeBytes }}</td>
-        <td>{{ metrics.metrics.gauges['direct-memory.direct.capacity'].value | humanizeBytes }}</td>
+        <td>{{ metrics.metrics.directCount }}</td>
+        <td>{{ metrics.metrics.directUsed }}</td>
+        <td>{{ metrics.metrics.directTotal }}</td>
       </tr>
       <tr>
         <td>Mapped</td>
-        <td>{{ metrics.metrics.gauges['direct-memory.mapped.count'].value }}</td>
-        <td>{{ metrics.metrics.gauges['direct-memory.mapped.used'].value | humanizeBytes }}</td>
-        <td>{{ metrics.metrics.gauges['direct-memory.mapped.capacity'].value | humanizeBytes }}</td>
+        <td>{{ metrics.metrics.mappedCount }}</td>
+        <td>{{ metrics.metrics.mappedUsed }}</td>
+        <td>{{ metrics.metrics.mappedMax }}</td>
       </tr>
     </tbody>
   </table>
-  <h1>Garbage Collection</h1>
+  <h1>Network</h1>
+  <h2>MemorySegments</h2>
   <table class="table table-properties">
     <thead>
       <tr>
-        <th>Collector</th>
+        <th>Type</th>
         <th>Count</th>
-        <th>Time</th>
       </tr>
     </thead>
     <tbody>
       <tr>
-        <td>PS-MarkSweep</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['gc.PS-MarkSweep.count'].value"></td>
-        <td table-property="table-property" value="metrics.metrics.gauges['gc.PS-MarkSweep.time'].value | humanizeDuration"></td>
+        <td>Available</td>
+        <td>{{ metrics.metrics.memorySegmentsAvailable }}</td>
       </tr>
       <tr>
-        <td>PS-Scavenge</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['gc.PS-Scavenge.count'].value"></td>
-        <td table-property="table-property" value="metrics.metrics.gauges['gc.PS-Scavenge.time'].value | humanizeDuration"></td>
+        <td>Total</td>
+        <td>{{ metrics.metrics.memorySegmentsTotal }}</td>
       </tr>
     </tbody>
   </table>
-  <h1>Other Memory Pools</h1>
+  <h1>Garbage Collection</h1>
   <table class="table table-properties">
     <thead>
       <tr>
-        <th>Pool</th>
-        <td>Relative Usage</td>
+        <th>Collector</th>
+        <th>Count</th>
+        <th>Time</th>
       </tr>
     </thead>
-    <tbody>
-      <tr>
-        <td>Code Cache</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['memory.pools.Code-Cache.usage'].value | number:2"></td>
-      </tr>
-      <tr>
-        <td>Compressed Class Space</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['memory.pools.Compressed-Class-Space.usage'].value | number:2"></td>
-      </tr>
-      <tr>
-        <td>Metaspace</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['memory.pools.Metaspace.usage'].value | number:2"></td>
-      </tr>
-      <tr>
-        <td>PS Eden Space</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['memory.pools.PS-Eden-Space.usage'].value | number:2"></td>
-      </tr>
-      <tr>
-        <td>PS Old Gen</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['memory.pools.PS-Old-Gen.usage'].value | number:2"></td>
-      </tr>
-      <tr>
-        <td>PS Survivor Space</td>
-        <td table-property="table-property" value="metrics.metrics.gauges['memory.pools.PS-Survivor-Space.usage'].value | number:2"></td>
+    <tbody ng-repeat="g in metrics.metrics.garbageCollectors">
+      <tr> 
+        <td>{{ g.name }}</td>
+        <td>{{ g.count }}</td>
+        <td>{{ g.time }}</td>
       </tr>
     </tbody>
   </table>

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 5fea8fb..0a294fa 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -123,20 +123,15 @@ under the License.
 		</dependency>
 
 		<dependency>
-			<groupId>io.dropwizard.metrics</groupId>
-			<artifactId>metrics-core</artifactId>
-			<version>${metrics.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>io.dropwizard.metrics</groupId>
-			<artifactId>metrics-jvm</artifactId>
-			<version>${metrics.version}</version>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-core</artifactId>
+			<version>${jackson.version}</version>
 		</dependency>
 
 		<dependency>
-			<groupId>io.dropwizard.metrics</groupId>
-			<artifactId>metrics-json</artifactId>
-			<version>${metrics.version}</version>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+			<version>${jackson.version}</version>
 		</dependency>
 
 		<dependency>

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime/src/main/java/org/apache/flink/runtime/instance/Instance.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/Instance.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/Instance.java
index 4a8139b..d63d475 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/Instance.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/Instance.java
@@ -75,8 +75,6 @@ public class Instance implements SlotOwner {
 	/** Time when last heat beat has been received from the task manager running on this taskManager. */
 	private volatile long lastReceivedHeartBeat = System.currentTimeMillis();
 
-	private byte[] lastMetricsReport;
-
 	/** Flag marking the instance as alive or as dead. */
 	private volatile boolean isDead;
 
@@ -189,14 +187,6 @@ public class Instance implements SlotOwner {
 		this.lastReceivedHeartBeat = System.currentTimeMillis();
 	}
 
-	public void setMetricsReport(byte[] lastMetricsReport) {
-		this.lastMetricsReport = lastMetricsReport;
-	}
-
-	public byte[] getLastMetricsReport() {
-		return lastMetricsReport;
-	}
-
 	/**
 	 * Checks whether the last heartbeat occurred within the last {@code n} milliseconds
 	 * before the given timestamp {@code now}.

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime/src/main/java/org/apache/flink/runtime/instance/InstanceManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/InstanceManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/InstanceManager.java
index b0e7e57..132ee6f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/instance/InstanceManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/instance/InstanceManager.java
@@ -96,7 +96,7 @@ public class InstanceManager {
 		}
 	}
 
-	public boolean reportHeartBeat(InstanceID instanceId, byte[] lastMetricsReport) {
+	public boolean reportHeartBeat(InstanceID instanceId) {
 		if (instanceId == null) {
 			throw new IllegalArgumentException("InstanceID may not be null.");
 		}
@@ -118,7 +118,6 @@ public class InstanceManager {
 			}
 
 			host.reportHeartBeat();
-			host.setMetricsReport(lastMetricsReport);
 
 			if (LOG.isDebugEnabled()) {
 				LOG.debug("Received heartbeat from TaskManager " + host);

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 516bbbe..bcfdd23 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -1028,12 +1028,12 @@ class JobManager(
         TaskManagerInstance(Option(instanceManager.getRegisteredInstanceById(instanceID)))
       )
 
-    case Heartbeat(instanceID, metricsReport, accumulators) =>
+    case Heartbeat(instanceID, accumulators) =>
       log.debug(s"Received heartbeat message from $instanceID.")
 
       updateAccumulators(accumulators)
 
-      instanceManager.reportHeartBeat(instanceID, metricsReport)
+      instanceManager.reportHeartBeat(instanceID)
 
     case message: AccumulatorMessage => handleAccumulatorMessage(message)
 

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
index b433015..a493b3d 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
@@ -52,15 +52,12 @@ object TaskManagerMessages {
 
   /**
    * Reports liveliness of the TaskManager instance with the given instance ID to the
-   * This message is sent to the job. This message reports the TaskManagers
-   * metrics, as a byte array.
+   * This message is sent to the job.
    *
    * @param instanceID The instance ID of the reporting TaskManager.
-   * @param metricsReport utf-8 encoded JSON metrics report from the metricRegistry.
    * @param accumulators Accumulators of tasks serialized as Tuple2[internal, user-defined]
    */
-  case class Heartbeat(instanceID: InstanceID, metricsReport: Array[Byte],
-     accumulators: Seq[AccumulatorSnapshot])
+  case class Heartbeat(instanceID: InstanceID, accumulators: Seq[AccumulatorSnapshot])
 
 
   // --------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 9727860..7608b87 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -19,7 +19,7 @@
 package org.apache.flink.runtime.taskmanager
 
 import java.io.{File, FileInputStream, IOException}
-import java.lang.management.{ManagementFactory, OperatingSystemMXBean}
+import java.lang.management.ManagementFactory
 import java.net.{InetAddress, InetSocketAddress}
 import java.util
 import java.util.UUID
@@ -28,10 +28,6 @@ import java.util.concurrent.TimeUnit
 import _root_.akka.actor._
 import _root_.akka.pattern.ask
 import _root_.akka.util.Timeout
-import com.codahale.metrics.json.MetricsModule
-import com.codahale.metrics.jvm.{BufferPoolMetricSet, GarbageCollectorMetricSet, MemoryUsageGaugeSet}
-import com.codahale.metrics.{Gauge, MetricFilter, MetricRegistry}
-import com.fasterxml.jackson.databind.ObjectMapper
 import grizzled.slf4j.Logger
 import org.apache.commons.lang3.exception.ExceptionUtils
 import org.apache.flink.configuration._
@@ -155,20 +151,8 @@ class TaskManager(
   /** Handler for distributed files cached by this TaskManager */
   protected val fileCache = new FileCache(config.configuration)
 
-  /** Registry of metrics periodically transmitted to the JobManager */
-  private val metricRegistry = TaskManager.createMetricsRegistry()
-
   private var taskManagerMetricGroup : TaskManagerMetricGroup = _
 
-  /** Metric serialization */
-  private val metricRegistryMapper: ObjectMapper = new ObjectMapper()
-    .registerModule(
-      new MetricsModule(
-        TimeUnit.SECONDS,
-        TimeUnit.MILLISECONDS,
-        false,
-        MetricFilter.ALL))
-
   /** Actors which want to be notified once this task manager has been
     * registered at the job manager */
   private val waitForRegistration = scala.collection.mutable.Set[ActorRef]()
@@ -1332,7 +1316,6 @@ class TaskManager(
   protected def sendHeartbeatToJobManager(): Unit = {
     try {
       log.debug("Sending heartbeat to JobManager")
-      val metricsReport: Array[Byte] = metricRegistryMapper.writeValueAsBytes(metricRegistry)
 
       val accumulatorEvents =
         scala.collection.mutable.Buffer[AccumulatorSnapshot]()
@@ -1351,7 +1334,7 @@ class TaskManager(
       }
 
        currentJobManager foreach {
-        jm => jm ! decorateMessage(Heartbeat(instanceID, metricsReport, accumulatorEvents))
+        jm => jm ! decorateMessage(Heartbeat(instanceID, accumulatorEvents))
       }
     }
     catch {
@@ -2481,49 +2464,4 @@ object TaskManager {
       case (_, id) => throw new IllegalArgumentException(s"Temporary file directory #$id is null.")
     }
   }
-
-  /**
-   * Creates the registry of default metrics, including stats about garbage collection, memory
-   * usage, and system CPU load.
-   *
-   * @return The registry with the default metrics.
-   */
-  private def createMetricsRegistry() : MetricRegistry = {
-    val metricRegistry = new MetricRegistry()
-
-    // register default metrics
-    metricRegistry.register("gc", new GarbageCollectorMetricSet)
-    metricRegistry.register("memory", new MemoryUsageGaugeSet)
-    metricRegistry.register("direct-memory", new BufferPoolMetricSet(
-      ManagementFactory.getPlatformMBeanServer))
-    metricRegistry.register("load", new Gauge[Double] {
-      override def getValue: Double =
-        ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage()
-    })
-
-    val osBean: OperatingSystemMXBean = ManagementFactory.getOperatingSystemMXBean()
-
-    try {
-      val fetchCPULoadMethod = Class.forName("com.sun.management.OperatingSystemMXBean")
-        .getMethods()
-        .find( _.getName() == "getProcessCpuLoad" )
-
-      // verify that we can invoke the method
-      fetchCPULoadMethod.map(_.invoke(osBean).asInstanceOf[Double]).getOrElse(-1.0)
-
-      metricRegistry.register("cpuLoad", new Gauge[Double] {
-        override def getValue: Double = fetchCPULoadMethod
-          .map(_.invoke(osBean).asInstanceOf[Double]).getOrElse(-1.0)
-      })
-    }
-    catch {
-      case t: Throwable =>
-        LOG.warn("Cannot access com.sun.management.OperatingSystemMXBean.getProcessCpuLoad()" +
-          " - CPU load metrics will not be available.")
-        metricRegistry.register("cpuLoad", new Gauge[Double] {
-          override def getValue: Double = -1.0
-        })
-    }
-    metricRegistry
-  }
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/cf4f3644/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceManagerTest.java
index f3747c8..2b10e09 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/instance/InstanceManagerTest.java
@@ -186,12 +186,12 @@ public class InstanceManagerTest{
 					probe3.getRef(), ici3, hardwareDescription, 1, leaderSessionID);
 
 			// report some immediate heart beats
-			assertTrue(cm.reportHeartBeat(instanceID1, new byte[] {}));
-			assertTrue(cm.reportHeartBeat(instanceID2, new byte[] {}));
-			assertTrue(cm.reportHeartBeat(instanceID3, new byte[] {}));
+			assertTrue(cm.reportHeartBeat(instanceID1));
+			assertTrue(cm.reportHeartBeat(instanceID2));
+			assertTrue(cm.reportHeartBeat(instanceID3));
 
 			// report heart beat for non-existing instance
-			assertFalse(cm.reportHeartBeat(new InstanceID(), new byte[] {}));
+			assertFalse(cm.reportHeartBeat(new InstanceID()));
 
 			final long WAIT = 200;
 			CommonTestUtils.sleepUninterruptibly(WAIT);
@@ -205,7 +205,7 @@ public class InstanceManagerTest{
 			long h3 = it.next().getLastHeartBeat();
 
 			// send one heart beat again and verify that the
-			assertTrue(cm.reportHeartBeat(instance1.getId(), new byte[] {}));
+			assertTrue(cm.reportHeartBeat(instance1.getId()));
 			long newH1 = instance1.getLastHeartBeat();
 
 			long now = System.currentTimeMillis();
@@ -244,7 +244,7 @@ public class InstanceManagerTest{
 				// expected
 			}
 			
-			assertFalse(cm.reportHeartBeat(new InstanceID(), new byte[] {}));
+			assertFalse(cm.reportHeartBeat(new InstanceID()));
 		}
 		catch (Exception e) {
 			System.err.println(e.getMessage());


[5/5] flink git commit: [FLINK-4733] Port Task IO metrics

Posted by ch...@apache.org.
[FLINK-4733] Port Task IO metrics


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ba2d007e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ba2d007e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ba2d007e

Branch: refs/heads/master
Commit: ba2d007e5ad270b9a403d037d186de61cdaac742
Parents: cf4f364
Author: zentol <ch...@apache.org>
Authored: Mon Oct 31 14:17:05 2016 +0100
Committer: zentol <ch...@apache.org>
Committed: Mon Oct 31 15:12:04 2016 +0100

----------------------------------------------------------------------
 .../runtime/webmonitor/WebRuntimeMonitor.java   | 12 +--
 .../webmonitor/handlers/JobDetailsHandler.java  | 53 +++++++++----
 .../handlers/JobVertexDetailsHandler.java       | 63 ++++++++-------
 .../handlers/JobVertexTaskManagersHandler.java  | 57 +++++++-------
 .../SubtaskCurrentAttemptDetailsHandler.java    |  5 +-
 .../SubtaskExecutionAttemptDetailsHandler.java  | 63 ++++++++-------
 .../accumulators/AccumulatorRegistry.java       | 80 +------------------
 .../accumulators/AccumulatorSnapshot.java       | 15 ----
 .../runtime/executiongraph/AccessExecution.java | 15 +---
 .../executiongraph/AccessExecutionGraph.java    | 11 ---
 .../AccessExecutionJobVertex.java               | 13 ----
 .../executiongraph/ArchivedExecution.java       | 18 ++---
 .../executiongraph/ArchivedExecutionGraph.java  | 21 -----
 .../ArchivedExecutionJobVertex.java             |  8 --
 .../flink/runtime/executiongraph/Execution.java | 24 +++---
 .../runtime/executiongraph/ExecutionGraph.java  | 28 +------
 .../executiongraph/ExecutionJobVertex.java      | 33 --------
 .../api/reader/AbstractRecordReader.java        |  8 --
 .../io/network/api/reader/BufferReader.java     |  6 --
 .../io/network/api/reader/ReaderBase.java       |  6 --
 .../AdaptiveSpanningRecordDeserializer.java     | 31 --------
 .../api/serialization/RecordDeserializer.java   |  6 --
 .../api/serialization/RecordSerializer.java     |  6 --
 .../serialization/SpanningRecordSerializer.java | 13 ----
 ...llingAdaptiveSpanningRecordDeserializer.java | 31 --------
 .../io/network/api/writer/RecordWriter.java     | 10 ---
 .../iterative/task/IterationHeadTask.java       |  4 +-
 .../flink/runtime/operators/BatchTask.java      | 22 ++----
 .../flink/runtime/operators/DataSinkTask.java   |  6 --
 .../flink/runtime/operators/DataSourceTask.java |  6 +-
 .../apache/flink/runtime/taskmanager/Task.java  |  4 +
 .../runtime/taskmanager/TaskExecutionState.java | 13 +++-
 .../flink/runtime/taskmanager/TaskManager.scala |  4 +-
 .../ArchivedExecutionGraphTest.java             | 23 ------
 .../ExecutionGraphDeploymentTest.java           |  7 +-
 .../network/api/reader/AbstractReaderTest.java  |  6 --
 .../testingUtils/TestingJobManagerLike.scala    |  3 +-
 .../TestingJobManagerMessages.scala             |  2 -
 .../runtime/io/StreamInputProcessor.java        |  7 --
 .../runtime/io/StreamTwoInputProcessor.java     |  7 --
 .../runtime/tasks/OneInputStreamTask.java       |  4 -
 .../streaming/runtime/tasks/OperatorChain.java  |  8 +-
 .../streaming/runtime/tasks/StreamTask.java     |  2 +-
 .../runtime/tasks/TwoInputStreamTask.java       |  4 -
 .../operators/StreamOperatorChainingTest.java   |  9 +--
 .../accumulators/AccumulatorLiveITCase.java     | 82 ++------------------
 46 files changed, 207 insertions(+), 652 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
index a0afba2..7d2b5b6 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/WebRuntimeMonitor.java
@@ -262,12 +262,12 @@ public class WebRuntimeMonitor implements WebMonitor {
 
 			.GET("/jobs", handler(new CurrentJobIdsHandler(DEFAULT_REQUEST_TIMEOUT)))
 
-			.GET("/jobs/:jobid", handler(new JobDetailsHandler(currentGraphs)))
-			.GET("/jobs/:jobid/vertices", handler(new JobDetailsHandler(currentGraphs)))
+			.GET("/jobs/:jobid", handler(new JobDetailsHandler(currentGraphs, metricFetcher)))
+			.GET("/jobs/:jobid/vertices", handler(new JobDetailsHandler(currentGraphs, metricFetcher)))
 
-			.GET("/jobs/:jobid/vertices/:vertexid", handler(new JobVertexDetailsHandler(currentGraphs)))
+			.GET("/jobs/:jobid/vertices/:vertexid", handler(new JobVertexDetailsHandler(currentGraphs, metricFetcher)))
 			.GET("/jobs/:jobid/vertices/:vertexid/subtasktimes", handler(new SubtasksTimesHandler(currentGraphs)))
-			.GET("/jobs/:jobid/vertices/:vertexid/taskmanagers", handler(new JobVertexTaskManagersHandler(currentGraphs)))
+			.GET("/jobs/:jobid/vertices/:vertexid/taskmanagers", handler(new JobVertexTaskManagersHandler(currentGraphs, metricFetcher)))
 			.GET("/jobs/:jobid/vertices/:vertexid/accumulators", handler(new JobVertexAccumulatorsHandler(currentGraphs)))
 			.GET("/jobs/:jobid/vertices/:vertexid/checkpoints", handler(new JobVertexCheckpointsHandler(currentGraphs)))
 			.GET("/jobs/:jobid/vertices/:vertexid/backpressure", handler(new JobVertexBackPressureHandler(
@@ -276,8 +276,8 @@ public class WebRuntimeMonitor implements WebMonitor {
 							refreshInterval)))
 			.GET("/jobs/:jobid/vertices/:vertexid/metrics", handler(new JobVertexMetricsHandler(metricFetcher)))
 			.GET("/jobs/:jobid/vertices/:vertexid/subtasks/accumulators", handler(new SubtasksAllAccumulatorsHandler(currentGraphs)))
-			.GET("/jobs/:jobid/vertices/:vertexid/subtasks/:subtasknum", handler(new SubtaskCurrentAttemptDetailsHandler(currentGraphs)))
-			.GET("/jobs/:jobid/vertices/:vertexid/subtasks/:subtasknum/attempts/:attempt", handler(new SubtaskExecutionAttemptDetailsHandler(currentGraphs)))
+			.GET("/jobs/:jobid/vertices/:vertexid/subtasks/:subtasknum", handler(new SubtaskCurrentAttemptDetailsHandler(currentGraphs, metricFetcher)))
+			.GET("/jobs/:jobid/vertices/:vertexid/subtasks/:subtasknum/attempts/:attempt", handler(new SubtaskExecutionAttemptDetailsHandler(currentGraphs, metricFetcher)))
 			.GET("/jobs/:jobid/vertices/:vertexid/subtasks/:subtasknum/attempts/:attempt/accumulators", handler(new SubtaskExecutionAttemptAccumulatorsHandler(currentGraphs)))
 
 			.GET("/jobs/:jobid/plan", handler(new JobPlanHandler(currentGraphs)))

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
index e7a2a8c..6de6dc5 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobDetailsHandler.java
@@ -20,16 +20,16 @@ package org.apache.flink.runtime.webmonitor.handlers;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.LongCounter;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
 import org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.IOMetrics;
 import org.apache.flink.runtime.jobgraph.JobStatus;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
+import org.apache.flink.runtime.webmonitor.metrics.MetricFetcher;
+import org.apache.flink.runtime.webmonitor.metrics.MetricStore;
 
 import java.io.StringWriter;
 import java.util.Map;
@@ -45,9 +45,12 @@ import java.util.Map;
  * </ul>
  */
 public class JobDetailsHandler extends AbstractExecutionGraphRequestHandler {
-	
-	public JobDetailsHandler(ExecutionGraphHolder executionGraphHolder) {
+
+	private final MetricFetcher fetcher;
+
+	public JobDetailsHandler(ExecutionGraphHolder executionGraphHolder, MetricFetcher fetcher) {
 		super(executionGraphHolder);
+		this.fetcher = fetcher;
 	}
 
 	@Override
@@ -124,13 +127,6 @@ public class JobDetailsHandler extends AbstractExecutionGraphRequestHandler {
 			ExecutionState jobVertexState = 
 					ExecutionJobVertex.getAggregateJobVertexState(tasksPerState, ejv.getParallelism());
 			jobVerticesPerState[jobVertexState.ordinal()]++;
-			
-			Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> metrics = ejv.getAggregatedMetricAccumulators();
-
-			LongCounter readBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_IN);
-			LongCounter writeBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_OUT);
-			LongCounter readRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_IN);
-			LongCounter writeRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_OUT);
 
 			gen.writeStartObject();
 			gen.writeStringField("id", ejv.getJobVertexId().toString());
@@ -148,11 +144,36 @@ public class JobDetailsHandler extends AbstractExecutionGraphRequestHandler {
 			}
 			gen.writeEndObject();
 			
+			long numBytesIn = 0;
+			long numBytesOut = 0;
+			long numRecordsIn = 0;
+			long numRecordsOut = 0;
+
+			for (AccessExecutionVertex vertex : ejv.getTaskVertices()) {
+				IOMetrics ioMetrics = vertex.getCurrentExecutionAttempt().getIOMetrics();
+
+				if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
+					numBytesIn += ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote();
+					numBytesOut += ioMetrics.getNumBytesOut();
+					numRecordsIn += ioMetrics.getNumRecordsIn();
+					numRecordsOut += ioMetrics.getNumRecordsOut();
+				} else { // execAttempt is still running, use MetricQueryService instead
+					fetcher.update();
+					MetricStore.SubtaskMetricStore metrics = fetcher.getMetricStore().getSubtaskMetricStore(graph.getJobID().toString(), ejv.getJobVertexId().toString(), vertex.getParallelSubtaskIndex());
+					if (metrics != null) {
+						numBytesIn += Long.valueOf(metrics.getMetric("numBytesInLocal", "0")) + Long.valueOf(metrics.getMetric("numBytesInRemote", "0"));
+						numBytesOut += Long.valueOf(metrics.getMetric("numBytesOut", "0"));
+						numRecordsIn += Long.valueOf(metrics.getMetric("numRecordsIn", "0"));
+						numRecordsOut += Long.valueOf(metrics.getMetric("numRecordsOut", "0"));
+					}
+				}
+			}
+
 			gen.writeObjectFieldStart("metrics");
-			gen.writeNumberField("read-bytes", readBytes != null ? readBytes.getLocalValuePrimitive() : -1L);
-			gen.writeNumberField("write-bytes", writeBytes != null ? writeBytes.getLocalValuePrimitive() : -1L);
-			gen.writeNumberField("read-records", readRecords != null ? readRecords.getLocalValuePrimitive() : -1L);
-			gen.writeNumberField("write-records",writeRecords != null ? writeRecords.getLocalValuePrimitive() : -1L);
+			gen.writeNumberField("read-bytes", numBytesIn);
+			gen.writeNumberField("write-bytes", numBytesOut);
+			gen.writeNumberField("read-records", numRecordsIn);
+			gen.writeNumberField("write-records", numRecordsOut);
 			gen.writeEndObject();
 			
 			gen.writeEndObject();

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexDetailsHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexDetailsHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexDetailsHandler.java
index fbdd86b..14dcd39 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexDetailsHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexDetailsHandler.java
@@ -20,14 +20,14 @@ package org.apache.flink.runtime.webmonitor.handlers;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.LongCounter;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
+import org.apache.flink.runtime.executiongraph.IOMetrics;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
+import org.apache.flink.runtime.webmonitor.metrics.MetricFetcher;
+import org.apache.flink.runtime.webmonitor.metrics.MetricStore;
 
 import java.io.StringWriter;
 import java.util.Map;
@@ -37,9 +37,12 @@ import java.util.Map;
  * and the runtime and metrics of all its subtasks.
  */
 public class JobVertexDetailsHandler extends AbstractJobVertexRequestHandler {
-	
-	public JobVertexDetailsHandler(ExecutionGraphHolder executionGraphHolder) {
+
+	private final MetricFetcher fetcher;
+
+	public JobVertexDetailsHandler(ExecutionGraphHolder executionGraphHolder, MetricFetcher fetcher) {
 		super(executionGraphHolder);
+		this.fetcher = fetcher;
 	}
 
 	@Override
@@ -71,25 +74,6 @@ public class JobVertexDetailsHandler extends AbstractJobVertexRequestHandler {
 			long endTime = status.isTerminal() ? vertex.getStateTimestamp(status) : -1;
 			long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1;
 			
-			Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> metrics = vertex.getCurrentExecutionAttempt().getFlinkAccumulators();
-			LongCounter readBytes;
-			LongCounter writeBytes;
-			LongCounter readRecords;
-			LongCounter writeRecords;
-			
-			if (metrics != null) {
-				readBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_IN);
-				writeBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_OUT);
-				readRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_IN);
-				writeRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_OUT);
-			}
-			else {
-				readBytes = null;
-				writeBytes = null;
-				readRecords = null;
-				writeRecords = null;
-			}
-			
 			gen.writeStartObject();
 			gen.writeNumberField("subtask", num);
 			gen.writeStringField("status", status.name());
@@ -99,11 +83,34 @@ public class JobVertexDetailsHandler extends AbstractJobVertexRequestHandler {
 			gen.writeNumberField("end-time", endTime);
 			gen.writeNumberField("duration", duration);
 
+			IOMetrics ioMetrics = vertex.getCurrentExecutionAttempt().getIOMetrics();
+
+			long numBytesIn = 0;
+			long numBytesOut = 0;
+			long numRecordsIn = 0;
+			long numRecordsOut = 0;
+
+			if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
+				numBytesIn = ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote();
+				numBytesOut = ioMetrics.getNumBytesOut();
+				numRecordsIn = ioMetrics.getNumRecordsIn();
+				numRecordsOut = ioMetrics.getNumRecordsOut();
+			} else { // execAttempt is still running, use MetricQueryService instead
+				fetcher.update();
+				MetricStore.SubtaskMetricStore metrics = fetcher.getMetricStore().getSubtaskMetricStore(params.get("jobid"), jobVertex.getJobVertexId().toString(), vertex.getParallelSubtaskIndex());
+				if (metrics != null) {
+					numBytesIn += Long.valueOf(metrics.getMetric("numBytesInLocal", "0")) + Long.valueOf(metrics.getMetric("numBytesInRemote", "0"));
+					numBytesOut += Long.valueOf(metrics.getMetric("numBytesOut", "0"));
+					numRecordsIn += Long.valueOf(metrics.getMetric("numRecordsIn", "0"));
+					numRecordsOut += Long.valueOf(metrics.getMetric("numRecordsOut", "0"));
+				}
+			}
+
 			gen.writeObjectFieldStart("metrics");
-			gen.writeNumberField("read-bytes", readBytes != null ? readBytes.getLocalValuePrimitive() : -1L);
-			gen.writeNumberField("write-bytes", writeBytes != null ? writeBytes.getLocalValuePrimitive() : -1L);
-			gen.writeNumberField("read-records", readRecords != null ? readRecords.getLocalValuePrimitive() : -1L);
-			gen.writeNumberField("write-records",writeRecords != null ? writeRecords.getLocalValuePrimitive() : -1L);
+			gen.writeNumberField("read-bytes", numBytesIn);
+			gen.writeNumberField("write-bytes", numBytesOut);
+			gen.writeNumberField("read-records", numRecordsIn);
+			gen.writeNumberField("write-records", numRecordsOut);
 			gen.writeEndObject();
 			
 			gen.writeEndObject();

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexTaskManagersHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexTaskManagersHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexTaskManagersHandler.java
index 0e94334..c1fabf8 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexTaskManagersHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/JobVertexTaskManagersHandler.java
@@ -19,15 +19,15 @@
 package org.apache.flink.runtime.webmonitor.handlers;
 
 import com.fasterxml.jackson.core.JsonGenerator;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.LongCounter;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecutionJobVertex;
 import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
 import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.IOMetrics;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
+import org.apache.flink.runtime.webmonitor.metrics.MetricFetcher;
+import org.apache.flink.runtime.webmonitor.metrics.MetricStore;
 
 import java.io.StringWriter;
 import java.util.ArrayList;
@@ -42,8 +42,11 @@ import java.util.Map.Entry;
  */
 public class JobVertexTaskManagersHandler extends AbstractJobVertexRequestHandler {
 
-	public JobVertexTaskManagersHandler(ExecutionGraphHolder executionGraphHolder) {
+	private final MetricFetcher fetcher;
+
+	public JobVertexTaskManagersHandler(ExecutionGraphHolder executionGraphHolder, MetricFetcher fetcher) {
 		super(executionGraphHolder);
+		this.fetcher = fetcher;
 	}
 
 	@Override
@@ -88,10 +91,10 @@ public class JobVertexTaskManagersHandler extends AbstractJobVertexRequestHandle
 			long endTime = 0;
 			boolean allFinished = true;
 
-			LongCounter tmReadBytes = new LongCounter();
-			LongCounter tmWriteBytes = new LongCounter();
-			LongCounter tmReadRecords = new LongCounter();
-			LongCounter tmWriteRecords = new LongCounter();
+			long numBytesIn = 0;
+			long numBytesOut = 0;
+			long numRecordsIn = 0;
+			long numRecordsOut = 0;
 
 			for (AccessExecutionVertex vertex : taskVertices) {
 				final ExecutionState state = vertex.getExecutionState();
@@ -106,20 +109,22 @@ public class JobVertexTaskManagersHandler extends AbstractJobVertexRequestHandle
 				allFinished &= state.isTerminal();
 				endTime = Math.max(endTime, vertex.getStateTimestamp(state));
 
-				Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> metrics = vertex.getCurrentExecutionAttempt().getFlinkAccumulators();
-
-				if (metrics != null) {
-					LongCounter readBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_IN);
-					tmReadBytes.merge(readBytes);
-
-					LongCounter writeBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_OUT);
-					tmWriteBytes.merge(writeBytes);
-
-					LongCounter readRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_IN);
-					tmReadRecords.merge(readRecords);
-
-					LongCounter writeRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_OUT);
-					tmWriteRecords.merge(writeRecords);
+				IOMetrics ioMetrics = vertex.getCurrentExecutionAttempt().getIOMetrics();
+
+				if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
+					numBytesIn += ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote();
+					numBytesOut += ioMetrics.getNumBytesOut();
+					numRecordsIn += ioMetrics.getNumRecordsIn();
+					numRecordsOut += ioMetrics.getNumRecordsOut();
+				} else { // execAttempt is still running, use MetricQueryService instead
+					fetcher.update();
+					MetricStore.SubtaskMetricStore metrics = fetcher.getMetricStore().getSubtaskMetricStore(params.get("jobid"), params.get("vertexid"), vertex.getParallelSubtaskIndex());
+					if (metrics != null) {
+						numBytesIn += Long.valueOf(metrics.getMetric("numBytesInLocal", "0")) + Long.valueOf(metrics.getMetric("numBytesInRemote", "0"));
+						numBytesOut += Long.valueOf(metrics.getMetric("numBytesOut", "0"));
+						numRecordsIn += Long.valueOf(metrics.getMetric("numRecordsIn", "0"));
+						numRecordsOut += Long.valueOf(metrics.getMetric("numRecordsOut", "0"));
+					}
 				}
 			}
 
@@ -152,10 +157,10 @@ public class JobVertexTaskManagersHandler extends AbstractJobVertexRequestHandle
 			gen.writeNumberField("duration", duration);
 
 			gen.writeObjectFieldStart("metrics");
-			gen.writeNumberField("read-bytes", tmReadBytes.getLocalValuePrimitive());
-			gen.writeNumberField("write-bytes", tmWriteBytes.getLocalValuePrimitive());
-			gen.writeNumberField("read-records", tmReadRecords.getLocalValuePrimitive());
-			gen.writeNumberField("write-records", tmWriteRecords.getLocalValuePrimitive());
+			gen.writeNumberField("read-bytes", numBytesIn);
+			gen.writeNumberField("write-bytes", numBytesOut);
+			gen.writeNumberField("read-records", numRecordsIn);
+			gen.writeNumberField("write-records", numRecordsOut);
 			gen.writeEndObject();
 
 			gen.writeObjectFieldStart("status-counts");

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskCurrentAttemptDetailsHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskCurrentAttemptDetailsHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskCurrentAttemptDetailsHandler.java
index 811bea6..6d09513 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskCurrentAttemptDetailsHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskCurrentAttemptDetailsHandler.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.webmonitor.handlers;
 
 import org.apache.flink.runtime.executiongraph.AccessExecutionVertex;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
+import org.apache.flink.runtime.webmonitor.metrics.MetricFetcher;
 
 import java.util.Map;
 
@@ -28,8 +29,8 @@ import java.util.Map;
  */
 public class SubtaskCurrentAttemptDetailsHandler extends SubtaskExecutionAttemptDetailsHandler {
 	
-	public SubtaskCurrentAttemptDetailsHandler(ExecutionGraphHolder executionGraphHolder) {
-		super(executionGraphHolder);
+	public SubtaskCurrentAttemptDetailsHandler(ExecutionGraphHolder executionGraphHolder, MetricFetcher fetcher) {
+		super(executionGraphHolder, fetcher);
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskExecutionAttemptDetailsHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskExecutionAttemptDetailsHandler.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskExecutionAttemptDetailsHandler.java
index 3cc7376..ca9c7ad 100644
--- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskExecutionAttemptDetailsHandler.java
+++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/handlers/SubtaskExecutionAttemptDetailsHandler.java
@@ -20,13 +20,13 @@ package org.apache.flink.runtime.webmonitor.handlers;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.LongCounter;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.executiongraph.AccessExecution;
+import org.apache.flink.runtime.executiongraph.IOMetrics;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.runtime.webmonitor.ExecutionGraphHolder;
+import org.apache.flink.runtime.webmonitor.metrics.MetricFetcher;
+import org.apache.flink.runtime.webmonitor.metrics.MetricStore;
 
 import java.io.StringWriter;
 import java.util.Map;
@@ -35,9 +35,12 @@ import java.util.Map;
  * Request handler providing details about a single task execution attempt.
  */
 public class SubtaskExecutionAttemptDetailsHandler extends AbstractSubtaskAttemptRequestHandler {
-	
-	public SubtaskExecutionAttemptDetailsHandler(ExecutionGraphHolder executionGraphHolder) {
+
+	private final MetricFetcher fetcher;
+
+	public SubtaskExecutionAttemptDetailsHandler(ExecutionGraphHolder executionGraphHolder, MetricFetcher fetcher) {
 		super(executionGraphHolder);
+		this.fetcher = fetcher;
 	}
 
 	@Override
@@ -55,25 +58,6 @@ public class SubtaskExecutionAttemptDetailsHandler extends AbstractSubtaskAttemp
 		long endTime = status.isTerminal() ? execAttempt.getStateTimestamp(status) : -1;
 		long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1;
 
-		Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> metrics = execAttempt.getFlinkAccumulators();
-		LongCounter readBytes;
-		LongCounter writeBytes;
-		LongCounter readRecords;
-		LongCounter writeRecords;
-
-		if (metrics != null) {
-			readBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_IN);
-			writeBytes = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_OUT);
-			readRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_IN);
-			writeRecords = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_OUT);
-		}
-		else {
-			readBytes = null;
-			writeBytes = null;
-			readRecords = null;
-			writeRecords = null;
-		}
-
 		StringWriter writer = new StringWriter();
 		JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer);
 
@@ -86,11 +70,34 @@ public class SubtaskExecutionAttemptDetailsHandler extends AbstractSubtaskAttemp
 		gen.writeNumberField("end-time", endTime);
 		gen.writeNumberField("duration", duration);
 
+		IOMetrics ioMetrics = execAttempt.getIOMetrics();
+
+		long numBytesIn = 0;
+		long numBytesOut = 0;
+		long numRecordsIn = 0;
+		long numRecordsOut = 0;
+		
+		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
+			numBytesIn = ioMetrics.getNumBytesInLocal() + ioMetrics.getNumBytesInRemote();
+			numBytesOut = ioMetrics.getNumBytesOut();
+			numRecordsIn = ioMetrics.getNumRecordsIn();
+			numRecordsOut = ioMetrics.getNumRecordsOut();
+		} else { // execAttempt is still running, use MetricQueryService instead
+			fetcher.update();
+			MetricStore.SubtaskMetricStore metrics = fetcher.getMetricStore().getSubtaskMetricStore(params.get("jobid"), params.get("vertexid"), execAttempt.getParallelSubtaskIndex());
+			if (metrics != null) {
+				numBytesIn = Long.valueOf(metrics.getMetric("numBytesInLocal", "0")) + Long.valueOf(metrics.getMetric("numBytesInRemote", "0"));
+				numBytesOut = Long.valueOf(metrics.getMetric("numBytesOut", "0"));
+				numRecordsIn = Long.valueOf(metrics.getMetric("numRecordsIn", "0"));
+				numRecordsOut = Long.valueOf(metrics.getMetric("numRecordsOut", "0"));
+			}
+		}
+		
 		gen.writeObjectFieldStart("metrics");
-		gen.writeNumberField("read-bytes", readBytes != null ? readBytes.getLocalValuePrimitive() : -1L);
-		gen.writeNumberField("write-bytes", writeBytes != null ? writeBytes.getLocalValuePrimitive() : -1L);
-		gen.writeNumberField("read-records", readRecords != null ? readRecords.getLocalValuePrimitive() : -1L);
-		gen.writeNumberField("write-records",writeRecords != null ? writeRecords.getLocalValuePrimitive() : -1L);
+		gen.writeNumberField("read-bytes", numBytesIn);
+		gen.writeNumberField("write-bytes", numBytesOut);
+		gen.writeNumberField("read-records", numRecordsIn);
+		gen.writeNumberField("write-records", numRecordsOut);
 		gen.writeEndObject();
 
 		gen.writeEndObject();

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorRegistry.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorRegistry.java
index 44714e7..ce6cb1b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorRegistry.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorRegistry.java
@@ -20,18 +20,16 @@ package org.apache.flink.runtime.accumulators;
 
 import org.apache.flink.api.common.JobID;
 import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.LongCounter;
 import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 
 /**
- * Main accumulator registry which encapsulates internal and user-defined accumulators.
+ * Main accumulator registry which encapsulates user-defined accumulators.
  */
 public class AccumulatorRegistry {
 
@@ -40,32 +38,13 @@ public class AccumulatorRegistry {
 	protected final JobID jobID;
 	protected final ExecutionAttemptID taskID;
 
-	/* Flink's internal Accumulator values stored for the executing task. */
-	private final Map<Metric, Accumulator<?, ?>> flinkAccumulators =
-			new HashMap<Metric, Accumulator<?, ?>>();
-
 	/* User-defined Accumulator values stored for the executing task. */
 	private final Map<String, Accumulator<?, ?>> userAccumulators =
 			new ConcurrentHashMap<>(4);
 
-	/* The reporter reference that is handed to the reporting tasks. */
-	private final ReadWriteReporter reporter;
-
-	/**
-	 * Flink metrics supported
-	 */
-	public enum Metric {
-		NUM_RECORDS_IN,
-		NUM_RECORDS_OUT,
-		NUM_BYTES_IN,
-		NUM_BYTES_OUT
-	}
-
-
 	public AccumulatorRegistry(JobID jobID, ExecutionAttemptID taskID) {
 		this.jobID = jobID;
 		this.taskID = taskID;
-		this.reporter = new ReadWriteReporter(flinkAccumulators);
 	}
 
 	/**
@@ -74,7 +53,7 @@ public class AccumulatorRegistry {
 	 */
 	public AccumulatorSnapshot getSnapshot() {
 		try {
-			return new AccumulatorSnapshot(jobID, taskID, flinkAccumulators, userAccumulators);
+			return new AccumulatorSnapshot(jobID, taskID, userAccumulators);
 		} catch (Throwable e) {
 			LOG.warn("Failed to serialize accumulators for task.", e);
 			return null;
@@ -88,59 +67,4 @@ public class AccumulatorRegistry {
 		return userAccumulators;
 	}
 
-	/**
-	 * Gets the reporter for flink internal metrics.
-	 */
-	public Reporter getReadWriteReporter() {
-		return reporter;
-	}
-
-	/**
-	 * Interface for Flink's internal accumulators.
-	 */
-	public interface Reporter {
-		void reportNumRecordsIn(long value);
-		void reportNumRecordsOut(long value);
-		void reportNumBytesIn(long value);
-		void reportNumBytesOut(long value);
-	}
-
-	/**
-	 * Accumulator based reporter for keeping track of internal metrics (e.g. bytes and records in/out)
-	 */
-	private static class ReadWriteReporter implements Reporter {
-
-		private LongCounter numRecordsIn = new LongCounter();
-		private LongCounter numRecordsOut = new LongCounter();
-		private LongCounter numBytesIn = new LongCounter();
-		private LongCounter numBytesOut = new LongCounter();
-
-		private ReadWriteReporter(Map<Metric, Accumulator<?,?>> accumulatorMap) {
-			accumulatorMap.put(Metric.NUM_RECORDS_IN, numRecordsIn);
-			accumulatorMap.put(Metric.NUM_RECORDS_OUT, numRecordsOut);
-			accumulatorMap.put(Metric.NUM_BYTES_IN, numBytesIn);
-			accumulatorMap.put(Metric.NUM_BYTES_OUT, numBytesOut);
-		}
-
-		@Override
-		public void reportNumRecordsIn(long value) {
-			numRecordsIn.add(value);
-		}
-
-		@Override
-		public void reportNumRecordsOut(long value) {
-			numRecordsOut.add(value);
-		}
-
-		@Override
-		public void reportNumBytesIn(long value) {
-			numBytesIn.add(value);
-		}
-
-		@Override
-		public void reportNumBytesOut(long value) {
-			numBytesOut.add(value);
-		}
-	}
-
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
index d0f4bad..0bfb1ac 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
@@ -40,21 +40,14 @@ public class AccumulatorSnapshot implements Serializable {
 	private final ExecutionAttemptID executionAttemptID;
 
 	/**
-	 * Flink internal accumulators which can be deserialized using the system class loader.
-	 */
-	private final SerializedValue<Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>> flinkAccumulators;
-
-	/**
 	 * Serialized user accumulators which may require the custom user class loader.
 	 */
 	private final SerializedValue<Map<String, Accumulator<?, ?>>> userAccumulators;
 
 	public AccumulatorSnapshot(JobID jobID, ExecutionAttemptID executionAttemptID,
-							Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators,
 							Map<String, Accumulator<?, ?>> userAccumulators) throws IOException {
 		this.jobID = jobID;
 		this.executionAttemptID = executionAttemptID;
-		this.flinkAccumulators = new SerializedValue<Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>>(flinkAccumulators);
 		this.userAccumulators = new SerializedValue<Map<String, Accumulator<?, ?>>>(userAccumulators);
 	}
 
@@ -67,14 +60,6 @@ public class AccumulatorSnapshot implements Serializable {
 	}
 
 	/**
-	 * Gets the Flink (internal) accumulators values.
-	 * @return the serialized map
-	 */
-	public Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> deserializeFlinkAccumulators() throws IOException, ClassNotFoundException {
-		return flinkAccumulators.deserializeValue(getClass().getClassLoader());
-	}
-
-	/**
 	 * Gets the user-defined accumulators values.
 	 * @return the serialized map
 	 */

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecution.java
index aefc17d..df558c5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecution.java
@@ -17,14 +17,10 @@
  */
 package org.apache.flink.runtime.executiongraph;
 
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 
-import java.util.Map;
-
 /**
  * Common interface for the runtime {@link Execution and {@link ArchivedExecution}.
  */
@@ -88,18 +84,11 @@ public interface AccessExecution {
 	StringifiedAccumulatorResult[] getUserAccumulatorsStringified();
 
 	/**
-	 * Returns the system-defined accumulators.
-	 *
-	 * @return system-defined accumulators.
-	 * @deprecated Will be removed in FLINK-4527
-	 */
-	@Deprecated
-	Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> getFlinkAccumulators();
-
-	/**
 	 * Returns the subtask index of this execution.
 	 *
 	 * @return subtask index of this execution.
 	 */
 	int getParallelSubtaskIndex();
+
+	IOMetrics getIOMetrics();
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
index 0fd97da..e7fe1b0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
@@ -18,8 +18,6 @@
 package org.apache.flink.runtime.executiongraph;
 
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.checkpoint.CheckpointCoordinator;
 import org.apache.flink.runtime.checkpoint.stats.CheckpointStatsTracker;
@@ -153,15 +151,6 @@ public interface AccessExecutionGraph {
 	Map<String, SerializedValue<Object>> getAccumulatorsSerialized() throws IOException;
 
 	/**
-	 * Returns the aggregated system-defined accumulators.
-	 *
-	 * @return aggregated system-defined accumulators.
-	 * @deprecated Will be removed in FLINK-4527
-	 */
-	@Deprecated
-	Map<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>> getFlinkAccumulators();
-
-	/**
 	 * Returns whether this execution graph was archived.
 	 *
 	 * @return true, if the execution graph was archived, false otherwise

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java
index c9bf604..92af0c8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionJobVertex.java
@@ -17,16 +17,12 @@
  */
 package org.apache.flink.runtime.executiongraph;
 
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.checkpoint.stats.OperatorCheckpointStats;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.jobgraph.JobVertexID;
 import scala.Option;
 
-import java.util.Map;
-
 /**
  * Common interface for the runtime {@link ExecutionJobVertex} and {@link ArchivedExecutionJobVertex}.
  */
@@ -81,15 +77,6 @@ public interface AccessExecutionJobVertex {
 	Option<OperatorCheckpointStats> getCheckpointStats();
 
 	/**
-	 * Returns the aggregated system-defined accumulators.
-	 *
-	 * @return aggregated system-defined accumulators.
-	 * @deprecated Will be removed in FLINK-4527
-	 */
-	@Deprecated
-	Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> getAggregatedMetricAccumulators();
-
-	/**
 	 * Returns the aggregated user-defined accumulators as strings.
 	 *
 	 * @return aggregated user-defined accumulators as strings.

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecution.java
index 0b2992f..c189d42 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecution.java
@@ -17,15 +17,12 @@
  */
 package org.apache.flink.runtime.executiongraph;
 
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.execution.ExecutionState;
 import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
 import org.apache.flink.util.ExceptionUtils;
 
 import java.io.Serializable;
-import java.util.Map;
 
 public class ArchivedExecution implements AccessExecution, Serializable {
 	private static final long serialVersionUID = 4817108757483345173L;
@@ -46,13 +43,12 @@ public class ArchivedExecution implements AccessExecution, Serializable {
 	/* Continuously updated map of user-defined accumulators */
 	private final StringifiedAccumulatorResult[] userAccumulators;
 
-	/* Continuously updated map of internal accumulators */
-	private final Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators;
 	private final int parallelSubtaskIndex;
 
+	private final IOMetrics ioMetrics;
+
 	public ArchivedExecution(Execution execution) {
 		this.userAccumulators = execution.getUserAccumulatorsStringified();
-		this.flinkAccumulators = execution.getFlinkAccumulators();
 		this.attemptId = execution.getAttemptId();
 		this.attemptNumber = execution.getAttemptNumber();
 		this.stateTimestamps = execution.getStateTimestamps();
@@ -60,6 +56,7 @@ public class ArchivedExecution implements AccessExecution, Serializable {
 		this.state = execution.getState();
 		this.failureCause = ExceptionUtils.stringifyException(execution.getFailureCause());
 		this.assignedResourceLocation = execution.getAssignedResourceLocation();
+		this.ioMetrics = execution.getIOMetrics();
 	}
 
 	// --------------------------------------------------------------------------------------------
@@ -106,13 +103,14 @@ public class ArchivedExecution implements AccessExecution, Serializable {
 		return userAccumulators;
 	}
 
-	@Override
-	public Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> getFlinkAccumulators() {
-		return flinkAccumulators;
-	}
 
 	@Override
 	public int getParallelSubtaskIndex() {
 		return parallelSubtaskIndex;
 	}
+
+	@Override
+	public IOMetrics getIOMetrics() {
+		return ioMetrics;
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
index d8c58c8..0bd5319 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
@@ -19,8 +19,6 @@ package org.apache.flink.runtime.executiongraph;
 
 import org.apache.flink.api.common.ArchivedExecutionConfig;
 import org.apache.flink.api.common.JobID;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.checkpoint.ArchivedCheckpointStatsTracker;
 import org.apache.flink.runtime.checkpoint.CheckpointCoordinator;
@@ -31,7 +29,6 @@ import org.apache.flink.util.SerializedValue;
 
 import java.io.Serializable;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -208,24 +205,6 @@ public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializabl
 		return tracker;
 	}
 
-	/**
-	 * Gets the internal flink accumulator map of maps which contains some metrics.
-	 *
-	 * @return A map of accumulators for every executed task.
-	 */
-	@Override
-	public Map<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>> getFlinkAccumulators() {
-		Map<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>> flinkAccumulators =
-			new HashMap<>();
-
-		for (AccessExecutionVertex vertex : getAllExecutionVertices()) {
-			Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> taskAccs = vertex.getCurrentExecutionAttempt().getFlinkAccumulators();
-			flinkAccumulators.put(vertex.getCurrentExecutionAttempt().getAttemptId(), taskAccs);
-		}
-
-		return flinkAccumulators;
-	}
-
 	@Override
 	public boolean isArchived() {
 		return true;

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java
index 4857bf5..8ae6bbd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionJobVertex.java
@@ -19,7 +19,6 @@ package org.apache.flink.runtime.executiongraph;
 
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.accumulators.AccumulatorHelper;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.checkpoint.stats.CheckpointStatsTracker;
 import org.apache.flink.runtime.checkpoint.stats.OperatorCheckpointStats;
@@ -46,7 +45,6 @@ public class ArchivedExecutionJobVertex implements AccessExecutionJobVertex, Ser
 
 	private final int maxParallelism;
 
-	private final Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> aggregatedMetricAccumulators;
 	private final Option<OperatorCheckpointStats> checkpointStats;
 	private final StringifiedAccumulatorResult[] archivedUserAccumulators;
 
@@ -56,8 +54,6 @@ public class ArchivedExecutionJobVertex implements AccessExecutionJobVertex, Ser
 			taskVertices[x] = jobVertex.getTaskVertices()[x].archive();
 		}
 
-		aggregatedMetricAccumulators = jobVertex.getAggregatedMetricAccumulators();
-
 		Map<String, Accumulator<?, ?>> tmpArchivedUserAccumulators = new HashMap<>();
 		for (ExecutionVertex vertex : jobVertex.getTaskVertices()) {
 			Map<String, Accumulator<?, ?>> next = vertex.getCurrentExecutionAttempt().getUserAccumulators();
@@ -116,10 +112,6 @@ public class ArchivedExecutionJobVertex implements AccessExecutionJobVertex, Ser
 		return getAggregateJobVertexState(num, parallelism);
 	}
 
-	public Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> getAggregatedMetricAccumulators() {
-		return this.aggregatedMetricAccumulators;
-	}
-
 	// --------------------------------------------------------------------------------------------
 	//  Static / pre-assigned input splits
 	// --------------------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 17e0df1..788dee4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -23,7 +23,6 @@ import akka.dispatch.OnFailure;
 import org.apache.flink.api.common.Archiveable;
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.runtime.JobException;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.clusterframework.types.ResourceID;
 import org.apache.flink.runtime.concurrent.BiFunction;
@@ -143,9 +142,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 
 	/* Continuously updated map of user-defined accumulators */
 	private volatile Map<String, Accumulator<?, ?>> userAccumulators;
-
-	/* Continuously updated map of internal accumulators */
-	private volatile Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators;
+	private IOMetrics ioMetrics;
 
 	// --------------------------------------------------------------------------------------------
 	
@@ -651,7 +648,7 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 		markFinished(null, null);
 	}
 
-	void markFinished(Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators, Map<String, Accumulator<?, ?>> userAccumulators) {
+	void markFinished(Map<String, Accumulator<?, ?>> userAccumulators, IOMetrics metrics) {
 
 		// this call usually comes during RUNNING, but may also come while still in deploying (very fast tasks!)
 		while (true) {
@@ -673,9 +670,9 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 						}
 
 						synchronized (accumulatorLock) {
-							this.flinkAccumulators = flinkAccumulators;
 							this.userAccumulators = userAccumulators;
 						}
+						this.ioMetrics = metrics;
 
 						assignedResource.releaseSlot();
 						vertex.getExecutionGraph().deregisterExecution(this);
@@ -1010,14 +1007,11 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	
 	/**
 	 * Update accumulators (discarded when the Execution has already been terminated).
-	 * @param flinkAccumulators the flink internal accumulators
 	 * @param userAccumulators the user accumulators
 	 */
-	public void setAccumulators(Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators,
-								Map<String, Accumulator<?, ?>> userAccumulators) {
+	public void setAccumulators(Map<String, Accumulator<?, ?>> userAccumulators) {
 		synchronized (accumulatorLock) {
 			if (!state.isTerminal()) {
-				this.flinkAccumulators = flinkAccumulators;
 				this.userAccumulators = userAccumulators;
 			}
 		}
@@ -1033,14 +1027,14 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
 	}
 
 	@Override
-	public Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> getFlinkAccumulators() {
-		return flinkAccumulators;
-	}
-
-	@Override
 	public int getParallelSubtaskIndex() {
 		return getVertex().getParallelSubtaskIndex();
 	}
+		
+	@Override
+	public IOMetrics getIOMetrics() {
+		return ioMetrics;
+	}
 
 	// ------------------------------------------------------------------------
 	//  Standard utilities

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 0a79cf2..074a04d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -29,7 +29,6 @@ import org.apache.flink.metrics.MetricGroup;
 import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
 import org.apache.flink.runtime.JobException;
 import org.apache.flink.runtime.StoppingException;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.AccumulatorSnapshot;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.blob.BlobKey;
@@ -600,23 +599,6 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	}
 
 	/**
-	 * Gets the internal flink accumulator map of maps which contains some metrics.
-	 * @return A map of accumulators for every executed task.
-	 */
-	@Override
-	public Map<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?,?>>> getFlinkAccumulators() {
-		Map<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>> flinkAccumulators =
-				new HashMap<ExecutionAttemptID, Map<AccumulatorRegistry.Metric, Accumulator<?, ?>>>();
-
-		for (ExecutionVertex vertex : getAllExecutionVertices()) {
-			Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> taskAccs = vertex.getCurrentExecutionAttempt().getFlinkAccumulators();
-			flinkAccumulators.put(vertex.getCurrentExecutionAttempt().getAttemptId(), taskAccs);
-		}
-
-		return flinkAccumulators;
-	}
-
-	/**
 	 * Merges all accumulator results from the tasks previously executed in the Executions.
 	 * @return The accumulator map
 	 */
@@ -1075,7 +1057,7 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 
 	/**
 	 * Updates the state of one of the ExecutionVertex's Execution attempts.
-	 * If the new status if "FINISHED", this also updates the
+	 * If the new status if "FINISHED", this also updates the accumulators.
 	 * 
 	 * @param state The state update.
 	 * @return True, if the task update was properly applied, false, if the execution attempt was not found.
@@ -1090,11 +1072,9 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 				case FINISHED:
 					try {
 						AccumulatorSnapshot accumulators = state.getAccumulators();
-						Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators =
-							accumulators.deserializeFlinkAccumulators();
 						Map<String, Accumulator<?, ?>> userAccumulators =
 							accumulators.deserializeUserAccumulators(userClassLoader);
-						attempt.markFinished(flinkAccumulators, userAccumulators);
+						attempt.markFinished(userAccumulators, state.getIOMetrics());
 					}
 					catch (Exception e) {
 						LOG.error("Failed to deserialize final accumulator results.", e);
@@ -1160,16 +1140,14 @@ public class ExecutionGraph implements AccessExecutionGraph, Archiveable<Archive
 	 * @param accumulatorSnapshot The serialized flink and user-defined accumulators
 	 */
 	public void updateAccumulators(AccumulatorSnapshot accumulatorSnapshot) {
-		Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> flinkAccumulators;
 		Map<String, Accumulator<?, ?>> userAccumulators;
 		try {
-			flinkAccumulators = accumulatorSnapshot.deserializeFlinkAccumulators();
 			userAccumulators = accumulatorSnapshot.deserializeUserAccumulators(userClassLoader);
 
 			ExecutionAttemptID execID = accumulatorSnapshot.getExecutionAttemptID();
 			Execution execution = currentExecutions.get(execID);
 			if (execution != null) {
-				execution.setAccumulators(flinkAccumulators, userAccumulators);
+				execution.setAccumulators(userAccumulators);
 			} else {
 				LOG.warn("Received accumulator result for unknown execution {}.", execID);
 			}

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index e7f16a2..2d9ec88 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -20,13 +20,11 @@ package org.apache.flink.runtime.executiongraph;
 
 import org.apache.flink.api.common.accumulators.Accumulator;
 import org.apache.flink.api.common.accumulators.AccumulatorHelper;
-import org.apache.flink.api.common.accumulators.LongCounter;
 import org.apache.flink.core.io.InputSplit;
 import org.apache.flink.core.io.InputSplitAssigner;
 import org.apache.flink.core.io.InputSplitSource;
 import org.apache.flink.core.io.LocatableInputSplit;
 import org.apache.flink.runtime.JobException;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
 import org.apache.flink.runtime.checkpoint.stats.CheckpointStatsTracker;
 import org.apache.flink.runtime.checkpoint.stats.OperatorCheckpointStats;
@@ -443,37 +441,6 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
 	// --------------------------------------------------------------------------------------------
 	//  Accumulators / Metrics
 	// --------------------------------------------------------------------------------------------
-	
-	public Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> getAggregatedMetricAccumulators() {
-		// some specialized code to speed things up
-		long bytesRead = 0;
-		long bytesWritten = 0;
-		long recordsRead = 0;
-		long recordsWritten = 0;
-		
-		for (ExecutionVertex v : getTaskVertices()) {
-			Map<AccumulatorRegistry.Metric, Accumulator<?, ?>> metrics = v.getCurrentExecutionAttempt().getFlinkAccumulators();
-			
-			if (metrics != null) {
-				LongCounter br = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_IN);
-				LongCounter bw = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_BYTES_OUT);
-				LongCounter rr = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_IN);
-				LongCounter rw = (LongCounter) metrics.get(AccumulatorRegistry.Metric.NUM_RECORDS_OUT);
-				
-				bytesRead += br != null ? br.getLocalValuePrimitive() : 0;
-				bytesWritten += bw != null ? bw.getLocalValuePrimitive() : 0;
-				recordsRead += rr != null ? rr.getLocalValuePrimitive() : 0;
-				recordsWritten += rw != null ? rw.getLocalValuePrimitive() : 0;
-			}
-		}
-
-		HashMap<AccumulatorRegistry.Metric, Accumulator<?, ?>> agg = new HashMap<>();
-		agg.put(AccumulatorRegistry.Metric.NUM_BYTES_IN, new LongCounter(bytesRead));
-		agg.put(AccumulatorRegistry.Metric.NUM_BYTES_OUT, new LongCounter(bytesWritten));
-		agg.put(AccumulatorRegistry.Metric.NUM_RECORDS_IN, new LongCounter(recordsRead));
-		agg.put(AccumulatorRegistry.Metric.NUM_RECORDS_OUT, new LongCounter(recordsWritten));
-		return agg;
-	}
 
 	public StringifiedAccumulatorResult[] getAggregatedUserAccumulatorsStringified() {
 		Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<String, Accumulator<?, ?>>();

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
index e0fe355..c5aeef7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/AbstractRecordReader.java
@@ -19,7 +19,6 @@
 package org.apache.flink.runtime.io.network.api.reader;
 
 import org.apache.flink.core.io.IOReadableWritable;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer;
 import org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer.DeserializationResult;
 import org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer;
@@ -123,11 +122,4 @@ abstract class AbstractRecordReader<T extends IOReadableWritable> extends Abstra
 			}
 		}
 	}
-
-	@Override
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-		for (RecordDeserializer<?> deserializer : recordDeserializers) {
-			deserializer.setReporter(reporter);
-		}
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
index debb352..ca59609 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/BufferReader.java
@@ -18,7 +18,6 @@
 
 package org.apache.flink.runtime.io.network.api.reader;
 
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
 import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
@@ -48,9 +47,4 @@ public final class BufferReader extends AbstractReader {
 			}
 		}
 	}
-
-	@Override
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-
-	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
index a1d705f..0cc77f0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/reader/ReaderBase.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.io.network.api.reader;
 
 import java.io.IOException;
 
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.event.TaskEvent;
 import org.apache.flink.runtime.util.event.EventListener;
 
@@ -52,9 +51,4 @@ public interface ReaderBase {
 
 	boolean hasReachedEndOfSuperstep();
 
-	/**
-	 * Setter for the reporter, e.g. for the number of records emitted and the number of bytes read.
-	 */
-	void setReporter(AccumulatorRegistry.Reporter reporter);
-
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
index cdd8731..8f2c8fd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/AdaptiveSpanningRecordDeserializer.java
@@ -21,7 +21,6 @@ package org.apache.flink.runtime.io.network.api.serialization;
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.util.DataInputDeserializer;
 import org.apache.flink.runtime.util.DataOutputSerializer;
@@ -46,8 +45,6 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
 
 	private Buffer currentBuffer;
 
-	private AccumulatorRegistry.Reporter reporter;
-
 	public AdaptiveSpanningRecordDeserializer() {
 		this.nonSpanningWrapper = new NonSpanningWrapper();
 		this.spanningWrapper = new SpanningWrapper();
@@ -93,18 +90,10 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
 		if (nonSpanningRemaining >= 4) {
 			int len = this.nonSpanningWrapper.readInt();
 
-			if (reporter != null) {
-				reporter.reportNumBytesIn(len);
-			}
-
 			if (len <= nonSpanningRemaining - 4) {
 				// we can get a full record from here
 				target.read(this.nonSpanningWrapper);
 
-				if (reporter != null) {
-					reporter.reportNumRecordsIn(1);
-				}
-
 				return (this.nonSpanningWrapper.remaining() == 0) ?
 						DeserializationResult.LAST_RECORD_FROM_BUFFER :
 						DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER;
@@ -128,10 +117,6 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
 			// get the full record
 			target.read(this.spanningWrapper);
 
-			if (reporter != null) {
-				reporter.reportNumRecordsIn(1);
-			}
-
 			// move the remainder to the non-spanning wrapper
 			// this does not copy it, only sets the memory segment
 			this.spanningWrapper.moveRemainderToNonSpanningDeserializer(this.nonSpanningWrapper);
@@ -159,12 +144,6 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
 		return this.nonSpanningWrapper.remaining() > 0 || this.spanningWrapper.getNumGatheredBytes() > 0;
 	}
 
-	@Override
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-		this.reporter = reporter;
-		this.spanningWrapper.setReporter(reporter);
-	}
-
 	// -----------------------------------------------------------------------------------------------------------------
 
 	private static final class NonSpanningWrapper implements DataInputView {
@@ -447,8 +426,6 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
 
 		private int recordLimit;
 
-		private AccumulatorRegistry.Reporter reporter;
-
 		public SpanningWrapper() {
 			this.lengthBuffer = ByteBuffer.allocate(4);
 			this.lengthBuffer.order(ByteOrder.BIG_ENDIAN);
@@ -486,10 +463,6 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
 				} else {
 					this.recordLength = this.lengthBuffer.getInt(0);
 
-					if (reporter != null) {
-						reporter.reportNumBytesIn(this.recordLength);
-					}
-
 					this.lengthBuffer.clear();
 					segmentPosition = toPut;
 				}
@@ -634,9 +607,5 @@ public class AdaptiveSpanningRecordDeserializer<T extends IOReadableWritable> im
 		public int read(byte[] b) throws IOException {
 			return this.serializationReadBuffer.read(b);
 		}
-
-		public void setReporter(AccumulatorRegistry.Reporter reporter) {
-			this.reporter = reporter;
-		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
index e4c7890..dd8ea06 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordDeserializer.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 
 /**
@@ -65,9 +64,4 @@ public interface RecordDeserializer<T extends IOReadableWritable> {
 	void clear();
 	
 	boolean hasUnfinishedData();
-
-	/**
-	 * Setter for the reporter, e.g. for the number of records emitted and the number of bytes read.
-	 */
-	void setReporter(AccumulatorRegistry.Reporter reporter);
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
index c76dd00..e8179dc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/RecordSerializer.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 
 /**
@@ -67,11 +66,6 @@ public interface RecordSerializer<T extends IOReadableWritable> {
 	boolean hasData();
 
 	/**
-	 * Setter for the reporter, e.g. for the number of records emitted and the number of bytes read.
-	 */
-	void setReporter(AccumulatorRegistry.Reporter reporter);
-
-	/**
 	 * Insantiates all metrics.
 	 *
 	 * @param metrics metric group

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
index 7c4d937..e36a16f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpanningRecordSerializer.java
@@ -26,7 +26,6 @@ import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.core.memory.MemorySegment;
 import org.apache.flink.metrics.Counter;
 import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.util.DataOutputSerializer;
 
@@ -53,8 +52,6 @@ public class SpanningRecordSerializer<T extends IOReadableWritable> implements R
 	/** Limit of current {@link MemorySegment} of target buffer */
 	private int limit;
 
-	private AccumulatorRegistry.Reporter reporter;
-
 	private transient Counter numBytesOut;
 
 	public SpanningRecordSerializer() {
@@ -84,11 +81,6 @@ public class SpanningRecordSerializer<T extends IOReadableWritable> implements R
 
 		int len = this.serializationBuffer.length();
 		this.lengthBuffer.putInt(0, len);
-
-		if (reporter != null) {
-			reporter.reportNumBytesOut(len);
-			reporter.reportNumRecordsOut(1);
-		}
 		
 		if (numBytesOut != null) {
 			numBytesOut.inc(len);
@@ -192,11 +184,6 @@ public class SpanningRecordSerializer<T extends IOReadableWritable> implements R
 	}
 
 	@Override
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-		this.reporter = reporter;
-	}
-
-	@Override
 	public void instantiateMetrics(TaskIOMetricGroup metrics) {
 		numBytesOut = metrics.getNumBytesOutCounter();
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
index eab8e7c..7c213b4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/SpillingAdaptiveSpanningRecordDeserializer.java
@@ -22,7 +22,6 @@ import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.core.memory.DataInputView;
 import org.apache.flink.core.memory.DataInputViewStreamWrapper;
 import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.buffer.Buffer;
 import org.apache.flink.runtime.util.DataInputDeserializer;
 import org.apache.flink.util.StringUtils;
@@ -59,8 +58,6 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
 
 	private Buffer currentBuffer;
 
-	private AccumulatorRegistry.Reporter reporter;
-
 	public SpillingAdaptiveSpanningRecordDeserializer(String[] tmpDirectories) {
 		this.nonSpanningWrapper = new NonSpanningWrapper();
 		this.spanningWrapper = new SpanningWrapper(tmpDirectories);
@@ -106,19 +103,11 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
 		if (nonSpanningRemaining >= 4) {
 			int len = this.nonSpanningWrapper.readInt();
 
-			if (reporter != null) {
-				reporter.reportNumBytesIn(len);
-			}
-
 			if (len <= nonSpanningRemaining - 4) {
 				// we can get a full record from here
 				try {
 					target.read(this.nonSpanningWrapper);
 
-					if (reporter != null) {
-						reporter.reportNumRecordsIn(1);
-					}
-
 					int remaining = this.nonSpanningWrapper.remaining();
 					if (remaining > 0) {
 						return DeserializationResult.INTERMEDIATE_RECORD_FROM_BUFFER;
@@ -153,10 +142,6 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
 		if (this.spanningWrapper.hasFullRecord()) {
 			// get the full record
 			target.read(this.spanningWrapper.getInputView());
-
-			if (reporter != null) {
-				reporter.reportNumRecordsIn(1);
-			}
 			
 			// move the remainder to the non-spanning wrapper
 			// this does not copy it, only sets the memory segment
@@ -182,12 +167,6 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
 		return this.nonSpanningWrapper.remaining() > 0 || this.spanningWrapper.getNumGatheredBytes() > 0;
 	}
 
-	@Override
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-		this.reporter = reporter;
-		this.spanningWrapper.setReporter(reporter);
-	}
-
 
 	// -----------------------------------------------------------------------------------------------------------------
 	
@@ -483,8 +462,6 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
 		
 		private DataInputViewStreamWrapper spillFileReader;
 
-		private AccumulatorRegistry.Reporter reporter;
-
 		public SpanningWrapper(String[] tempDirs) {
 			this.tempDirs = tempDirs;
 			
@@ -538,10 +515,6 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
 				} else {
 					this.recordLength = this.lengthBuffer.getInt(0);
 
-					if (reporter != null) {
-						reporter.reportNumBytesIn(recordLength);
-					}
-
 					this.lengthBuffer.clear();
 					segmentPosition = toPut;
 					
@@ -672,9 +645,5 @@ public class SpillingAdaptiveSpanningRecordDeserializer<T extends IOReadableWrit
 			random.nextBytes(bytes);
 			return StringUtils.byteToHexString(bytes);
 		}
-
-		public void setReporter(AccumulatorRegistry.Reporter reporter) {
-			this.reporter = reporter;
-		}
 	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
index 96eea23..1e224c1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.io.network.api.writer;
 
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.event.AbstractEvent;
 import org.apache.flink.runtime.io.network.api.serialization.RecordSerializer;
 import org.apache.flink.runtime.io.network.api.serialization.SpanningRecordSerializer;
@@ -201,15 +200,6 @@ public class RecordWriter<T extends IOReadableWritable> {
 	}
 
 	/**
-	 * Counter for the number of records emitted and the records processed.
-	 */
-	public void setReporter(AccumulatorRegistry.Reporter reporter) {
-		for(RecordSerializer<?> serializer : serializers) {
-			serializer.setReporter(reporter);
-		}
-	}
-
-	/**
 	 * Sets the metric group for this RecordWriter.
 	 * @param metrics
      */

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java
index 4bc4532..2e3285c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
 import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
 import org.apache.flink.runtime.operators.Driver;
@@ -114,9 +113,8 @@ public class IterationHeadTask<X, Y, S extends Function, OT> extends AbstractIte
 		List<RecordWriter<?>> finalOutputWriters = new ArrayList<RecordWriter<?>>();
 		final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
 		final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
-		AccumulatorRegistry.Reporter reporter = getEnvironment().getAccumulatorRegistry().getReadWriteReporter();
 		this.finalOutputCollector = BatchTask.getOutputCollector(this, finalOutConfig,
-				userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs(), reporter);
+				userCodeClassLoader, finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());
 
 		// sanity check the setup
 		final int writersIntoStepFunction = this.eventualOutputs.size();

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
index e896639..f748079 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/BatchTask.java
@@ -31,7 +31,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.metrics.MetricGroup;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.broadcast.BroadcastVariableMaterialization;
 import org.apache.flink.runtime.execution.CancelTaskException;
 import org.apache.flink.runtime.execution.Environment;
@@ -654,9 +653,6 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 
 		int currentReaderOffset = 0;
 
-		AccumulatorRegistry registry = getEnvironment().getAccumulatorRegistry();
-		AccumulatorRegistry.Reporter reporter = registry.getReadWriteReporter();
-
 		for (int i = 0; i < numInputs; i++) {
 			//  ---------------- create the input readers ---------------------
 			// in case where a logical input unions multiple physical inputs, create a union reader
@@ -680,8 +676,6 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 				throw new Exception("Illegal input group size in task configuration: " + groupSize);
 			}
 
-			inputReaders[i].setReporter(reporter);
-
 			currentReaderOffset += groupSize;
 		}
 		this.inputReaders = inputReaders;
@@ -1015,13 +1009,10 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 
 		ClassLoader userCodeClassLoader = getUserCodeClassLoader();
 
-		AccumulatorRegistry accumulatorRegistry = getEnvironment().getAccumulatorRegistry();
-		AccumulatorRegistry.Reporter reporter = accumulatorRegistry.getReadWriteReporter();
-
-		this.accumulatorMap = accumulatorRegistry.getUserMap();
+		this.accumulatorMap = getEnvironment().getAccumulatorRegistry().getUserMap();
 
 		this.output = initOutputs(this, userCodeClassLoader, this.config, this.chainedTasks, this.eventualOutputs,
-				this.getExecutionConfig(), reporter, this.accumulatorMap);
+				this.getExecutionConfig(), this.accumulatorMap);
 	}
 
 	public DistributedRuntimeUDFContext createRuntimeContext(MetricGroup metrics) {
@@ -1215,7 +1206,7 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 	 * @return The OutputCollector that data produced in this task is submitted to.
 	 */
 	public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl,
-			List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs, AccumulatorRegistry.Reporter reporter) throws Exception
+			List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception
 	{
 		if (numOutputs == 0) {
 			return null;
@@ -1248,8 +1239,6 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 			final RecordWriter<SerializationDelegate<T>> recordWriter =
 					new RecordWriter<SerializationDelegate<T>>(task.getEnvironment().getWriter(outputOffset + i), oe);
 
-			// setup live accumulator counters
-			recordWriter.setReporter(reporter);
 			recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup());
 
 			writers.add(recordWriter);
@@ -1269,7 +1258,6 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 										List<ChainedDriver<?, ?>> chainedTasksTarget,
 										List<RecordWriter<?>> eventualOutputs,
 										ExecutionConfig executionConfig,
-										AccumulatorRegistry.Reporter reporter,
 										Map<String, Accumulator<?,?>> accumulatorMap)
 	throws Exception
 	{
@@ -1304,7 +1292,7 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 
 				if (i == numChained - 1) {
 					// last in chain, instantiate the output collector for this task
-					previous = getOutputCollector(containingTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs(), reporter);
+					previous = getOutputCollector(containingTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs());
 				}
 
 				ct.setup(chainedStubConf, taskName, previous, containingTask, cl, executionConfig, accumulatorMap);
@@ -1322,7 +1310,7 @@ public class BatchTask<S extends Function, OT> extends AbstractInvokable impleme
 		// else
 
 		// instantiate the output collector the default way from this configuration
-		return getOutputCollector(containingTask , config, cl, eventualOutputs, 0, numOutputs, reporter);
+		return getOutputCollector(containingTask , config, cl, eventualOutputs, 0, numOutputs);
 	}
 	
 	// --------------------------------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/flink/blob/ba2d007e/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
index eb7999c..bd052f5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
@@ -29,7 +29,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.core.io.IOReadableWritable;
 import org.apache.flink.metrics.Counter;
-import org.apache.flink.runtime.accumulators.AccumulatorRegistry;
 import org.apache.flink.runtime.execution.CancelTaskException;
 import org.apache.flink.runtime.execution.Environment;
 import org.apache.flink.runtime.io.network.api.reader.MutableReader;
@@ -354,11 +353,6 @@ public class DataSinkTask<IT> extends AbstractInvokable {
 		} else {
 			throw new Exception("Illegal input group size in task configuration: " + groupSize);
 		}
-
-		final AccumulatorRegistry accumulatorRegistry = getEnvironment().getAccumulatorRegistry();
-		final AccumulatorRegistry.Reporter reporter = accumulatorRegistry.getReadWriteReporter();
-
-		inputReader.setReporter(reporter);
 		
 		this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
 		@SuppressWarnings({ "rawtypes" })