You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2016/04/13 10:31:10 UTC

[12/14] flink git commit: [FLINK-3126] [core] Remove accumulator type from "value" in web frontend

[FLINK-3126] [core] Remove accumulator type from "value" in web frontend

This closes #1868


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/db85f385
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/db85f385
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/db85f385

Branch: refs/heads/master
Commit: db85f385846a6541c0707ed1ba6fed78446423b5
Parents: 342db48
Author: Zack Pierce <zp...@newrelic.com>
Authored: Mon Apr 11 10:17:34 2016 -0700
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Apr 13 01:10:55 2016 +0200

----------------------------------------------------------------------
 .../StringifiedAccumulatorResult.java           |  14 +-
 .../StringifiedAccumulatorResultTest.java       | 138 +++++++++++++++++++
 2 files changed, 149 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/db85f385/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
index a0d1eda..c4faad1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
@@ -55,6 +55,9 @@ public class StringifiedAccumulatorResult implements java.io.Serializable{
 	//  Utilities
 	// ------------------------------------------------------------------------
 
+	/**
+	 * Flatten a map of accumulator names to Accumulator instances into an array of StringifiedAccumulatorResult values
+     */
 	public static StringifiedAccumulatorResult[] stringifyAccumulatorResults(Map<String, Accumulator<?, ?>> accs) {
 		if (accs == null || accs.isEmpty()) {
 			return new StringifiedAccumulatorResult[0];
@@ -65,9 +68,14 @@ public class StringifiedAccumulatorResult implements java.io.Serializable{
 			int i = 0;
 			for (Map.Entry<String, Accumulator<?, ?>> entry : accs.entrySet()) {
 				StringifiedAccumulatorResult result;
-				Accumulator<?, ?> value = entry.getValue();
-				if (value != null) {
-					result = new StringifiedAccumulatorResult(entry.getKey(), value.getClass().getSimpleName(), value.toString());
+				Accumulator<?, ?> accumulator = entry.getValue();
+				if (accumulator != null) {
+					Object localValue = accumulator.getLocalValue();
+					if (localValue != null) {
+						result = new StringifiedAccumulatorResult(entry.getKey(), accumulator.getClass().getSimpleName(), localValue.toString());
+					} else {
+						result = new StringifiedAccumulatorResult(entry.getKey(), accumulator.getClass().getSimpleName(), "null");
+					}
 				} else {
 					result = new StringifiedAccumulatorResult(entry.getKey(), "null", "null");
 				}

http://git-wip-us.apache.org/repos/asf/flink/blob/db85f385/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java
new file mode 100644
index 0000000..e6d637b
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.accumulators;
+
+import org.apache.flink.api.common.accumulators.Accumulator;
+import org.apache.flink.api.common.accumulators.IntCounter;
+import org.apache.flink.api.common.accumulators.SimpleAccumulator;
+import org.apache.flink.core.testutils.CommonTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class StringifiedAccumulatorResultTest {
+
+	@Test
+	public void testSerialization() throws IOException {
+		final String name = "a";
+		final String type = "b";
+		final String value = "c";
+		final StringifiedAccumulatorResult original = new StringifiedAccumulatorResult(name, type, value);
+
+		// Confirm no funny business in the constructor to getter pathway
+		assertEquals(name, original.getName());
+		assertEquals(type, original.getType());
+		assertEquals(value, original.getValue());
+
+		final StringifiedAccumulatorResult copy = CommonTestUtils.createCopySerializable(original);
+
+		// Copy should have equivalent core fields
+		assertEquals(name, copy.getName());
+		assertEquals(type, copy.getType());
+		assertEquals(value, copy.getValue());
+	}
+
+	@Test
+	public void stringifyingResultsShouldIncorporateAccumulatorLocalValueDirectly() {
+		final String name = "a";
+		final int targetValue = 314159;
+		final IntCounter acc = new IntCounter();
+		acc.add(targetValue);
+		final Map<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
+		accumulatorMap.put(name, acc);
+
+
+		final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
+
+		assertEquals(1, results.length);
+
+		final StringifiedAccumulatorResult firstResult = results[0];
+		assertEquals(name, firstResult.getName());
+		assertEquals("IntCounter", firstResult.getType());
+		assertEquals(Integer.toString(targetValue), firstResult.getValue());
+	}
+
+	@Test
+	public void stringifyingResultsShouldReportNullLocalValueAsNonnullValueString() {
+		final String name = "a";
+		final NullBearingAccumulator acc = new NullBearingAccumulator();
+		final Map<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
+		accumulatorMap.put(name, acc);
+
+		final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
+
+		assertEquals(1, results.length);
+
+		// Note the use of a String with a content of "null" rather than a null value
+		final StringifiedAccumulatorResult firstResult = results[0];
+		assertEquals(name, firstResult.getName());
+		assertEquals("NullBearingAccumulator", firstResult.getType());
+		assertEquals("null", firstResult.getValue());
+	}
+
+	@Test
+	public void stringifyingResultsShouldReportNullAccumulatorWithNonnullValueAndTypeString() {
+		final String name = "a";
+		final Map<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
+		accumulatorMap.put(name, null);
+
+		final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
+
+		assertEquals(1, results.length);
+
+		// Note the use of String values with content of "null" rather than null values
+		final StringifiedAccumulatorResult firstResult = results[0];
+		assertEquals(name, firstResult.getName());
+		assertEquals("null", firstResult.getType());
+		assertEquals("null", firstResult.getValue());
+	}
+
+	private static class NullBearingAccumulator implements SimpleAccumulator<Serializable> {
+
+		@Override
+		public void add(Serializable value) {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public Serializable getLocalValue() {
+			return null;
+		}
+
+		@Override
+		public void resetLocal() {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public void merge(Accumulator<Serializable, Serializable> other) {
+			throw new UnsupportedOperationException();
+		}
+
+		@Override
+		public Accumulator<Serializable, Serializable> clone() {
+			return new NullBearingAccumulator();
+		}
+	}
+}