You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2018/03/26 07:33:35 UTC
[5/5] flink git commit: [FLINK-8721][flip6] Handle archiving failures
for accumulators
[FLINK-8721][flip6] Handle archiving failures for accumulators
During archivization, wrap errors thrown by users' Accumulators into a OptionalFailure
and do not fail the job because of that.
This closes #5737.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a6a7623d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a6a7623d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a6a7623d
Branch: refs/heads/master
Commit: a6a7623dd8a549f4a31bbc361b891c65b5feef07
Parents: 1132cd7
Author: Piotr Nowojski <pi...@gmail.com>
Authored: Wed Mar 21 13:08:36 2018 +0100
Committer: Till Rohrmann <tr...@apache.org>
Committed: Mon Mar 26 09:33:06 2018 +0200
----------------------------------------------------------------------
.../flink/client/program/ClusterClient.java | 7 +-
.../flink/client/program/MiniClusterClient.java | 11 +-
.../client/program/rest/RestClusterClient.java | 5 +-
.../program/rest/RestClusterClientTest.java | 11 +-
.../flink/api/common/JobExecutionResult.java | 14 +-
.../common/accumulators/AccumulatorHelper.java | 63 +++++---
.../FailedAccumulatorSerialization.java | 73 ---------
.../common/operators/CollectionExecutor.java | 3 +-
.../org/apache/flink/util/OptionalFailure.java | 135 +++++++++++++++++
.../flink/util/function/CheckedSupplier.java | 12 ++
.../FailedAccumulatorSerializationTest.java | 89 -----------
.../accumulators/AccumulatorSnapshot.java | 2 +-
.../StringifiedAccumulatorResult.java | 54 +++++--
.../client/SerializedJobExecutionResult.java | 13 +-
.../executiongraph/AccessExecutionGraph.java | 3 +-
.../executiongraph/ArchivedExecutionGraph.java | 10 +-
.../flink/runtime/executiongraph/Execution.java | 10 +-
.../runtime/executiongraph/ExecutionGraph.java | 46 +++---
.../executiongraph/ExecutionJobVertex.java | 3 +-
.../flink/runtime/jobmaster/JobResult.java | 11 +-
.../handler/job/JobAccumulatorsHandler.java | 3 +-
.../rest/messages/JobAccumulatorsInfo.java | 7 +-
.../messages/json/JobResultDeserializer.java | 9 +-
.../rest/messages/json/JobResultSerializer.java | 7 +-
.../accumulators/AccumulatorMessages.scala | 7 +-
.../StringifiedAccumulatorResultTest.java | 30 +++-
.../SerializedJobExecutionResultTest.java | 46 ++++--
.../ArchivedExecutionGraphTest.java | 12 +-
...ExecutionAttemptAccumulatorsHandlerTest.java | 9 +-
.../utils/ArchivedExecutionGraphBuilder.java | 7 +-
.../job/JobExecutionResultResponseBodyTest.java | 3 +-
.../TestingJobManagerMessages.scala | 3 +-
.../accumulators/AccumulatorErrorITCase.java | 149 +++++++++++--------
.../test/accumulators/AccumulatorITCase.java | 1 +
.../accumulators/AccumulatorLiveITCase.java | 2 +-
.../LegacyAccumulatorLiveITCase.java | 7 +-
.../utils/SavepointMigrationTestBase.java | 11 +-
37 files changed, 524 insertions(+), 364 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
index 166d977..79a5383 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/ClusterClient.java
@@ -65,6 +65,7 @@ import org.apache.flink.runtime.util.LeaderConnectionInfo;
import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import org.apache.flink.util.FlinkException;
import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;
@@ -792,7 +793,7 @@ public abstract class ClusterClient<T> {
* @param jobID The job identifier of a job.
* @return A Map containing the accumulator's name and its value.
*/
- public Map<String, Object> getAccumulators(JobID jobID) throws Exception {
+ public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID) throws Exception {
return getAccumulators(jobID, ClassLoader.getSystemClassLoader());
}
@@ -803,7 +804,7 @@ public abstract class ClusterClient<T> {
* @param loader The class loader for deserializing the accumulator results.
* @return A Map containing the accumulator's name and its value.
*/
- public Map<String, Object> getAccumulators(JobID jobID, ClassLoader loader) throws Exception {
+ public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID, ClassLoader loader) throws Exception {
ActorGateway jobManagerGateway = getJobManagerGateway();
Future<Object> response;
@@ -816,7 +817,7 @@ public abstract class ClusterClient<T> {
Object result = Await.result(response, timeout);
if (result instanceof AccumulatorResultsFound) {
- Map<String, SerializedValue<Object>> serializedAccumulators =
+ Map<String, SerializedValue<OptionalFailure<Object>>> serializedAccumulators =
((AccumulatorResultsFound) result).result();
return AccumulatorHelper.deserializeAccumulators(serializedAccumulators, loader);
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java
index 276df62..44f6ef6 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/MiniClusterClient.java
@@ -41,6 +41,7 @@ import org.apache.flink.runtime.util.LeaderConnectionInfo;
import org.apache.flink.runtime.util.LeaderRetrievalUtils;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedValue;
import javax.annotation.Nonnull;
@@ -132,16 +133,16 @@ public class MiniClusterClient extends ClusterClient<MiniClusterClient.MiniClust
}
@Override
- public Map<String, Object> getAccumulators(JobID jobID) throws Exception {
+ public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID) throws Exception {
return getAccumulators(jobID, ClassLoader.getSystemClassLoader());
}
@Override
- public Map<String, Object> getAccumulators(JobID jobID, ClassLoader loader) throws Exception {
+ public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID, ClassLoader loader) throws Exception {
AccessExecutionGraph executionGraph = guardWithSingleRetry(() -> miniCluster.getExecutionGraph(jobID), scheduledExecutor).get();
- Map<String, SerializedValue<Object>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized();
- Map<String, Object> result = new HashMap<>(accumulatorsSerialized.size());
- for (Map.Entry<String, SerializedValue<Object>> acc : accumulatorsSerialized.entrySet()) {
+ Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized();
+ Map<String, OptionalFailure<Object>> result = new HashMap<>(accumulatorsSerialized.size());
+ for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> acc : accumulatorsSerialized.entrySet()) {
result.put(acc.getKey(), acc.getValue().deserializeValue(loader));
}
return result;
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
index f3f1961..2e1ffb0 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/rest/RestClusterClient.java
@@ -89,6 +89,7 @@ import org.apache.flink.runtime.webmonitor.retriever.LeaderRetriever;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.ExecutorUtils;
import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.function.CheckedSupplier;
@@ -409,7 +410,7 @@ public class RestClusterClient<T> extends ClusterClient<T> {
}
@Override
- public Map<String, Object> getAccumulators(final JobID jobID, ClassLoader loader) throws Exception {
+ public Map<String, OptionalFailure<Object>> getAccumulators(final JobID jobID, ClassLoader loader) throws Exception {
final JobAccumulatorsHeaders accumulatorsHeaders = JobAccumulatorsHeaders.getInstance();
final JobAccumulatorsMessageParameters accMsgParams = accumulatorsHeaders.getUnresolvedMessageParameters();
accMsgParams.jobPathParameter.resolve(jobID);
@@ -420,7 +421,7 @@ public class RestClusterClient<T> extends ClusterClient<T> {
accMsgParams
);
- Map<String, Object> result = Collections.emptyMap();
+ Map<String, OptionalFailure<Object>> result = Collections.emptyMap();
try {
result = responseFuture.thenApply((JobAccumulatorsInfo accumulatorsInfo) -> {
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
index e98ba43..77a4113 100644
--- a/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
+++ b/flink-clients/src/test/java/org/apache/flink/client/program/rest/RestClusterClientTest.java
@@ -84,6 +84,7 @@ import org.apache.flink.runtime.util.ExecutorThreadFactory;
import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
import org.apache.flink.testutils.category.Flip6;
import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedThrowable;
import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
@@ -362,7 +363,7 @@ public class RestClusterClientTest extends TestLogger {
JobExecutionResultResponseBody.created(new JobResult.Builder()
.jobId(jobId)
.netRuntime(Long.MAX_VALUE)
- .accumulatorResults(Collections.singletonMap("testName", new SerializedValue<>(1.0)))
+ .accumulatorResults(Collections.singletonMap("testName", new SerializedValue<>(OptionalFailure.of(1.0))))
.build()),
JobExecutionResultResponseBody.created(new JobResult.Builder()
.jobId(jobId)
@@ -558,12 +559,12 @@ public class RestClusterClientTest extends TestLogger {
JobID id = new JobID();
{
- Map<String, Object> accumulators = restClusterClient.getAccumulators(id);
+ Map<String, OptionalFailure<Object>> accumulators = restClusterClient.getAccumulators(id);
assertNotNull(accumulators);
assertEquals(1, accumulators.size());
assertEquals(true, accumulators.containsKey("testKey"));
- assertEquals("testValue", accumulators.get("testKey").toString());
+ assertEquals("testValue", accumulators.get("testKey").get().toString());
}
}
}
@@ -594,9 +595,9 @@ public class RestClusterClientTest extends TestLogger {
userTaskAccumulators.add(new JobAccumulatorsInfo.UserTaskAccumulator("testName", "testType", "testValue"));
if (includeSerializedValue) {
- Map<String, SerializedValue<Object>> serializedUserTaskAccumulators = new HashMap<>(1);
+ Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserTaskAccumulators = new HashMap<>(1);
try {
- serializedUserTaskAccumulators.put("testKey", new SerializedValue<>("testValue"));
+ serializedUserTaskAccumulators.put("testKey", new SerializedValue<>(OptionalFailure.of("testValue")));
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
index a200d12..9e1a3a5 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
@@ -20,10 +20,12 @@ package org.apache.flink.api.common;
import org.apache.flink.annotation.Public;
import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.util.OptionalFailure;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
/**
* The result of a job execution. Gives access to the execution time of the job,
@@ -34,7 +36,7 @@ public class JobExecutionResult extends JobSubmissionResult {
private final long netRuntime;
- private final Map<String, Object> accumulatorResults;
+ private final Map<String, OptionalFailure<Object>> accumulatorResults;
/**
* Creates a new JobExecutionResult.
@@ -43,7 +45,7 @@ public class JobExecutionResult extends JobSubmissionResult {
* @param netRuntime The net runtime of the job (excluding pre-flight phase like the optimizer) in milliseconds
* @param accumulators A map of all accumulators produced by the job.
*/
- public JobExecutionResult(JobID jobID, long netRuntime, Map<String, Object> accumulators) {
+ public JobExecutionResult(JobID jobID, long netRuntime, Map<String, OptionalFailure<Object>> accumulators) {
super(jobID);
this.netRuntime = netRuntime;
@@ -85,7 +87,7 @@ public class JobExecutionResult extends JobSubmissionResult {
*/
@SuppressWarnings("unchecked")
public <T> T getAccumulatorResult(String accumulatorName) {
- return (T) this.accumulatorResults.get(accumulatorName);
+ return (T) this.accumulatorResults.get(accumulatorName).getUnchecked();
}
/**
@@ -95,7 +97,9 @@ public class JobExecutionResult extends JobSubmissionResult {
* @return A map containing all accumulators produced by the job.
*/
public Map<String, Object> getAllAccumulatorResults() {
- return this.accumulatorResults;
+ return accumulatorResults.entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getUnchecked()));
}
/**
@@ -109,7 +113,7 @@ public class JobExecutionResult extends JobSubmissionResult {
@Deprecated
@PublicEvolving
public Integer getIntCounterResult(String accumulatorName) {
- Object result = this.accumulatorResults.get(accumulatorName);
+ Object result = this.accumulatorResults.get(accumulatorName).getUnchecked();
if (result == null) {
return null;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
index 78fb68b..9bc1299 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
@@ -19,20 +19,27 @@
package org.apache.flink.api.common.accumulators;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.IOException;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.function.Supplier;
/**
* Helper functions for the interaction with {@link Accumulator}.
*/
@Internal
public class AccumulatorHelper {
+ private static final Logger LOG = LoggerFactory.getLogger(AccumulatorHelper.class);
/**
* Merge two collections of accumulators. The second will be merged into the
@@ -44,19 +51,28 @@ public class AccumulatorHelper {
* The collection of accumulators that will be merged into the
* other
*/
- public static void mergeInto(Map<String, Accumulator<?, ?>> target, Map<String, Accumulator<?, ?>> toMerge) {
+ public static void mergeInto(Map<String, OptionalFailure<Accumulator<?, ?>>> target, Map<String, Accumulator<?, ?>> toMerge) {
for (Map.Entry<String, Accumulator<?, ?>> otherEntry : toMerge.entrySet()) {
- Accumulator<?, ?> ownAccumulator = target.get(otherEntry.getKey());
+ OptionalFailure<Accumulator<?, ?>> ownAccumulator = target.get(otherEntry.getKey());
if (ownAccumulator == null) {
// Create initial counter (copy!)
- target.put(otherEntry.getKey(), otherEntry.getValue().clone());
+ target.put(
+ otherEntry.getKey(),
+ wrapUnchecked(otherEntry.getKey(), () -> otherEntry.getValue().clone()));
+ }
+ else if (ownAccumulator.isFailure()) {
+ continue;
}
else {
+ Accumulator<?, ?> accumulator = ownAccumulator.getUnchecked();
// Both should have the same type
- AccumulatorHelper.compareAccumulatorTypes(otherEntry.getKey(),
- ownAccumulator.getClass(), otherEntry.getValue().getClass());
+ compareAccumulatorTypes(otherEntry.getKey(),
+ accumulator.getClass(), otherEntry.getValue().getClass());
// Merge target counter with other counter
- mergeSingle(ownAccumulator, otherEntry.getValue());
+
+ target.put(
+ otherEntry.getKey(),
+ wrapUnchecked(otherEntry.getKey(), () -> mergeSingle(accumulator, otherEntry.getValue().clone())));
}
}
}
@@ -64,8 +80,8 @@ public class AccumulatorHelper {
/**
* Workaround method for type safety.
*/
- private static <V, R extends Serializable> void mergeSingle(Accumulator<?, ?> target,
- Accumulator<?, ?> toMerge) {
+ private static <V, R extends Serializable> Accumulator<V, R> mergeSingle(Accumulator<?, ?> target,
+ Accumulator<?, ?> toMerge) {
@SuppressWarnings("unchecked")
Accumulator<V, R> typedTarget = (Accumulator<V, R>) target;
@@ -73,6 +89,8 @@ public class AccumulatorHelper {
Accumulator<V, R> typedToMerge = (Accumulator<V, R>) toMerge;
typedTarget.merge(typedToMerge);
+
+ return typedTarget;
}
/**
@@ -106,14 +124,25 @@ public class AccumulatorHelper {
* Transform the Map with accumulators into a Map containing only the
* results.
*/
- public static Map<String, Object> toResultMap(Map<String, Accumulator<?, ?>> accumulators) {
- Map<String, Object> resultMap = new HashMap<String, Object>();
+ public static Map<String, OptionalFailure<Object>> toResultMap(Map<String, Accumulator<?, ?>> accumulators) {
+ Map<String, OptionalFailure<Object>> resultMap = new HashMap<>();
for (Map.Entry<String, Accumulator<?, ?>> entry : accumulators.entrySet()) {
- resultMap.put(entry.getKey(), entry.getValue().getLocalValue());
+ resultMap.put(entry.getKey(), wrapUnchecked(entry.getKey(), () -> entry.getValue().getLocalValue()));
}
return resultMap;
}
+ private static <R> OptionalFailure<R> wrapUnchecked(String name, Supplier<R> supplier) {
+ return OptionalFailure.createFrom(() -> {
+ try {
+ return supplier.get();
+ } catch (RuntimeException ex) {
+ LOG.error("Unexpected error while handling accumulator [" + name + "]", ex);
+ throw new FlinkException(ex);
+ }
+ });
+ }
+
public static String getResultsFormatted(Map<String, Object> map) {
StringBuilder builder = new StringBuilder();
for (Map.Entry<String, Object> entry : map.entrySet()) {
@@ -152,19 +181,19 @@ public class AccumulatorHelper {
* @throws IOException
* @throws ClassNotFoundException
*/
- public static Map<String, Object> deserializeAccumulators(
- Map<String, SerializedValue<Object>> serializedAccumulators, ClassLoader loader)
- throws IOException, ClassNotFoundException {
+ public static Map<String, OptionalFailure<Object>> deserializeAccumulators(
+ Map<String, SerializedValue<OptionalFailure<Object>>> serializedAccumulators,
+ ClassLoader loader) throws IOException, ClassNotFoundException {
if (serializedAccumulators == null || serializedAccumulators.isEmpty()) {
return Collections.emptyMap();
}
- Map<String, Object> accumulators = new HashMap<>(serializedAccumulators.size());
+ Map<String, OptionalFailure<Object>> accumulators = new HashMap<>(serializedAccumulators.size());
- for (Map.Entry<String, SerializedValue<Object>> entry : serializedAccumulators.entrySet()) {
+ for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> entry : serializedAccumulators.entrySet()) {
- Object value = null;
+ OptionalFailure<Object> value = null;
if (entry.getValue() != null) {
value = entry.getValue().deserializeValue(loader);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java
deleted file mode 100644
index b208b9e..0000000
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerialization.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.common.accumulators;
-
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.Preconditions;
-
-import java.io.Serializable;
-
-/**
- * {@link Accumulator} implementation which indicates a serialization problem with the original
- * accumulator. Accessing any of the {@link Accumulator} method will result in throwing the
- * serialization exception.
- *
- * @param <V> type of the value
- * @param <R> type of the accumulator result
- */
-public class FailedAccumulatorSerialization<V, R extends Serializable> implements Accumulator<V, R> {
- private static final long serialVersionUID = 6965908827065879760L;
-
- private final Throwable throwable;
-
- public FailedAccumulatorSerialization(Throwable throwable) {
- this.throwable = Preconditions.checkNotNull(throwable);
- }
-
- public Throwable getThrowable() {
- return throwable;
- }
-
- @Override
- public void add(V value) {
- ExceptionUtils.rethrow(throwable);
- }
-
- @Override
- public R getLocalValue() {
- ExceptionUtils.rethrow(throwable);
- return null;
- }
-
- @Override
- public void resetLocal() {
- ExceptionUtils.rethrow(throwable);
- }
-
- @Override
- public void merge(Accumulator<V, R> other) {
- ExceptionUtils.rethrow(throwable);
- }
-
- @Override
- public Accumulator<V, R> clone() {
- ExceptionUtils.rethrow(throwable);
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
index 07f48fc..55f3df7 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/CollectionExecutor.java
@@ -50,6 +50,7 @@ import org.apache.flink.core.fs.local.LocalFileSystem;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
import org.apache.flink.types.Value;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Visitor;
import java.util.ArrayList;
@@ -115,7 +116,7 @@ public class CollectionExecutor {
}
long endTime = System.currentTimeMillis();
- Map<String, Object> accumulatorResults = AccumulatorHelper.toResultMap(accumulators);
+ Map<String, OptionalFailure<Object>> accumulatorResults = AccumulatorHelper.toResultMap(accumulators);
return new JobExecutionResult(null, endTime - startTime, accumulatorResults);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java b/flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java
new file mode 100644
index 0000000..ace3cad
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/util/OptionalFailure.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.util;
+
+import org.apache.flink.util.function.CheckedSupplier;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.Objects;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Wrapper around an object representing either a success (with a given value) or a failure cause.
+ */
+public class OptionalFailure<T> implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Nullable
+ private transient T value;
+
+ @Nullable
+ private Throwable failureCause;
+
+ private OptionalFailure(@Nullable T value, @Nullable Throwable failureCause) {
+ this.value = value;
+ this.failureCause = failureCause;
+ }
+
+ public static <T> OptionalFailure<T> of(T value) {
+ return new OptionalFailure<>(value, null);
+ }
+
+ public static <T> OptionalFailure<T> ofFailure(Throwable failureCause) {
+ return new OptionalFailure<>(null, failureCause);
+ }
+
+ /**
+ * @return wrapped {@link OptionalFailure} returned by {@code valueSupplier} or wrapped failure if
+ * {@code valueSupplier} has thrown an {@link Exception}.
+ */
+ public static <T> OptionalFailure<T> createFrom(CheckedSupplier<T> valueSupplier) {
+ try {
+ return of(valueSupplier.get());
+ } catch (Exception ex) {
+ return ofFailure(ex);
+ }
+ }
+
+ /**
+ * @return stored value or throw a {@link FlinkException} with {@code failureCause}.
+ */
+ public T get() throws FlinkException {
+ if (value != null) {
+ return value;
+ }
+ checkNotNull(failureCause);
+ throw new FlinkException(failureCause);
+ }
+
+ /**
+ * @return same as {@link #get()} but throws a {@link FlinkRuntimeException}.
+ */
+ public T getUnchecked() throws FlinkRuntimeException {
+ if (value != null) {
+ return value;
+ }
+ checkNotNull(failureCause);
+ throw new FlinkRuntimeException(failureCause);
+ }
+
+ public Throwable getFailureCause() {
+ return checkNotNull(failureCause);
+ }
+
+ public boolean isFailure() {
+ return failureCause != null;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(value, failureCause);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof OptionalFailure<?>)) {
+ return false;
+ }
+ OptionalFailure<?> other = (OptionalFailure<?>) obj;
+ return Objects.equals(value, other.value) &&
+ Objects.equals(failureCause, other.failureCause);
+ }
+
+ private void writeObject(ObjectOutputStream stream) throws IOException {
+ stream.defaultWriteObject();
+ stream.writeObject(value);
+ }
+
+ private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException {
+ stream.defaultReadObject();
+ value = (T) stream.readObject();
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "{value=" + value + ", failureCause=" + failureCause + "}";
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java b/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java
index a0bcc13..585d705 100644
--- a/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java
+++ b/flink-core/src/main/java/org/apache/flink/util/function/CheckedSupplier.java
@@ -18,6 +18,8 @@
package org.apache.flink.util.function;
+import org.apache.flink.util.FlinkException;
+
import java.util.function.Supplier;
/**
@@ -36,4 +38,14 @@ public interface CheckedSupplier<R> extends SupplierWithException<R, Exception>
};
}
+ static <R> CheckedSupplier<R> checked(Supplier<R> supplier) {
+ return () -> {
+ try {
+ return supplier.get();
+ }
+ catch (RuntimeException e) {
+ throw new FlinkException(e);
+ }
+ };
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java b/flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java
deleted file mode 100644
index 7335d30..0000000
--- a/flink-core/src/test/java/org/apache/flink/api/common/accumulators/FailedAccumulatorSerializationTest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.common.accumulators;
-
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.flink.util.TestLogger;
-
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-/**
- * Tests for the {@link FailedAccumulatorSerialization}.
- */
-public class FailedAccumulatorSerializationTest extends TestLogger {
-
- private static final IOException TEST_EXCEPTION = new IOException("Test exception");
-
- /**
- * Tests that any method call will throw the contained throwable (wrapped in an
- * unchecked exception if it is checked).
- */
- @Test
- public void testMethodCallThrowsException() {
- final FailedAccumulatorSerialization<Integer, Integer> accumulator = new FailedAccumulatorSerialization<>(TEST_EXCEPTION);
-
- try {
- accumulator.getLocalValue();
- } catch (RuntimeException re) {
- assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true));
- }
-
- try {
- accumulator.resetLocal();
- } catch (RuntimeException re) {
- assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true));
- }
-
- try {
- accumulator.add(1);
- } catch (RuntimeException re) {
- assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true));
- }
-
- try {
- accumulator.merge(new IntMinimum());
- } catch (RuntimeException re) {
- assertThat(ExceptionUtils.findThrowableWithMessage(re, TEST_EXCEPTION.getMessage()).isPresent(), is(true));
- }
- }
-
- /**
- * Tests that the class can be serialized and deserialized using Java serialization.
- */
- @Test
- public void testSerialization() throws Exception {
- final FailedAccumulatorSerialization<?, ?> accumulator = new FailedAccumulatorSerialization<>(TEST_EXCEPTION);
-
- final byte[] serializedAccumulator = InstantiationUtil.serializeObject(accumulator);
-
- final FailedAccumulatorSerialization<?, ?> deserializedAccumulator = InstantiationUtil.deserializeObject(serializedAccumulator, ClassLoader.getSystemClassLoader());
-
- assertThat(deserializedAccumulator.getThrowable(), is(instanceOf(TEST_EXCEPTION.getClass())));
- assertThat(deserializedAccumulator.getThrowable().getMessage(), is(equalTo(TEST_EXCEPTION.getMessage())));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
index 0bfb1ac..d01cd32 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorSnapshot.java
@@ -48,7 +48,7 @@ public class AccumulatorSnapshot implements Serializable {
Map<String, Accumulator<?, ?>> userAccumulators) throws IOException {
this.jobID = jobID;
this.executionAttemptID = executionAttemptID;
- this.userAccumulators = new SerializedValue<Map<String, Accumulator<?, ?>>>(userAccumulators);
+ this.userAccumulators = new SerializedValue<>(userAccumulators);
}
public JobID getJobID() {
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
index b55159c..f283bcd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
@@ -19,6 +19,13 @@
package org.apache.flink.runtime.accumulators;
import org.apache.flink.api.common.accumulators.Accumulator;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.OptionalFailure;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
import java.util.Map;
@@ -26,6 +33,7 @@ import java.util.Map;
* Container class that transports the result of an accumulator as set of strings.
*/
public class StringifiedAccumulatorResult implements java.io.Serializable{
+ private static final Logger LOG = LoggerFactory.getLogger(StringifiedAccumulatorResult.class);
private static final long serialVersionUID = -4642311296836822611L;
@@ -58,7 +66,7 @@ public class StringifiedAccumulatorResult implements java.io.Serializable{
/**
* Flatten a map of accumulator names to Accumulator instances into an array of StringifiedAccumulatorResult values.
*/
- public static StringifiedAccumulatorResult[] stringifyAccumulatorResults(Map<String, Accumulator<?, ?>> accs) {
+ public static StringifiedAccumulatorResult[] stringifyAccumulatorResults(Map<String, OptionalFailure<Accumulator<?, ?>>> accs) {
if (accs == null || accs.isEmpty()) {
return new StringifiedAccumulatorResult[0];
}
@@ -66,23 +74,37 @@ public class StringifiedAccumulatorResult implements java.io.Serializable{
StringifiedAccumulatorResult[] results = new StringifiedAccumulatorResult[accs.size()];
int i = 0;
- for (Map.Entry<String, Accumulator<?, ?>> entry : accs.entrySet()) {
- StringifiedAccumulatorResult result;
- Accumulator<?, ?> accumulator = entry.getValue();
- if (accumulator != null) {
- Object localValue = accumulator.getLocalValue();
- if (localValue != null) {
- result = new StringifiedAccumulatorResult(entry.getKey(), accumulator.getClass().getSimpleName(), localValue.toString());
- } else {
- result = new StringifiedAccumulatorResult(entry.getKey(), accumulator.getClass().getSimpleName(), "null");
- }
- } else {
- result = new StringifiedAccumulatorResult(entry.getKey(), "null", "null");
- }
-
- results[i++] = result;
+ for (Map.Entry<String, OptionalFailure<Accumulator<?, ?>>> entry : accs.entrySet()) {
+ results[i++] = stringifyAccumulatorResult(entry.getKey(), entry.getValue());
}
return results;
}
}
+
+ private static StringifiedAccumulatorResult stringifyAccumulatorResult(
+ String name,
+ @Nullable OptionalFailure<Accumulator<?, ?>> accumulator) {
+ if (accumulator == null) {
+ return new StringifiedAccumulatorResult(name, "null", "null");
+ }
+ else if (accumulator.isFailure()) {
+ return new StringifiedAccumulatorResult(
+ name,
+ "null",
+ ExceptionUtils.stringifyException(accumulator.getFailureCause()));
+ }
+ else {
+ Object localValue;
+ String simpleName = "null";
+ try {
+ simpleName = accumulator.getUnchecked().getClass().getSimpleName();
+ localValue = accumulator.getUnchecked().getLocalValue();
+ }
+ catch (RuntimeException exception) {
+ LOG.error("Failed to stringify accumulator [" + name + "]", exception);
+ localValue = ExceptionUtils.stringifyException(exception);
+ }
+ return new StringifiedAccumulatorResult(name, simpleName, localValue != null ? localValue.toString() : "null");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java
index ec2312f..0ca7a41 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.client;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.accumulators.AccumulatorHelper;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedValue;
import java.io.IOException;
@@ -37,7 +38,7 @@ public class SerializedJobExecutionResult implements java.io.Serializable {
private final JobID jobId;
- private final Map<String, SerializedValue<Object>> accumulatorResults;
+ private final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults;
private final long netRuntime;
@@ -48,8 +49,10 @@ public class SerializedJobExecutionResult implements java.io.Serializable {
* @param netRuntime The net runtime of the job (excluding pre-flight phase like the optimizer) in milliseconds
* @param accumulators A map of all accumulator results produced by the job, in serialized form
*/
- public SerializedJobExecutionResult(JobID jobID, long netRuntime,
- Map<String, SerializedValue<Object>> accumulators) {
+ public SerializedJobExecutionResult(
+ JobID jobID,
+ long netRuntime,
+ Map<String, SerializedValue<OptionalFailure<Object>>> accumulators) {
this.jobId = jobID;
this.netRuntime = netRuntime;
this.accumulatorResults = accumulators;
@@ -74,12 +77,12 @@ public class SerializedJobExecutionResult implements java.io.Serializable {
return desiredUnit.convert(getNetRuntime(), TimeUnit.MILLISECONDS);
}
- public Map<String, SerializedValue<Object>> getSerializedAccumulatorResults() {
+ public Map<String, SerializedValue<OptionalFailure<Object>>> getSerializedAccumulatorResults() {
return this.accumulatorResults;
}
public JobExecutionResult toJobExecutionResult(ClassLoader loader) throws IOException, ClassNotFoundException {
- Map<String, Object> accumulators =
+ Map<String, OptionalFailure<Object>> accumulators =
AccumulatorHelper.deserializeAccumulators(accumulatorResults, loader);
return new JobExecutionResult(jobId, netRuntime, accumulators);
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
index cc56209..de578bd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/AccessExecutionGraph.java
@@ -25,6 +25,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedValue;
import javax.annotation.Nullable;
@@ -155,7 +156,7 @@ public interface AccessExecutionGraph {
*
* @return map containing serialized values of user-defined accumulators
*/
- Map<String, SerializedValue<Object>> getAccumulatorsSerialized();
+ Map<String, SerializedValue<OptionalFailure<Object>>> getAccumulatorsSerialized();
/**
* Returns whether this execution graph was archived.
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
index d471f86..2489721 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraph.java
@@ -25,6 +25,7 @@ import org.apache.flink.runtime.checkpoint.CheckpointStatsSnapshot;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;
@@ -86,7 +87,7 @@ public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializabl
private final StringifiedAccumulatorResult[] archivedUserAccumulators;
private final ArchivedExecutionConfig archivedExecutionConfig;
private final boolean isStoppable;
- private final Map<String, SerializedValue<Object>> serializedUserAccumulators;
+ private final Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators;
@Nullable
private final CheckpointCoordinatorConfiguration jobCheckpointingConfiguration;
@@ -104,7 +105,7 @@ public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializabl
@Nullable ErrorInfo failureCause,
String jsonPlan,
StringifiedAccumulatorResult[] archivedUserAccumulators,
- Map<String, SerializedValue<Object>> serializedUserAccumulators,
+ Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators,
ArchivedExecutionConfig executionConfig,
boolean isStoppable,
@Nullable CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
@@ -245,7 +246,7 @@ public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializabl
}
@Override
- public Map<String, SerializedValue<Object>> getAccumulatorsSerialized() {
+ public Map<String, SerializedValue<OptionalFailure<Object>>> getAccumulatorsSerialized() {
return serializedUserAccumulators;
}
@@ -312,7 +313,8 @@ public class ArchivedExecutionGraph implements AccessExecutionGraph, Serializabl
archivedTasks.put(task.getJobVertexId(), archivedTask);
}
- final Map<String, SerializedValue<Object>> serializedUserAccumulators = executionGraph.getAccumulatorsSerialized();
+ final Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators =
+ executionGraph.getAccumulatorsSerialized();
final long[] timestamps = new long[JobStatus.values().length];
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 2fb831a..1af4fc9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -53,6 +53,7 @@ import org.apache.flink.runtime.taskmanager.TaskManagerLocation;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.OptionalFailure;
import org.slf4j.Logger;
@@ -69,6 +70,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.stream.Collectors;
import static org.apache.flink.runtime.execution.ExecutionState.CANCELED;
import static org.apache.flink.runtime.execution.ExecutionState.CANCELING;
@@ -1366,7 +1368,13 @@ public class Execution implements AccessExecution, Archiveable<ArchivedExecution
@Override
public StringifiedAccumulatorResult[] getUserAccumulatorsStringified() {
- return StringifiedAccumulatorResult.stringifyAccumulatorResults(userAccumulators);
+ Map<String, OptionalFailure<Accumulator<?, ?>>> accumulators =
+ userAccumulators == null ?
+ null :
+ userAccumulators.entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, entry -> OptionalFailure.of(entry.getValue())));
+ return StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulators);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 590fb07..22c11ef 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -24,7 +24,6 @@ import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.accumulators.AccumulatorHelper;
-import org.apache.flink.api.common.accumulators.FailedAccumulatorSerialization;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.JobException;
@@ -69,6 +68,7 @@ import org.apache.flink.runtime.taskmanager.TaskExecutionState;
import org.apache.flink.types.Either;
import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedThrowable;
import org.apache.flink.util.SerializedValue;
@@ -103,6 +103,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLongFieldUpdater;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.stream.Collectors;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -749,9 +750,9 @@ public class ExecutionGraph implements AccessExecutionGraph {
* Merges all accumulator results from the tasks previously executed in the Executions.
* @return The accumulator map
*/
- public Map<String, Accumulator<?, ?>> aggregateUserAccumulators() {
+ public Map<String, OptionalFailure<Accumulator<?, ?>>> aggregateUserAccumulators() {
- Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<>();
+ Map<String, OptionalFailure<Accumulator<?, ?>>> userAccumulators = new HashMap<>();
for (ExecutionVertex vertex : getAllExecutionVertices()) {
Map<String, Accumulator<?, ?>> next = vertex.getCurrentExecutionAttempt().getUserAccumulators();
@@ -768,28 +769,29 @@ public class ExecutionGraph implements AccessExecutionGraph {
* @return The accumulator map with serialized accumulator values.
*/
@Override
- public Map<String, SerializedValue<Object>> getAccumulatorsSerialized() {
-
- Map<String, Accumulator<?, ?>> accumulatorMap = aggregateUserAccumulators();
-
- Map<String, SerializedValue<Object>> result = new HashMap<>(accumulatorMap.size());
- for (Map.Entry<String, Accumulator<?, ?>> entry : accumulatorMap.entrySet()) {
+ public Map<String, SerializedValue<OptionalFailure<Object>>> getAccumulatorsSerialized() {
+ return aggregateUserAccumulators()
+ .entrySet()
+ .stream()
+ .collect(Collectors.toMap(
+ Map.Entry::getKey,
+ entry -> serializeAccumulator(entry.getKey(), entry.getValue())));
+ }
+ private static SerializedValue<OptionalFailure<Object>> serializeAccumulator(String name, OptionalFailure<Accumulator<?, ?>> accumulator) {
+ try {
+ if (accumulator.isFailure()) {
+ return new SerializedValue<>(OptionalFailure.ofFailure(accumulator.getFailureCause()));
+ }
+ return new SerializedValue<>(OptionalFailure.of(accumulator.getUnchecked().getLocalValue()));
+ } catch (IOException ioe) {
+ LOG.error("Could not serialize accumulator " + name + '.', ioe);
try {
- final SerializedValue<Object> serializedValue = new SerializedValue<>(entry.getValue().getLocalValue());
- result.put(entry.getKey(), serializedValue);
- } catch (IOException ioe) {
- LOG.error("Could not serialize accumulator " + entry.getKey() + '.', ioe);
-
- try {
- result.put(entry.getKey(), new SerializedValue<>(new FailedAccumulatorSerialization(ioe)));
- } catch (IOException e) {
- throw new RuntimeException("It should never happen that we cannot serialize the accumulator serialization exception.", e);
- }
+ return new SerializedValue<>(OptionalFailure.ofFailure(ioe));
+ } catch (IOException e) {
+ throw new RuntimeException("It should never happen that we cannot serialize the accumulator serialization exception.", e);
}
}
-
- return result;
}
/**
@@ -798,7 +800,7 @@ public class ExecutionGraph implements AccessExecutionGraph {
*/
@Override
public StringifiedAccumulatorResult[] getAccumulatorResultsStringified() {
- Map<String, Accumulator<?, ?>> accumulatorMap = aggregateUserAccumulators();
+ Map<String, OptionalFailure<Accumulator<?, ?>>> accumulatorMap = aggregateUserAccumulators();
return StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index 6e578fa..e5b7aa5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -48,6 +48,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider;
import org.apache.flink.runtime.state.KeyGroupRangeAssignment;
import org.apache.flink.types.Either;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;
@@ -597,7 +598,7 @@ public class ExecutionJobVertex implements AccessExecutionJobVertex, Archiveable
// --------------------------------------------------------------------------------------------
public StringifiedAccumulatorResult[] getAggregatedUserAccumulatorsStringified() {
- Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<String, Accumulator<?, ?>>();
+ Map<String, OptionalFailure<Accumulator<?, ?>>> userAccumulators = new HashMap<>();
for (ExecutionVertex vertex : taskVertices) {
Map<String, Accumulator<?, ?>> next = vertex.getCurrentExecutionAttempt().getUserAccumulators();
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java
index 28fbc30..7688404 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobResult.java
@@ -27,6 +27,7 @@ import org.apache.flink.runtime.executiongraph.AccessExecutionGraph;
import org.apache.flink.runtime.executiongraph.ErrorInfo;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedThrowable;
import org.apache.flink.util.SerializedValue;
@@ -53,7 +54,7 @@ public class JobResult implements Serializable {
private final JobID jobId;
- private final Map<String, SerializedValue<Object>> accumulatorResults;
+ private final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults;
private final long netRuntime;
@@ -63,7 +64,7 @@ public class JobResult implements Serializable {
private JobResult(
final JobID jobId,
- final Map<String, SerializedValue<Object>> accumulatorResults,
+ final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults,
final long netRuntime,
@Nullable final SerializedThrowable serializedThrowable) {
@@ -86,7 +87,7 @@ public class JobResult implements Serializable {
return jobId;
}
- public Map<String, SerializedValue<Object>> getAccumulatorResults() {
+ public Map<String, SerializedValue<OptionalFailure<Object>>> getAccumulatorResults() {
return accumulatorResults;
}
@@ -133,7 +134,7 @@ public class JobResult implements Serializable {
private JobID jobId;
- private Map<String, SerializedValue<Object>> accumulatorResults;
+ private Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults;
private long netRuntime = -1;
@@ -144,7 +145,7 @@ public class JobResult implements Serializable {
return this;
}
- public Builder accumulatorResults(final Map<String, SerializedValue<Object>> accumulatorResults) {
+ public Builder accumulatorResults(final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults) {
this.accumulatorResults = accumulatorResults;
return this;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java
index 0fe9201..964aee3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/handler/job/JobAccumulatorsHandler.java
@@ -31,6 +31,7 @@ import org.apache.flink.runtime.rest.messages.JobAccumulatorsMessageParameters;
import org.apache.flink.runtime.rest.messages.MessageHeaders;
import org.apache.flink.runtime.webmonitor.RestfulGateway;
import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedValue;
import java.util.ArrayList;
@@ -87,7 +88,7 @@ public class JobAccumulatorsHandler extends AbstractExecutionGraphHandler<JobAcc
}
if (includeSerializedValue) {
- Map<String, SerializedValue<Object>> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized();
+ Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized();
accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators);
} else {
accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap());
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java
index 2262120..192eaa0 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/JobAccumulatorsInfo.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.rest.messages;
import org.apache.flink.runtime.rest.handler.job.JobAccumulatorsHandler;
import org.apache.flink.runtime.rest.messages.json.SerializedValueDeserializer;
import org.apache.flink.runtime.rest.messages.json.SerializedValueSerializer;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;
@@ -50,13 +51,13 @@ public class JobAccumulatorsInfo implements ResponseBody {
@JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS)
@JsonSerialize(contentUsing = SerializedValueSerializer.class)
- private Map<String, SerializedValue<Object>> serializedUserAccumulators;
+ private Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators;
@JsonCreator
public JobAccumulatorsInfo(
@JsonProperty(FIELD_NAME_JOB_ACCUMULATORS) List<JobAccumulator> jobAccumulators,
@JsonProperty(FIELD_NAME_USER_TASK_ACCUMULATORS) List<UserTaskAccumulator> userAccumulators,
- @JsonDeserialize(contentUsing = SerializedValueDeserializer.class) @JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS) Map<String, SerializedValue<Object>> serializedUserAccumulators) {
+ @JsonDeserialize(contentUsing = SerializedValueDeserializer.class) @JsonProperty(FIELD_NAME_SERIALIZED_USER_TASK_ACCUMULATORS) Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators) {
this.jobAccumulators = Preconditions.checkNotNull(jobAccumulators);
this.userAccumulators = Preconditions.checkNotNull(userAccumulators);
this.serializedUserAccumulators = Preconditions.checkNotNull(serializedUserAccumulators);
@@ -73,7 +74,7 @@ public class JobAccumulatorsInfo implements ResponseBody {
}
@JsonIgnore
- public Map<String, SerializedValue<Object>> getSerializedUserAccumulators() {
+ public Map<String, SerializedValue<OptionalFailure<Object>>> getSerializedUserAccumulators() {
return serializedUserAccumulators;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java
index 52bb43c..e568f47 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultDeserializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.runtime.rest.messages.json;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobmaster.JobResult;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedThrowable;
import org.apache.flink.util.SerializedValue;
@@ -69,7 +70,7 @@ public class JobResultDeserializer extends StdDeserializer<JobResult> {
JobID jobId = null;
long netRuntime = -1;
SerializedThrowable serializedThrowable = null;
- Map<String, SerializedValue<Object>> accumulatorResults = null;
+ Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults = null;
while (true) {
final JsonToken jsonToken = p.nextToken();
@@ -117,11 +118,11 @@ public class JobResultDeserializer extends StdDeserializer<JobResult> {
}
@SuppressWarnings("unchecked")
- private Map<String, SerializedValue<Object>> parseAccumulatorResults(
+ private Map<String, SerializedValue<OptionalFailure<Object>>> parseAccumulatorResults(
final JsonParser p,
final DeserializationContext ctxt) throws IOException {
- final Map<String, SerializedValue<Object>> accumulatorResults = new HashMap<>();
+ final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults = new HashMap<>();
while (true) {
final JsonToken jsonToken = p.nextToken();
assertNotEndOfInput(p, jsonToken);
@@ -132,7 +133,7 @@ public class JobResultDeserializer extends StdDeserializer<JobResult> {
p.nextValue();
accumulatorResults.put(
accumulatorName,
- (SerializedValue<Object>) serializedValueDeserializer.deserialize(p, ctxt));
+ (SerializedValue<OptionalFailure<Object>>) serializedValueDeserializer.deserialize(p, ctxt));
}
return accumulatorResults;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java
index a53716a..694fa2f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/rest/messages/json/JobResultSerializer.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.rest.messages.json;
import org.apache.flink.runtime.jobmaster.JobResult;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedThrowable;
import org.apache.flink.util.SerializedValue;
@@ -77,10 +78,10 @@ public class JobResultSerializer extends StdSerializer<JobResult> {
gen.writeFieldName(FIELD_NAME_ACCUMULATOR_RESULTS);
gen.writeStartObject();
- final Map<String, SerializedValue<Object>> accumulatorResults = result.getAccumulatorResults();
- for (final Map.Entry<String, SerializedValue<Object>> nameValue : accumulatorResults.entrySet()) {
+ final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults = result.getAccumulatorResults();
+ for (final Map.Entry<String, SerializedValue<OptionalFailure<Object>>> nameValue : accumulatorResults.entrySet()) {
final String name = nameValue.getKey();
- final SerializedValue<Object> value = nameValue.getValue();
+ final SerializedValue<OptionalFailure<Object>> value = nameValue.getValue();
gen.writeFieldName(name);
serializedValueSerializer.serialize(value, gen, provider);
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala
index 107ba82..9ed01aa 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/accumulators/AccumulatorMessages.scala
@@ -20,7 +20,7 @@ package org.apache.flink.runtime.messages.accumulators
import org.apache.flink.api.common.JobID
import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult
-import org.apache.flink.util.SerializedValue
+import org.apache.flink.util.{OptionalFailure, SerializedValue}
/**
* Base trait of all accumulator messages
@@ -62,8 +62,9 @@ case class RequestAccumulatorResultsStringified(jobID: JobID)
* @param jobID Job Id of the job that the accumulator belongs to
* @param result The accumulator result values, in serialized form.
*/
-case class AccumulatorResultsFound(jobID: JobID,
- result: java.util.Map[String, SerializedValue[Object]])
+case class AccumulatorResultsFound(
+ jobID: JobID,
+ result: java.util.Map[String, SerializedValue[OptionalFailure[Object]]])
extends AccumulatorResultsResponse
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java
index 65c4b7a..a06bdfc 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResultTest.java
@@ -22,6 +22,8 @@ import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.accumulators.IntCounter;
import org.apache.flink.api.common.accumulators.SimpleAccumulator;
import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.OptionalFailure;
import org.junit.Test;
@@ -31,6 +33,7 @@ import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Tests for the {@link StringifiedAccumulatorResult}.
@@ -63,8 +66,8 @@ public class StringifiedAccumulatorResultTest {
final int targetValue = 314159;
final IntCounter acc = new IntCounter();
acc.add(targetValue);
- final Map<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
- accumulatorMap.put(name, acc);
+ final Map<String, OptionalFailure<Accumulator<?, ?>>> accumulatorMap = new HashMap<>();
+ accumulatorMap.put(name, OptionalFailure.of(acc));
final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
@@ -80,8 +83,8 @@ public class StringifiedAccumulatorResultTest {
public void stringifyingResultsShouldReportNullLocalValueAsNonnullValueString() {
final String name = "a";
final NullBearingAccumulator acc = new NullBearingAccumulator();
- final Map<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
- accumulatorMap.put(name, acc);
+ final Map<String, OptionalFailure<Accumulator<?, ?>>> accumulatorMap = new HashMap<>();
+ accumulatorMap.put(name, OptionalFailure.of(acc));
final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
@@ -97,7 +100,7 @@ public class StringifiedAccumulatorResultTest {
@Test
public void stringifyingResultsShouldReportNullAccumulatorWithNonnullValueAndTypeString() {
final String name = "a";
- final Map<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
+ final Map<String, OptionalFailure<Accumulator<?, ?>>> accumulatorMap = new HashMap<>();
accumulatorMap.put(name, null);
final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
@@ -111,6 +114,23 @@ public class StringifiedAccumulatorResultTest {
assertEquals("null", firstResult.getValue());
}
+ @Test
+ public void stringifyingFailureResults() {
+ final String name = "a";
+ final Map<String, OptionalFailure<Accumulator<?, ?>>> accumulatorMap = new HashMap<>();
+ accumulatorMap.put(name, OptionalFailure.ofFailure(new FlinkRuntimeException("Test")));
+
+ final StringifiedAccumulatorResult[] results = StringifiedAccumulatorResult.stringifyAccumulatorResults(accumulatorMap);
+
+ assertEquals(1, results.length);
+
+ // Note the use of String values with content of "null" rather than null values
+ final StringifiedAccumulatorResult firstResult = results[0];
+ assertEquals(name, firstResult.getName());
+ assertEquals("null", firstResult.getType());
+ assertTrue(firstResult.getValue().startsWith("org.apache.flink.util.FlinkRuntimeException: Test"));
+ }
+
private static class NullBearingAccumulator implements SimpleAccumulator<Serializable> {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java
index 38447e2..c16acec 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/client/SerializedJobExecutionResultTest.java
@@ -21,19 +21,28 @@ package org.apache.flink.runtime.client;
import org.apache.flink.api.common.JobExecutionResult;
import org.apache.flink.api.common.JobID;
import org.apache.flink.core.testutils.CommonTestUtils;
+import org.apache.flink.runtime.operators.testutils.ExpectedTestException;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedValue;
+import org.apache.flink.util.TestLogger;
+
import org.junit.Test;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Tests for the SerializedJobExecutionResult
*/
-public class SerializedJobExecutionResultTest {
+public class SerializedJobExecutionResultTest extends TestLogger {
@Test
public void testSerialization() throws Exception {
@@ -42,9 +51,10 @@ public class SerializedJobExecutionResultTest {
JobID origJobId = new JobID();
long origTime = 65927436589267L;
- Map<String, SerializedValue<Object>> origMap = new HashMap<String, SerializedValue<Object>>();
- origMap.put("name1", new SerializedValue<Object>(723L));
- origMap.put("name2", new SerializedValue<Object>("peter"));
+ Map<String, SerializedValue<OptionalFailure<Object>>> origMap = new HashMap<>();
+ origMap.put("name1", new SerializedValue<>(OptionalFailure.of(723L)));
+ origMap.put("name2", new SerializedValue<>(OptionalFailure.of("peter")));
+ origMap.put("name3", new SerializedValue<>(OptionalFailure.ofFailure(new ExpectedTestException())));
SerializedJobExecutionResult result = new SerializedJobExecutionResult(origJobId, origTime, origMap);
@@ -67,11 +77,29 @@ public class SerializedJobExecutionResultTest {
assertEquals(origTime, jResultCopied.getNetRuntime());
assertEquals(origTime, jResultCopied.getNetRuntime(TimeUnit.MILLISECONDS));
- for (Map.Entry<String, SerializedValue<Object>> entry : origMap.entrySet()) {
+ for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> entry : origMap.entrySet()) {
String name = entry.getKey();
- Object value = entry.getValue().deserializeValue(classloader);
- assertEquals(value, jResult.getAccumulatorResult(name));
- assertEquals(value, jResultCopied.getAccumulatorResult(name));
+ OptionalFailure<Object> value = entry.getValue().deserializeValue(classloader);
+ if (value.isFailure()) {
+ try {
+ jResult.getAccumulatorResult(name);
+ fail("expected failure");
+ }
+ catch (FlinkRuntimeException ex) {
+ assertTrue(ExceptionUtils.findThrowable(ex, ExpectedTestException.class).isPresent());
+ }
+ try {
+ jResultCopied.getAccumulatorResult(name);
+ fail("expected failure");
+ }
+ catch (FlinkRuntimeException ex) {
+ assertTrue(ExceptionUtils.findThrowable(ex, ExpectedTestException.class).isPresent());
+ }
+ }
+ else {
+ assertEquals(value.get(), jResult.getAccumulatorResult(name));
+ assertEquals(value.get(), jResultCopied.getAccumulatorResult(name));
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
index f15dca1..7c2a02c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ArchivedExecutionGraphTest.java
@@ -44,6 +44,7 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedValue;
import org.apache.flink.util.TestLogger;
@@ -58,6 +59,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
@@ -310,11 +312,13 @@ public class ArchivedExecutionGraphTest extends TestLogger {
}
}
- private static void compareSerializedAccumulators(Map<String, SerializedValue<Object>> runtimeAccs, Map<String, SerializedValue<Object>> archivedAccs) throws IOException, ClassNotFoundException {
+ private static void compareSerializedAccumulators(
+ Map<String, SerializedValue<OptionalFailure<Object>>> runtimeAccs,
+ Map<String, SerializedValue<OptionalFailure<Object>>> archivedAccs) throws IOException, ClassNotFoundException {
assertEquals(runtimeAccs.size(), archivedAccs.size());
- for (Map.Entry<String, SerializedValue<Object>> runtimeAcc : runtimeAccs.entrySet()) {
- long runtimeUserAcc = (long) runtimeAcc.getValue().deserializeValue(ClassLoader.getSystemClassLoader());
- long archivedUserAcc = (long) archivedAccs.get(runtimeAcc.getKey()).deserializeValue(ClassLoader.getSystemClassLoader());
+ for (Entry<String, SerializedValue<OptionalFailure<Object>>> runtimeAcc : runtimeAccs.entrySet()) {
+ long runtimeUserAcc = (long) runtimeAcc.getValue().deserializeValue(ClassLoader.getSystemClassLoader()).getUnchecked();
+ long archivedUserAcc = (long) archivedAccs.get(runtimeAcc.getKey()).deserializeValue(ClassLoader.getSystemClassLoader()).getUnchecked();
assertEquals(runtimeUserAcc, archivedUserAcc);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java
index 318541d..df4ff04 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/SubtaskExecutionAttemptAccumulatorsHandlerTest.java
@@ -36,6 +36,8 @@ import org.apache.flink.runtime.rest.messages.job.SubtaskExecutionAttemptAccumul
import org.apache.flink.runtime.rest.messages.job.SubtaskExecutionAttemptAccumulatorsInfo;
import org.apache.flink.runtime.rest.messages.job.UserAccumulator;
import org.apache.flink.runtime.testingUtils.TestingUtils;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.TestLogger;
import org.junit.Test;
@@ -76,9 +78,10 @@ public class SubtaskExecutionAttemptAccumulatorsHandlerTest extends TestLogger {
new SubtaskAttemptMessageParameters()
);
- final Map<String, Accumulator<?, ?>> userAccumulators = new HashMap<>(2);
- userAccumulators.put("IntCounter", new IntCounter(10));
- userAccumulators.put("LongCounter", new LongCounter(100L));
+ final Map<String, OptionalFailure<Accumulator<?, ?>>> userAccumulators = new HashMap<>(3);
+ userAccumulators.put("IntCounter", OptionalFailure.of(new IntCounter(10)));
+ userAccumulators.put("LongCounter", OptionalFailure.of(new LongCounter(100L)));
+ userAccumulators.put("Failure", OptionalFailure.ofFailure(new FlinkRuntimeException("Test")));
// Instance the expected result.
final StringifiedAccumulatorResult[] accumulatorResults =
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java
index ee7ceda..689e1b3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/utils/ArchivedExecutionGraphBuilder.java
@@ -26,6 +26,7 @@ import org.apache.flink.runtime.executiongraph.ArchivedExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.ErrorInfo;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.Preconditions;
import org.apache.flink.util.SerializedValue;
@@ -53,7 +54,7 @@ public class ArchivedExecutionGraphBuilder {
private StringifiedAccumulatorResult[] archivedUserAccumulators;
private ArchivedExecutionConfig archivedExecutionConfig;
private boolean isStoppable;
- private Map<String, SerializedValue<Object>> serializedUserAccumulators;
+ private Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators;
public ArchivedExecutionGraphBuilder setJobID(JobID jobID) {
this.jobID = jobID;
@@ -111,7 +112,7 @@ public class ArchivedExecutionGraphBuilder {
return this;
}
- public ArchivedExecutionGraphBuilder setSerializedUserAccumulators(Map<String, SerializedValue<Object>> serializedUserAccumulators) {
+ public ArchivedExecutionGraphBuilder setSerializedUserAccumulators(Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserAccumulators) {
this.serializedUserAccumulators = serializedUserAccumulators;
return this;
}
@@ -134,7 +135,7 @@ public class ArchivedExecutionGraphBuilder {
failureCause,
jsonPlan != null ? jsonPlan : "{\"jobid\":\"" + jobID + "\", \"name\":\"" + jobName + "\", \"nodes\":[]}",
archivedUserAccumulators != null ? archivedUserAccumulators : new StringifiedAccumulatorResult[0],
- serializedUserAccumulators != null ? serializedUserAccumulators : Collections.<String, SerializedValue<Object>>emptyMap(),
+ serializedUserAccumulators != null ? serializedUserAccumulators : Collections.emptyMap(),
archivedExecutionConfig != null ? archivedExecutionConfig : new ArchivedExecutionConfigBuilder().build(),
isStoppable,
null,
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java
index d7dd7eb..9534d2b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/job/JobExecutionResultResponseBodyTest.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.rest.messages.job;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobmaster.JobResult;
import org.apache.flink.runtime.rest.messages.RestResponseMarshallingTestBase;
+import org.apache.flink.util.OptionalFailure;
import org.apache.flink.util.SerializedThrowable;
import org.apache.flink.util.SerializedValue;
@@ -54,7 +55,7 @@ public class JobExecutionResultResponseBodyTest
private static final String TEST_ACCUMULATOR_NAME = "test";
- private static final Map<String, SerializedValue<Object>> TEST_ACCUMULATORS = Collections.singletonMap(
+ private static final Map<String, SerializedValue<OptionalFailure<Object>>> TEST_ACCUMULATORS = Collections.singletonMap(
TEST_ACCUMULATOR_NAME,
SerializedValue.fromBytes(TEST_ACCUMULATOR_VALUE));
http://git-wip-us.apache.org/repos/asf/flink/blob/a6a7623d/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
index ce32316..c8529a9 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingJobManagerMessages.scala
@@ -30,6 +30,7 @@ import org.apache.flink.runtime.instance.ActorGateway
import org.apache.flink.runtime.jobgraph.JobStatus
import org.apache.flink.runtime.messages.RequiresLeaderSessionID
import org.apache.flink.runtime.messages.checkpoint.AbstractCheckpointMessage
+import org.apache.flink.util.OptionalFailure
object TestingJobManagerMessages {
@@ -108,7 +109,7 @@ object TestingJobManagerMessages {
* Reports updated accumulators back to the listener.
*/
case class UpdatedAccumulators(jobID: JobID,
- userAccumulators: Map[String, Accumulator[_,_]])
+ userAccumulators: Map[String, OptionalFailure[Accumulator[_,_]]])
/** Notifies the sender when the [[TestingJobManager]] has been elected as the leader
*