You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/04/10 14:04:45 UTC
[3/4] flink git commit: [FLINK-1535] [core] Rework accumulators
[FLINK-1535] [core] Rework accumulators
- Accumulators rely consistently on java serialization
- Fix accumulator transport to support custom classes
- Fix collect() accumulator to use correct class loader
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6774054a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6774054a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6774054a
Branch: refs/heads/master
Commit: 6774054a7bf1ff6947ec774d74ef6f3941c58b27
Parents: 201bea3
Author: Stephan Ewen <se...@apache.org>
Authored: Tue Apr 7 17:30:19 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Apr 10 12:58:39 2015 +0200
----------------------------------------------------------------------
.../org/apache/flink/client/LocalExecutor.java | 7 +-
.../org/apache/flink/client/program/Client.java | 15 +-
.../flink/api/common/JobExecutionResult.java | 6 +-
.../api/common/accumulators/Accumulator.java | 34 +--
.../common/accumulators/AccumulatorHelper.java | 25 +-
.../api/common/accumulators/DoubleCounter.java | 24 +-
.../api/common/accumulators/Histogram.java | 33 +--
.../api/common/accumulators/IntCounter.java | 23 +-
.../common/accumulators/ListAccumulator.java | 50 +---
.../api/common/accumulators/LongCounter.java | 26 +-
.../accumulators/SerializedListAccumulator.java | 103 ++++++++
.../common/accumulators/SimpleAccumulator.java | 1 -
.../util/AbstractRuntimeUDFContext.java | 13 +-
.../relational/EmptyFieldsCountAccumulator.java | 33 +--
.../java/org/apache/flink/api/java/DataSet.java | 32 ++-
.../java/org/apache/flink/api/java/Utils.java | 6 +-
.../runtime/accumulators/AccumulatorEvent.java | 159 +-----------
.../StringifiedAccumulatorResult.java | 49 ++++
.../client/SerializedJobExecutionResult.java | 85 +++++++
.../runtime/execution/RuntimeEnvironment.java | 1 -
.../runtime/executiongraph/ExecutionGraph.java | 2 +
.../accumulators/AccumulatorManager.java | 63 ++++-
.../accumulators/JobAccumulators.java | 1 -
.../jobmanager/web/JobManagerInfoServlet.java | 249 +++++++++----------
.../flink/runtime/util/SerializedValue.java | 84 +++++++
.../apache/flink/runtime/client/JobClient.scala | 31 +--
.../flink/runtime/jobmanager/JobManager.scala | 101 ++++++--
.../runtime/messages/JobManagerMessages.scala | 51 +---
.../accumulators/AccumulatorMessages.scala | 108 ++++++++
.../SerializedJobExecutionResultTest.java | 99 ++++++++
.../operators/testutils/MockEnvironment.java | 6 +
.../flink/runtime/util/SerializedValueTest.java | 72 ++++++
.../runtime/jobmanager/JobManagerITCase.scala | 6 +-
.../runtime/jobmanager/RecoveryITCase.scala | 6 +-
.../org/apache/flink/api/scala/DataSet.scala | 32 ++-
.../flink/streaming/util/ClusterUtil.java | 11 +-
.../streaming/util/TestStreamEnvironment.java | 10 +-
.../flink/test/util/RecordAPITestBase.java | 8 +-
.../apache/flink/test/util/TestEnvironment.java | 9 +-
.../test/accumulators/AccumulatorITCase.java | 36 +--
.../JobSubmissionFailsITCase.java | 10 +-
.../PackagedProgramEndToEndITCase.java | 11 +-
.../wordcount/WordCountAccumulators.java | 31 +--
.../src/test/resources/log4j-test.properties | 2 +-
.../api/scala/actions/CountCollectITCase.scala | 4 +-
.../api/scala/operators/SumMinMaxITCase.scala | 6 +-
.../jobmanager/JobManagerFailsITCase.scala | 2 +-
.../taskmanager/TaskManagerFailsITCase.scala | 2 +-
48 files changed, 1099 insertions(+), 679 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java b/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
index b1705df..22e17d0 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/LocalExecutor.java
@@ -30,6 +30,7 @@ import org.apache.flink.api.common.Program;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.client.JobClient;
+import org.apache.flink.runtime.client.SerializedJobExecutionResult;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.optimizer.DataStatistics;
@@ -182,8 +183,10 @@ public class LocalExecutor extends PlanExecutor {
ActorRef jobClient = flink.getJobClient();
- return JobClient.submitJobAndWait(jobGraph, printStatusDuringExecution,
- jobClient, flink.timeout());
+ SerializedJobExecutionResult result =
+ JobClient.submitJobAndWait(jobGraph, printStatusDuringExecution, jobClient, flink.timeout());
+
+ return result.toJobExecutionResult(ClassLoader.getSystemClassLoader());
}
finally {
if (shutDownAtEnd) {
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
----------------------------------------------------------------------
diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
index 2bbbe91..80bdcb8 100644
--- a/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
+++ b/flink-clients/src/main/java/org/apache/flink/client/program/Client.java
@@ -47,6 +47,7 @@ import org.apache.flink.configuration.GlobalConfiguration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.client.JobClient;
import org.apache.flink.runtime.client.JobExecutionException;
+import org.apache.flink.runtime.client.SerializedJobExecutionResult;
import org.apache.flink.runtime.jobgraph.JobGraph;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -74,7 +75,7 @@ public class Client {
/**
* If != -1, this field specifies the total number of available slots on the cluster
- * conntected to the client.
+ * connected to the client.
*/
private int maxSlots = -1;
@@ -83,7 +84,7 @@ public class Client {
*/
private JobID lastJobId = null;
- private ClassLoader userCodeClassLoader; // TODO: use userCodeClassloader to deserialize accumulator results.
+ private ClassLoader userCodeClassLoader;
// ------------------------------------------------------------------------
// Construction
@@ -343,7 +344,15 @@ public class Client {
try{
if (wait) {
- return JobClient.submitJobAndWait(jobGraph, printStatusDuringExecution, client, timeout);
+ SerializedJobExecutionResult result =
+ JobClient.submitJobAndWait(jobGraph, printStatusDuringExecution, client, timeout);
+ try {
+ return result.toJobExecutionResult(this.userCodeClassLoader);
+ }
+ catch (Exception e) {
+ throw new ProgramInvocationException(
+ "Failed to deserialize the accumulator result after the job execution", e);
+ }
}
else {
JobClient.submitJobDetached(jobGraph, client, timeout);
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
index 68506ae..62848f7 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/JobExecutionResult.java
@@ -27,12 +27,13 @@ import java.util.Map;
public class JobExecutionResult extends JobSubmissionResult {
private long netRuntime;
+
private Map<String, Object> accumulatorResults;
/**
* Creates a new JobExecutionResult.
*
- * @param jobID
+ * @param jobID The job's ID.
* @param netRuntime The net runtime of the job (excluding pre-flight phase like the optimizer)
* @param accumulators A map of all accumulators produced by the job.
*/
@@ -93,7 +94,4 @@ public class JobExecutionResult extends JobSubmissionResult {
}
return (Integer) result;
}
-
-
- // TODO Create convenience methods for the other shipped accumulator types
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Accumulator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Accumulator.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Accumulator.java
index f1aac4d..123d956 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Accumulator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Accumulator.java
@@ -16,20 +16,18 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.accumulators;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.io.Serializable;
-
/**
- * Interface for custom accumulator objects. Data are written to in a UDF and
- * merged by the system at the end of the job. The result can be read at the end
- * of the job from the calling client. Inspired by Hadoop/MapReduce counters.<br>
- * <br>
+ * Accumulators collect distributed statistics or aggregates in a from user functions
+ * and operators. Each parallel instance creates and updates its own accumulator object,
+ * and the different parallel instances of the accumulator are later merged.
+ * merged by the system at the end of the job. The result can be obtained from the
+ * result of a job execution, or from teh web runtime monitor.
+ *
+ * The accumulators are inspired by the Hadoop/MapReduce counters.
*
* The type added to the accumulator might differ from the type returned. This
* is the case e.g. for a set-accumulator: We add single objects, but the result
@@ -41,7 +39,7 @@ import java.io.Serializable;
* Type of the accumulator result as it will be reported to the
* client
*/
-public interface Accumulator<V, R extends Serializable> extends Serializable, Cloneable{
+public interface Accumulator<V, R extends Serializable> extends Serializable, Cloneable {
/**
* @param value
@@ -68,20 +66,10 @@ public interface Accumulator<V, R extends Serializable> extends Serializable, Cl
void merge(Accumulator<V, R> other);
/**
- * Serialization method of accumulators
- *
- * @param oos
- * @throws IOException
- */
- void write(ObjectOutputStream oos) throws IOException;
-
- /**
- * Deserialization method of accumulators
+ * Duplicates the accumulator. All subclasses need to properly implement
+ * cloning and cannot throw a {@link java.lang.CloneNotSupportedException}
*
- * @param ois
- * @throws IOException
+ * @return The duplicated accumulator.
*/
- void read(ObjectInputStream ois) throws IOException;
-
Accumulator<V, R> clone();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
index da871ef..9b0e019 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/AccumulatorHelper.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.accumulators;
import java.io.Serializable;
@@ -69,13 +68,27 @@ public class AccumulatorHelper {
* Compare both classes and throw {@link UnsupportedOperationException} if
* they differ
*/
+ @SuppressWarnings("rawtypes")
public static void compareAccumulatorTypes(Object name,
- @SuppressWarnings("rawtypes") Class<? extends Accumulator> first,
- @SuppressWarnings("rawtypes") Class<? extends Accumulator> second)
- throws UnsupportedOperationException {
+ Class<? extends Accumulator> first,
+ Class<? extends Accumulator> second)
+ throws UnsupportedOperationException
+ {
+ if (first == null || second == null) {
+ throw new NullPointerException();
+ }
+
if (first != second) {
- throw new UnsupportedOperationException("The accumulator object '" + name
- + "' was created with two different types: " + first + " and " + second);
+ if (!first.getName().equals(second.getName())) {
+ throw new UnsupportedOperationException("The accumulator object '" + name
+ + "' was created with two different types: " + first.getName() + " and " + second.getName());
+ } else {
+ // damn, name is the same, but different classloaders
+ throw new UnsupportedOperationException("The accumulator object '" + name
+ + "' was created with two different classes: " + first + " and " + second
+ + " Both have the same type (" + first.getName() + ") but different classloaders: "
+ + first.getClassLoader() + " and " + second.getClassLoader());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/DoubleCounter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/DoubleCounter.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/DoubleCounter.java
index 6f02650..85d187c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/DoubleCounter.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/DoubleCounter.java
@@ -16,13 +16,11 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.accumulators;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
+/**
+ * An accumulator that sums up {@code double} values.
+ */
public class DoubleCounter implements SimpleAccumulator<Double> {
private static final long serialVersionUID = 1L;
@@ -41,7 +39,7 @@ public class DoubleCounter implements SimpleAccumulator<Double> {
@Override
public void merge(Accumulator<Double, Double> other) {
- this.localValue += ((DoubleCounter) other).getLocalValue();
+ this.localValue += other.getLocalValue();
}
@Override
@@ -50,26 +48,14 @@ public class DoubleCounter implements SimpleAccumulator<Double> {
}
@Override
- public void write(ObjectOutputStream out) throws IOException {
- out.writeDouble(localValue);
- }
-
- @Override
- public void read(ObjectInputStream in) throws IOException {
- this.localValue = in.readDouble();
- }
-
- @Override
public Accumulator<Double, Double> clone() {
DoubleCounter result = new DoubleCounter();
result.localValue = localValue;
-
return result;
}
@Override
public String toString() {
- return "DoubleCounter object. Local value: " + this.localValue;
+ return "DoubleCounter " + this.localValue;
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Histogram.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Histogram.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Histogram.java
index e204b81..f5e959a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Histogram.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/Histogram.java
@@ -16,22 +16,18 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.accumulators;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.util.Map;
import java.util.TreeMap;
/**
- * Histogram for discrete-data. Let's you populate a histogram distributedly.
+ * Histogram accumulator, which builds a histogram in a distributed manner.
* Implemented as a Integer->Integer TreeMap, so that the entries are sorted
* according to the values.
*
- * Could be extended to continuous values later, but then we need to dynamically
- * decide about the bin size in an online algorithm (or ask the user)
+ * This class does not extend to continuous values later, because it makes no
+ * attempt to put the data in bins.
*/
public class Histogram implements Accumulator<Integer, TreeMap<Integer, Integer>> {
@@ -54,8 +50,7 @@ public class Histogram implements Accumulator<Integer, TreeMap<Integer, Integer>
@Override
public void merge(Accumulator<Integer, TreeMap<Integer, Integer>> other) {
// Merge the values into this map
- for (Map.Entry<Integer, Integer> entryFromOther : ((Histogram) other).getLocalValue()
- .entrySet()) {
+ for (Map.Entry<Integer, Integer> entryFromOther : other.getLocalValue().entrySet()) {
Integer ownValue = this.treeMap.get(entryFromOther.getKey());
if (ownValue == null) {
this.treeMap.put(entryFromOther.getKey(), entryFromOther.getValue());
@@ -76,29 +71,9 @@ public class Histogram implements Accumulator<Integer, TreeMap<Integer, Integer>
}
@Override
- public void write(ObjectOutputStream out) throws IOException {
- out.writeInt(treeMap.size());
- for (Map.Entry<Integer, Integer> entry : treeMap.entrySet()) {
- out.writeInt(entry.getKey());
- out.writeInt(entry.getValue());
- }
- }
-
- @Override
- public void read(ObjectInputStream in) throws IOException {
- int size = in.readInt();
- for (int i = 0; i < size; ++i) {
- treeMap.put(in.readInt(), in.readInt());
- }
- }
-
- @Override
public Accumulator<Integer, TreeMap<Integer, Integer>> clone() {
Histogram result = new Histogram();
-
result.treeMap = new TreeMap<Integer, Integer>(treeMap);
-
return result;
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/IntCounter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/IntCounter.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/IntCounter.java
index f1c45a1..c88c73d 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/IntCounter.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/IntCounter.java
@@ -19,10 +19,9 @@
package org.apache.flink.api.common.accumulators;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
+/**
+ * An accumulator that sums up {@code Integer} values.
+ */
public class IntCounter implements SimpleAccumulator<Integer> {
private static final long serialVersionUID = 1L;
@@ -41,7 +40,7 @@ public class IntCounter implements SimpleAccumulator<Integer> {
@Override
public void merge(Accumulator<Integer, Integer> other) {
- this.localValue += ((IntCounter) other).getLocalValue();
+ this.localValue += other.getLocalValue();
}
@Override
@@ -50,26 +49,14 @@ public class IntCounter implements SimpleAccumulator<Integer> {
}
@Override
- public void write(ObjectOutputStream out) throws IOException {
- out.writeInt(localValue);
- }
-
- @Override
- public void read(ObjectInputStream in) throws IOException {
- localValue = in.readInt();
- }
-
- @Override
public Accumulator<Integer, Integer> clone() {
IntCounter result = new IntCounter();
result.localValue = localValue;
-
return result;
}
@Override
public String toString() {
- return "IntCounter object. Local value: " + this.localValue;
+ return "IntCounter " + this.localValue;
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/ListAccumulator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/ListAccumulator.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/ListAccumulator.java
index 5a973ac..3af785c 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/ListAccumulator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/ListAccumulator.java
@@ -16,42 +16,29 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.accumulators;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
import java.util.ArrayList;
-import org.apache.commons.lang3.SerializationUtils;
-
/**
- * * This accumulator stores a collection of objects which are immediately serialized to cope with object reuse.
- * * When the objects are requested again, they are deserialized.
+ * This accumulator stores a collection of objects.
+ *
* @param <T> The type of the accumulated objects
*/
public class ListAccumulator<T> implements Accumulator<T, ArrayList<T>> {
private static final long serialVersionUID = 1L;
- private ArrayList<byte[]> localValue = new ArrayList<byte[]>();
+ private ArrayList<T> localValue = new ArrayList<T>();
@Override
public void add(T value) {
- byte[] byteArray = SerializationUtils.serialize((Serializable) value);
- localValue.add(byteArray);
+ localValue.add(value);
}
@Override
public ArrayList<T> getLocalValue() {
- ArrayList<T> arrList = new ArrayList<T>();
- for (byte[] byteArr : localValue) {
- T item = SerializationUtils.deserialize(byteArr);
- arrList.add(item);
- }
- return arrList;
+ return localValue;
}
@Override
@@ -61,37 +48,18 @@ public class ListAccumulator<T> implements Accumulator<T, ArrayList<T>> {
@Override
public void merge(Accumulator<T, ArrayList<T>> other) {
- localValue.addAll(((ListAccumulator<T>) other).localValue);
+ localValue.addAll(other.getLocalValue());
}
@Override
public Accumulator<T, ArrayList<T>> clone() {
ListAccumulator<T> newInstance = new ListAccumulator<T>();
- for (byte[] item : localValue) {
- newInstance.localValue.add(item.clone());
- }
+ newInstance.localValue = new ArrayList<T>(localValue);
return newInstance;
}
@Override
- public void write(ObjectOutputStream out) throws IOException {
- int numItems = localValue.size();
- out.writeInt(numItems);
- for (byte[] item : localValue) {
- out.writeInt(item.length);
- out.write(item);
- }
+ public String toString() {
+ return "List Accumulator " + localValue;
}
-
- @Override
- public void read(ObjectInputStream in) throws IOException {
- int numItems = in.readInt();
- for (int i = 0; i < numItems; i++) {
- int len = in.readInt();
- byte[] obj = new byte[len];
- in.read(obj);
- localValue.add(obj);
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/LongCounter.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/LongCounter.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/LongCounter.java
index dfa3526..70f2417 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/LongCounter.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/LongCounter.java
@@ -16,13 +16,11 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.accumulators;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-
+/**
+ * An accumulator that sums up {@code long} values.
+ */
public class LongCounter implements SimpleAccumulator<Long> {
private static final long serialVersionUID = 1L;
@@ -41,7 +39,7 @@ public class LongCounter implements SimpleAccumulator<Long> {
@Override
public void merge(Accumulator<Long, Long> other) {
- this.localValue += ((LongCounter)other).getLocalValue();
+ this.localValue += other.getLocalValue();
}
@Override
@@ -50,26 +48,14 @@ public class LongCounter implements SimpleAccumulator<Long> {
}
@Override
- public void write(ObjectOutputStream out) throws IOException {
- out.writeLong(this.localValue);
- }
-
- @Override
- public void read(ObjectInputStream in) throws IOException {
- this.localValue = in.readLong();
- }
-
- @Override
- public Accumulator<Long, Long> clone() {
+ public LongCounter clone() {
LongCounter result = new LongCounter();
result.localValue = localValue;
-
return result;
}
@Override
public String toString() {
- return "LongCounter object. Local value: " + this.localValue;
+ return "LongCounter " + this.localValue;
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SerializedListAccumulator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SerializedListAccumulator.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SerializedListAccumulator.java
new file mode 100644
index 0000000..4ab339b
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SerializedListAccumulator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.accumulators;
+
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This accumulator stores a collection of objects in serialized form, so that the stored objects
+ * are not affected by modifications to the original objects.
+ *
+ * Objects may be deserialized on demand with a specific classloader.
+ *
+ * @param <T> The type of the accumulated objects
+ */
+public class SerializedListAccumulator<T> implements Accumulator<T, ArrayList<byte[]>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private ArrayList<byte[]> localValue = new ArrayList<byte[]>();
+
+ @Override
+ public void add(T value) {
+ if (value == null) {
+ throw new NullPointerException("Value to accumulate must nor be null");
+ }
+
+ try {
+ byte[] byteArray = InstantiationUtil.serializeObject(value);
+ localValue.add(byteArray);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Serialization of accumulated value failed", e);
+ }
+ }
+
+ @Override
+ public ArrayList<byte[]> getLocalValue() {
+ return localValue;
+ }
+
+ public ArrayList<T> deserializeLocalValue(ClassLoader classLoader) {
+ try {
+ ArrayList<T> arrList = new ArrayList<T>(localValue.size());
+ for (byte[] byteArr : localValue) {
+ @SuppressWarnings("unchecked")
+ T item = (T) InstantiationUtil.deserializeObject(byteArr, classLoader);
+ arrList.add(item);
+ }
+ return arrList;
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Cannot deserialize accumulator list element", e);
+ }
+ }
+
+ @Override
+ public void resetLocal() {
+ localValue.clear();
+ }
+
+ @Override
+ public void merge(Accumulator<T, ArrayList<byte[]>> other) {
+ localValue.addAll(other.getLocalValue());
+ }
+
+ @Override
+ public SerializedListAccumulator<T> clone() {
+ SerializedListAccumulator<T> newInstance = new SerializedListAccumulator<T>();
+ newInstance.localValue = new ArrayList<byte[]>(localValue);
+ return newInstance;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> List<T> deserializeList(ArrayList<byte[]> data, ClassLoader loader)
+ throws IOException, ClassNotFoundException
+ {
+ List<T> result = new ArrayList<T>(data.size());
+ for (byte[] bytes : data) {
+ result.add((T) InstantiationUtil.deserializeObject(bytes, loader));
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SimpleAccumulator.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SimpleAccumulator.java b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SimpleAccumulator.java
index 596bf63..6614e13 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SimpleAccumulator.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/accumulators/SimpleAccumulator.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.api.common.accumulators;
import java.io.Serializable;
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
index c04548c..ff531c7 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
@@ -110,7 +110,7 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
@Override
public <V, A extends Serializable> void addAccumulator(String name, Accumulator<V, A> accumulator) {
if (accumulators.containsKey(name)) {
- throw new UnsupportedOperationException("The counter '" + name
+ throw new UnsupportedOperationException("The accumulator '" + name
+ "' already exists and cannot be added.");
}
accumulators.put(name, accumulator);
@@ -141,8 +141,8 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
@SuppressWarnings("unchecked")
private <V, A extends Serializable> Accumulator<V, A> getAccumulator(String name,
- Class<? extends Accumulator<V, A>> accumulatorClass) {
-
+ Class<? extends Accumulator<V, A>> accumulatorClass)
+ {
Accumulator<?, ?> accumulator = accumulators.get(name);
if (accumulator != null) {
@@ -151,10 +151,9 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
// Create new accumulator
try {
accumulator = accumulatorClass.newInstance();
- } catch (InstantiationException e) {
- e.printStackTrace();
- } catch (IllegalAccessException e) {
- e.printStackTrace();
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Cannot create accumulator " + accumulatorClass.getName());
}
accumulators.put(name, accumulator);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/relational/EmptyFieldsCountAccumulator.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/relational/EmptyFieldsCountAccumulator.java b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/relational/EmptyFieldsCountAccumulator.java
index 09b88fa..2016eaa 100644
--- a/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/relational/EmptyFieldsCountAccumulator.java
+++ b/flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/relational/EmptyFieldsCountAccumulator.java
@@ -15,11 +15,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.flink.examples.java.relational;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -206,14 +204,14 @@ public class EmptyFieldsCountAccumulator {
* Increases the result vector component at the specified position by 1.
*/
@Override
- public void add(final Integer position) {
+ public void add(Integer position) {
updateResultVector(position, 1);
}
/**
* Increases the result vector component at the specified position by the specified delta.
*/
- private void updateResultVector(final int position, final int delta) {
+ private void updateResultVector(int position, int delta) {
// inflate the vector to contain the given position
while (this.resultVector.size() <= position) {
this.resultVector.add(0);
@@ -245,31 +243,8 @@ public class EmptyFieldsCountAccumulator {
}
@Override
- public void write(final ObjectOutputStream out) throws IOException {
- // binary serialization of the result vector:
- // [number of components, component 0, component 1, ...]
- out.writeInt(this.resultVector.size());
- for (final Integer component : this.resultVector) {
- out.writeInt(component);
- }
- }
-
- @Override
- public void read(final ObjectInputStream in) throws IOException {
- // binary deserialization of the result vector
- final int size = in.readInt();
- for (int numReadComponents = 0; numReadComponents < size; numReadComponents++) {
- final int component = in.readInt();
- this.resultVector.add(component);
- }
- }
-
- @Override
public Accumulator<Integer, ArrayList<Integer>> clone() {
- VectorAccumulator result = new VectorAccumulator(new ArrayList<Integer>(resultVector));
-
- return result;
+ return new VectorAccumulator(new ArrayList<Integer>(resultVector));
}
-
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
index 1e91eeb..7cee323 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
@@ -18,11 +18,14 @@
package org.apache.flink.api.java;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.Validate;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.accumulators.SerializedListAccumulator;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.GroupCombineFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
@@ -399,7 +402,8 @@ public abstract class DataSet<T> {
}
- /* Convenience method to get the elements of a DataSet as a List
+ /**
+ * Convenience method to get the elements of a DataSet as a List
* As DataSet can contain a lot of data, this method should be used with caution.
*
* @return A List containing the elements of the DataSet
@@ -407,15 +411,31 @@ public abstract class DataSet<T> {
* @see org.apache.flink.api.java.Utils.CollectHelper
*/
public List<T> collect() throws Exception {
+ // validate that our type is actually serializable
+ Class<?> typeClass = getType().getTypeClass();
+ ClassLoader cl = typeClass.getClassLoader() == null ? ClassLoader.getSystemClassLoader()
+ : typeClass.getClassLoader();
- final String id = new AbstractID().toString();
+ if (!java.io.Serializable.class.isAssignableFrom(typeClass)) {
+ throw new UnsupportedOperationException("collect() can only be used with serializable data types. "
+ + "The DataSet type '" + typeClass.getName() + "' does not implement java.io.Serializable.");
+ }
- this.flatMap(new Utils.CollectHelper<T>(id)).output(
- new DiscardingOutputFormat<T>());
+ final String id = new AbstractID().toString();
- JobExecutionResult res = this.getExecutionEnvironment().execute();
+ this.flatMap(new Utils.CollectHelper<T>(id)).output(new DiscardingOutputFormat<T>());
+ JobExecutionResult res = getExecutionEnvironment().execute();
- return (List<T>) res.getAccumulatorResult(id);
+ ArrayList<byte[]> accResult = res.getAccumulatorResult(id);
+ try {
+ return SerializedListAccumulator.deserializeList(accResult, cl);
+ }
+ catch (ClassNotFoundException e) {
+ throw new RuntimeException("Cannot find type class of collected data type.", e);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Serialization error while deserializing collected data", e);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-java/src/main/java/org/apache/flink/api/java/Utils.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/Utils.java b/flink-java/src/main/java/org/apache/flink/api/java/Utils.java
index b860b77..5351484 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/Utils.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/Utils.java
@@ -19,6 +19,7 @@
package org.apache.flink.api.java;
import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.api.common.accumulators.SerializedListAccumulator;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.CompositeType;
import org.apache.flink.api.java.typeutils.GenericTypeInfo;
@@ -26,7 +27,6 @@ import org.apache.flink.api.java.typeutils.GenericTypeInfo;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.List;
-import org.apache.flink.api.common.accumulators.ListAccumulator;
import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.util.Collector;
@@ -97,11 +97,11 @@ public class Utils {
private static final long serialVersionUID = 1L;
private final String id;
- private final ListAccumulator<T> accumulator;
+ private final SerializedListAccumulator<T> accumulator;
public CollectHelper(String id) {
this.id = id;
- this.accumulator = new ListAccumulator<T>();
+ this.accumulator = new SerializedListAccumulator<T>();
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorEvent.java
index f345d6a..ad7fabd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorEvent.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/AccumulatorEvent.java
@@ -18,179 +18,32 @@
package org.apache.flink.runtime.accumulators;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.Map;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.JobID;
-import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.runtime.util.SerializedValue;
/**
* This class encapsulates a map of accumulators for a single job. It is used
* for the transfer from TaskManagers to the JobManager and from the JobManager
* to the Client.
*/
-public class AccumulatorEvent implements Serializable {
+public class AccumulatorEvent extends SerializedValue<Map<String, Accumulator<?, ?>>> {
private static final long serialVersionUID = 8965894516006882735L;
- private JobID jobID;
-
- private Map<String, Accumulator<?, ?>> accumulators;
-
- // staging deserialized data until the classloader is available
- private byte[] serializedData;
+ /** JobID for the target job */
+ private final JobID jobID;
- // Removing this causes an EOFException in the RPC service. The RPC should
- // be improved in this regard (error message is very unspecific).
- public AccumulatorEvent() {
- this.accumulators = Collections.emptyMap();
- }
-
- public AccumulatorEvent(JobID jobID, Map<String, Accumulator<?, ?>> accumulators) {
- this.accumulators = accumulators;
+ public AccumulatorEvent(JobID jobID, Map<String, Accumulator<?, ?>> accumulators) throws IOException {
+ super(accumulators);
this.jobID = jobID;
}
public JobID getJobID() {
return this.jobID;
}
-
- public Map<String, Accumulator<?, ?>> getAccumulators(ClassLoader loader) {
- if (loader == null) {
- throw new NullPointerException();
- }
-
- if (this.accumulators == null) {
- // deserialize
- // we have read the binary data, but not yet turned into the objects
- ByteArrayInputStream bais = new ByteArrayInputStream(serializedData);
- ObjectInputStream ios = null;
- try {
- ios = new ObjectInputStream(bais);
- } catch (IOException e) {
- throw new RuntimeException("Error while creating the object input stream.");
- }
-
- int numAccumulators = 0;
- try {
- numAccumulators = ios.readInt();
- } catch (IOException e) {
- throw new RuntimeException("Error while reading the number of serialized " +
- "accumulators.");
- }
-
- this.accumulators = new HashMap<String, Accumulator<?, ?>>(numAccumulators);
-
- for (int i = 0; i < numAccumulators; i++) {
- String accumulatorName;
- try {
- accumulatorName = ios.readUTF();
- } catch (IOException e) {
- throw new RuntimeException("Error while reading the "+ i +"th accumulator " +
- "name.");
- }
-
- String className;
- try {
- className = ios.readUTF();
- } catch (IOException e) {
- throw new RuntimeException("Error while reading the "+ i +"th accumulator " +
- "class name.");
- }
- Accumulator<?, ?> accumulator;
-
- try {
- @SuppressWarnings("unchecked")
- Class<? extends Accumulator<?, ?>> valClass = (Class<? extends Accumulator<?,
- ?>>) Class.forName(className, true, loader);
- accumulator = InstantiationUtil.instantiate(valClass, Accumulator.class);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException("Could not load user-defined class '" +
- className + "'.", e);
- } catch (ClassCastException e) {
- throw new RuntimeException("User-defined accumulator class is not an Accumulator sublass.");
- }
-
- try {
- accumulator.read(ios);
- } catch (IOException e) {
- throw new RuntimeException("Error while deserializing the user-defined aggregate class.", e);
- }
-
- accumulators.put(accumulatorName, accumulator);
-
- }
-
- try {
- ios.close();
- } catch (IOException e) {
- throw new RuntimeException("Error while closing the InputObjectStream.");
- }
-
- try {
- bais.close();
- } catch (IOException e) {
- throw new RuntimeException("Error while closing the ByteArrayInputStream.");
- }
-
- }
-
- return accumulators;
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws IOException{
- out.writeObject(jobID);
-
- byte[] buffer = null;
-
- if(accumulators != null) {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(baos);
-
- oos.writeInt(accumulators.size());
-
- for (Map.Entry<String, Accumulator<?, ?>> entry : this.accumulators.entrySet()) {
- oos.writeUTF(entry.getKey());
- oos.writeUTF(entry.getValue().getClass().getName());
-
- entry.getValue().write(oos);
- }
-
- oos.flush();
- oos.close();
- baos.close();
-
- buffer = baos.toByteArray();
- } else if(serializedData != null) {
- buffer = serializedData;
- } else {
- throw new RuntimeException("The AccumulatorEvent's accumulator is null and there is " +
- "no serialized data attached to it.");
- }
-
- out.writeInt(buffer.length);
- out.write(buffer);
- }
-
- private void readObject(java.io.ObjectInputStream in) throws IOException,
- ClassNotFoundException{
- this.accumulators = null; // this makes sure we deserialize
-
- jobID = (JobID) in.readObject();
-
- int bufferLength = in.readInt();
-
- serializedData = new byte[bufferLength];
-
- in.readFully(serializedData);
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
new file mode 100644
index 0000000..3b25fe0
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/accumulators/StringifiedAccumulatorResult.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.accumulators;
+
+/**
+ * Container class that transports the result of an accumulator as set of strings.
+ */
+public class StringifiedAccumulatorResult implements java.io.Serializable{
+
+ private static final long serialVersionUID = -4642311296836822611L;
+
+ private final String name;
+ private final String type;
+ private final String value;
+
+ public StringifiedAccumulatorResult(String name, String type, String value) {
+ this.name = name;
+ this.type = type;
+ this.value = value;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public String getValue() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java
new file mode 100644
index 0000000..e0b1ad1
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/SerializedJobExecutionResult.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.client;
+
+import org.apache.flink.api.common.JobExecutionResult;
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.util.SerializedValue;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * A variant of the {@link org.apache.flink.api.common.JobExecutionResult} that holds
+ * its accumulator data in serialized form.
+ */
+public class SerializedJobExecutionResult implements java.io.Serializable {
+
+ private static final long serialVersionUID = -6301865617099921789L;
+
+ private final JobID jobId;
+
+ private final Map<String, SerializedValue<Object>> accumulatorResults;
+
+ private final long netRuntime;
+
+ /**
+ * Creates a new SerializedJobExecutionResult.
+ *
+ * @param jobID The job's ID.
+ * @param netRuntime The net runtime of the job (excluding pre-flight phase like the optimizer)
+ * @param accumulators A map of all accumulator results produced by the job, in serialized form
+ */
+ public SerializedJobExecutionResult(JobID jobID, long netRuntime,
+ Map<String, SerializedValue<Object>> accumulators) {
+ this.jobId = jobID;
+ this.netRuntime = netRuntime;
+ this.accumulatorResults = accumulators;
+ }
+
+ public JobID getJobId() {
+ return jobId;
+ }
+
+ public long getNetRuntime() {
+ return netRuntime;
+ }
+
+ public Map<String, SerializedValue<Object>> getSerializedAccumulatorResults() {
+ return this.accumulatorResults;
+ }
+
+ public JobExecutionResult toJobExecutionResult(ClassLoader loader) throws IOException, ClassNotFoundException {
+ Map<String, Object> accumulators = null;
+ if (accumulatorResults != null) {
+ accumulators = accumulatorResults.isEmpty() ?
+ Collections.<String, Object>emptyMap() :
+ new HashMap<String, Object>(this.accumulatorResults.size());
+
+ for (Map.Entry<String, SerializedValue<Object>> entry : this.accumulatorResults.entrySet()) {
+ Object o = entry.getValue() == null ? null : entry.getValue().deserializeValue(loader);
+ accumulators.put(entry.getKey(), o);
+ }
+ }
+
+ return new JobExecutionResult(jobId, netRuntime, accumulators);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
index 5416f48..d567156 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.execution;
import akka.actor.ActorRef;
import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.accumulators.AccumulatorHelper;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.accumulators.AccumulatorEvent;
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index 5ce89b3..08e44f9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -176,6 +176,8 @@ public class ExecutionGraph implements Serializable {
* Once this value has reached the number of vertices, the job is done. */
private int nextVertexToFinish;
+
+
private ActorContext parentContext;
private ActorRef stateCheckpointerActor;
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/AccumulatorManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/AccumulatorManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/AccumulatorManager.java
index 5bf8ebe..c824232 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/AccumulatorManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/AccumulatorManager.java
@@ -16,15 +16,18 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.jobmanager.accumulators;
+import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
+import org.apache.flink.runtime.util.SerializedValue;
/**
* This class manages the accumulators for different jobs. Either the jobs are
@@ -38,12 +41,13 @@ import org.apache.flink.api.common.JobID;
*/
public class AccumulatorManager {
- // Map of accumulators belonging to recently started jobs
+ /** Map of accumulators belonging to recently started jobs */
private final Map<JobID, JobAccumulators> jobAccumulators = new HashMap<JobID, JobAccumulators>();
private final LinkedList<JobID> lru = new LinkedList<JobID>();
private int maxEntries;
+
public AccumulatorManager(int maxEntries) {
this.maxEntries = maxEntries;
}
@@ -69,11 +73,13 @@ public class AccumulatorManager {
public Map<String, Object> getJobAccumulatorResults(JobID jobID) {
Map<String, Object> result = new HashMap<String, Object>();
- JobAccumulators jobAccumulator = jobAccumulators.get(jobID);
+ JobAccumulators acc;
+ synchronized (jobAccumulators) {
+ acc = jobAccumulators.get(jobID);
+ }
- if(jobAccumulator != null) {
- for (Map.Entry<String, Accumulator<?, ?>> entry : jobAccumulator.getAccumulators().
- entrySet()) {
+ if (acc != null) {
+ for (Map.Entry<String, Accumulator<?, ?>> entry : acc.getAccumulators().entrySet()) {
result.put(entry.getKey(), entry.getValue().getLocalValue());
}
}
@@ -81,9 +87,50 @@ public class AccumulatorManager {
return result;
}
+ public Map<String, SerializedValue<Object>> getJobAccumulatorResultsSerialized(JobID jobID) throws IOException {
+ JobAccumulators acc;
+ synchronized (jobAccumulators) {
+ acc = jobAccumulators.get(jobID);
+ }
+
+ if (acc == null || acc.getAccumulators().isEmpty()) {
+ return Collections.emptyMap();
+ }
+
+ Map<String, SerializedValue<Object>> result = new HashMap<String, SerializedValue<Object>>();
+ for (Map.Entry<String, Accumulator<?, ?>> entry : acc.getAccumulators().entrySet()) {
+ result.put(entry.getKey(), new SerializedValue<Object>(entry.getValue().getLocalValue()));
+ }
+
+ return result;
+ }
+
+ public StringifiedAccumulatorResult[] getJobAccumulatorResultsStringified(JobID jobID) throws IOException {
+ JobAccumulators acc;
+ synchronized (jobAccumulators) {
+ acc = jobAccumulators.get(jobID);
+ }
+
+ if (acc == null || acc.getAccumulators().isEmpty()) {
+ return new StringifiedAccumulatorResult[0];
+ }
+
+ Map<String, Accumulator<?, ?>> accMap = acc.getAccumulators();
+
+ StringifiedAccumulatorResult[] result = new StringifiedAccumulatorResult[accMap.size()];
+ int i = 0;
+ for (Map.Entry<String, Accumulator<?, ?>> entry : accMap.entrySet()) {
+ String type = entry.getValue() == null ? "(null)" : entry.getValue().getClass().getSimpleName();
+ String value = entry.getValue() == null ? "(null)" : entry.getValue().toString();
+ result[i++] = new StringifiedAccumulatorResult(entry.getKey(), type, value);
+ }
+ return result;
+ }
+
/**
- * Cleanup data for the oldest jobs if the maximum number of entries is
- * reached.
+ * Cleanup data for the oldest jobs if the maximum number of entries is reached.
+ *
+ * @param jobId The (potentially new) JobId.
*/
private void cleanup(JobID jobId) {
if (!lru.contains(jobId)) {
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/JobAccumulators.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/JobAccumulators.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/JobAccumulators.java
index 2835b30..970d993 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/JobAccumulators.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/accumulators/JobAccumulators.java
@@ -16,7 +16,6 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.jobmanager.accumulators;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JobManagerInfoServlet.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JobManagerInfoServlet.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JobManagerInfoServlet.java
index 008be65..144c477 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JobManagerInfoServlet.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JobManagerInfoServlet.java
@@ -25,8 +25,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
@@ -37,18 +35,20 @@ import akka.actor.ActorRef;
import akka.pattern.Patterns;
import akka.util.Timeout;
+import org.apache.flink.runtime.accumulators.StringifiedAccumulatorResult;
import org.apache.flink.runtime.instance.InstanceConnectionInfo;
import org.apache.flink.runtime.messages.ArchiveMessages.ArchivedJobs;
import org.apache.flink.runtime.messages.ArchiveMessages;
import org.apache.flink.runtime.messages.JobManagerMessages;
-import org.apache.flink.runtime.messages.JobManagerMessages.AccumulatorResultsResponse;
-import org.apache.flink.runtime.messages.JobManagerMessages.AccumulatorResultsFound;
import org.apache.flink.runtime.messages.JobManagerMessages.RunningJobs;
import org.apache.flink.runtime.messages.JobManagerMessages.CancelJob;
-import org.apache.flink.runtime.messages.JobManagerMessages.RequestAccumulatorResults;
import org.apache.flink.runtime.messages.JobManagerMessages.RequestJob;
import org.apache.flink.runtime.messages.JobManagerMessages.JobResponse;
import org.apache.flink.runtime.messages.JobManagerMessages.JobFound;
+import org.apache.flink.runtime.messages.accumulators.AccumulatorResultStringsFound;
+import org.apache.flink.runtime.messages.accumulators.AccumulatorResultsErroneous;
+import org.apache.flink.runtime.messages.accumulators.AccumulatorResultsNotFound;
+import org.apache.flink.runtime.messages.accumulators.RequestAccumulatorResultsStringified;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.runtime.execution.ExecutionState;
@@ -69,34 +69,34 @@ import scala.concurrent.Future;
import scala.concurrent.duration.FiniteDuration;
public class JobManagerInfoServlet extends HttpServlet {
-
+
private static final long serialVersionUID = 1L;
-
+
private static final Logger LOG = LoggerFactory.getLogger(JobManagerInfoServlet.class);
-
+
/** Underlying JobManager */
private final ActorRef jobmanager;
private final ActorRef archive;
private final FiniteDuration timeout;
-
-
+
+
public JobManagerInfoServlet(ActorRef jobmanager, ActorRef archive, FiniteDuration timeout) {
this.jobmanager = jobmanager;
this.archive = archive;
this.timeout = timeout;
}
-
-
+
+
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException,
IOException {
-
+
resp.setStatus(HttpServletResponse.SC_OK);
resp.setContentType("application/json");
Future<Object> response;
Object result;
-
+
try {
if("archive".equals(req.getParameter("get"))) {
response = Patterns.ask(archive, ArchiveMessages.getRequestArchivedJobs(),
@@ -132,7 +132,7 @@ public class JobManagerInfoServlet extends HttpServlet {
if(jobResponse instanceof JobFound){
ExecutionGraph archivedJob = ((JobFound)result).executionGraph();
writeJsonForArchivedJob(resp.getWriter(), archivedJob);
- } else {
+ } else {
LOG.warn("DoGet:job: Could not find job for job ID " + jobId);
}
}
@@ -157,7 +157,7 @@ public class JobManagerInfoServlet extends HttpServlet {
writeJsonForArchivedJobGroupvertex(resp.getWriter(), archivedJob,
JobVertexID.fromHexString(groupvertexId));
- } else {
+ } else {
LOG.warn("DoGet:groupvertex: Could not find job for job ID " + jobId);
}
}
@@ -226,7 +226,7 @@ public class JobManagerInfoServlet extends HttpServlet {
writeJsonForJobs(resp.getWriter(), runningJobs);
}
}
-
+
} catch (Exception e) {
resp.setStatus(HttpServletResponse.SC_BAD_REQUEST);
resp.getWriter().print(e.getMessage());
@@ -235,10 +235,10 @@ public class JobManagerInfoServlet extends HttpServlet {
}
}
}
-
+
/**
* Writes ManagementGraph as Json for all recent jobs
- *
+ *
* @param wrt
* @param graphs
*/
@@ -250,23 +250,23 @@ public class JobManagerInfoServlet extends HttpServlet {
// Loop Jobs
while(it.hasNext()){
ExecutionGraph graph = it.next();
-
+
writeJsonForJob(wrt, graph);
-
+
//Write seperator between json objects
if(it.hasNext()) {
wrt.write(",");
}
}
wrt.write("]");
-
+
} catch (EofException eof) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, EofException");
- } catch (IOException ioe) { // Connection closed by client
+ } catch (IOException ioe) { // Connection closed by client
LOG.info("Info server for jobmanager: Connection closed by client, IOException");
- }
+ }
}
-
+
private void writeJsonForJob(PrintWriter wrt, ExecutionGraph graph) throws IOException {
//Serialize job to json
wrt.write("{");
@@ -274,35 +274,35 @@ public class JobManagerInfoServlet extends HttpServlet {
wrt.write("\"jobname\": \"" + graph.getJobName()+"\",");
wrt.write("\"status\": \""+ graph.getState() + "\",");
wrt.write("\"time\": " + graph.getStatusTimestamp(graph.getState())+",");
-
+
// Serialize ManagementGraph to json
wrt.write("\"groupvertices\": [");
boolean first = true;
-
+
for (ExecutionJobVertex groupVertex : graph.getVerticesTopologically()) {
//Write seperator between json objects
if(first) {
first = false;
} else {
wrt.write(","); }
-
+
wrt.write(JsonFactory.toJson(groupVertex));
}
wrt.write("]");
wrt.write("}");
-
+
}
-
+
/**
* Writes Json with a list of currently archived jobs, sorted by time
- *
+ *
* @param wrt
* @param graphs
*/
private void writeJsonForArchive(PrintWriter wrt, List<ExecutionGraph> graphs) {
-
+
wrt.write("[");
-
+
// sort jobs by time
Collections.sort(graphs, new Comparator<ExecutionGraph>() {
@Override
@@ -313,41 +313,39 @@ public class JobManagerInfoServlet extends HttpServlet {
return -1;
}
}
-
+
});
-
+
// Loop Jobs
for (int i = 0; i < graphs.size(); i++) {
ExecutionGraph graph = graphs.get(i);
-
+
//Serialize job to json
wrt.write("{");
wrt.write("\"jobid\": \"" + graph.getJobID() + "\",");
wrt.write("\"jobname\": \"" + graph.getJobName()+"\",");
wrt.write("\"status\": \""+ graph.getState() + "\",");
wrt.write("\"time\": " + graph.getStatusTimestamp(graph.getState()));
-
+
wrt.write("}");
-
+
//Write seperator between json objects
if(i != graphs.size() - 1) {
wrt.write(",");
}
}
wrt.write("]");
-
+
}
-
+
/**
* Writes infos about archived job in Json format, including groupvertices and groupverticetimes
- *
+ *
* @param wrt
* @param graph
*/
private void writeJsonForArchivedJob(PrintWriter wrt, ExecutionGraph graph) {
-
try {
-
wrt.write("[");
//Serialize job to json
@@ -402,113 +400,108 @@ public class JobManagerInfoServlet extends HttpServlet {
// write accumulators
final Future<Object> response = Patterns.ask(jobmanager,
- new RequestAccumulatorResults(graph.getJobID()), new Timeout(timeout));
-
- Object result = null;
+ new RequestAccumulatorResultsStringified(graph.getJobID()), new Timeout(timeout));
+ Object result;
try {
result = Await.result(response, timeout);
} catch (Exception ex) {
- throw new IOException("Could not retrieve the accumulator results from the " +
- "job manager.", ex);
+ throw new IOException("Could not retrieve the accumulator results from the job manager.", ex);
}
- if (!(result instanceof AccumulatorResultsResponse)) {
+ if (result instanceof AccumulatorResultStringsFound) {
+ StringifiedAccumulatorResult[] accumulators = ((AccumulatorResultStringsFound) result).result();
+
+ wrt.write("\n\"accumulators\": [");
+ int i = 0;
+ for (StringifiedAccumulatorResult accumulator : accumulators) {
+ wrt.write("{ \"name\": \"" + accumulator.getName() + " (" + accumulator.getType() + ")\","
+ + " \"value\": \"" + accumulator.getValue() + "\"}\n");
+ if (++i < accumulators.length) {
+ wrt.write(",");
+ }
+ }
+ wrt.write("],\n");
+ }
+ else if (result instanceof AccumulatorResultsNotFound) {
+ wrt.write("\n\"accumulators\": [],");
+ }
+ else if (result instanceof AccumulatorResultsErroneous) {
+ LOG.error("Could not obtain accumulators for job " + graph.getJobID(),
+ ((AccumulatorResultsErroneous) result).cause());
+ }
+ else {
throw new RuntimeException("RequestAccumulatorResults requires a response of type " +
- "AccumulatorResultsReponse. Instead the response is of type " +
+ "AccumulatorResultStringsFound. Instead the response is of type " +
result.getClass() + ".");
- } else {
- final AccumulatorResultsResponse accumulatorResponse =
- (AccumulatorResultsResponse) result;
-
- if (accumulatorResponse instanceof AccumulatorResultsFound) {
- Map<String, Object> accMap = ((AccumulatorResultsFound) accumulatorResponse).
- asJavaMap();
-
- wrt.write("\n\"accumulators\": [");
- int i = 0;
- for (Entry<String, Object> accumulator : accMap.entrySet()) {
- wrt.write("{ \"name\": \"" + accumulator.getKey() + " (" + accumulator.getValue().getClass().getName() + ")\","
- + " \"value\": \"" + accumulator.getValue().toString() + "\"}\n");
- if (++i < accMap.size()) {
- wrt.write(",");
- }
- }
- wrt.write("],\n");
-
- wrt.write("\"groupverticetimes\": {");
- first = true;
- for (ExecutionJobVertex groupVertex : graph.getVerticesTopologically()) {
-
- if (first) {
- first = false;
- } else {
- wrt.write(",");
- }
-
- // Calculate start and end time for groupvertex
- long started = Long.MAX_VALUE;
- long ended = 0;
+ }
- // Take earliest running state and latest endstate of groupmembers
- for (ExecutionVertex vertex : groupVertex.getTaskVertices()) {
+ wrt.write("\"groupverticetimes\": {");
+ first = true;
- long running = vertex.getStateTimestamp(ExecutionState.RUNNING);
- if (running != 0 && running < started) {
- started = running;
- }
+ for (ExecutionJobVertex groupVertex : graph.getVerticesTopologically()) {
+ if (first) {
+ first = false;
+ } else {
+ wrt.write(",");
+ }
- long finished = vertex.getStateTimestamp(ExecutionState.FINISHED);
- long canceled = vertex.getStateTimestamp(ExecutionState.CANCELED);
- long failed = vertex.getStateTimestamp(ExecutionState.FAILED);
+ // Calculate start and end time for groupvertex
+ long started = Long.MAX_VALUE;
+ long ended = 0;
- if (finished != 0 && finished > ended) {
- ended = finished;
- }
+ // Take earliest running state and latest endstate of groupmembers
+ for (ExecutionVertex vertex : groupVertex.getTaskVertices()) {
- if (canceled != 0 && canceled > ended) {
- ended = canceled;
- }
+ long running = vertex.getStateTimestamp(ExecutionState.RUNNING);
+ if (running != 0 && running < started) {
+ started = running;
+ }
- if (failed != 0 && failed > ended) {
- ended = failed;
- }
+ long finished = vertex.getStateTimestamp(ExecutionState.FINISHED);
+ long canceled = vertex.getStateTimestamp(ExecutionState.CANCELED);
+ long failed = vertex.getStateTimestamp(ExecutionState.FAILED);
- }
+ if (finished != 0 && finished > ended) {
+ ended = finished;
+ }
- wrt.write("\"" + groupVertex.getJobVertexId() + "\": {");
- wrt.write("\"groupvertexid\": \"" + groupVertex.getJobVertexId() + "\",");
- wrt.write("\"groupvertexname\": \"" + groupVertex + "\",");
- wrt.write("\"STARTED\": " + started + ",");
- wrt.write("\"ENDED\": " + ended);
- wrt.write("}");
+ if (canceled != 0 && canceled > ended) {
+ ended = canceled;
+ }
+ if (failed != 0 && failed > ended) {
+ ended = failed;
}
- } else {
- LOG.warn("Could not find accumulator results for job ID " + graph.getJobID());
- }
- wrt.write("}");
+ }
+ wrt.write("\"" + groupVertex.getJobVertexId() + "\": {");
+ wrt.write("\"groupvertexid\": \"" + groupVertex.getJobVertexId() + "\",");
+ wrt.write("\"groupvertexname\": \"" + groupVertex + "\",");
+ wrt.write("\"STARTED\": " + started + ",");
+ wrt.write("\"ENDED\": " + ended);
wrt.write("}");
-
- wrt.write("]");
}
- } catch (Exception ex) { // Connection closed by client
- LOG.info("Info server for jobmanager: Failed to write json for archived jobs, " +
- "because {}.", StringUtils.stringifyException(ex));
+
+ wrt.write("}");
+ wrt.write("}");
+ wrt.write("]");
+ }
+ catch (Exception ex) { // Connection closed by client
+ LOG.error("Info server for JobManager: Failed to write json for archived jobs", ex);
}
}
/**
* Writes all updates (events) for a given job since a given time
- *
+ *
* @param wrt
* @param jobId
*/
private void writeJsonUpdatesForJob(PrintWriter wrt, JobID jobId) {
-
+
try {
final Future<Object> responseArchivedJobs = Patterns.ask(jobmanager,
JobManagerMessages.getRequestRunningJobs(),
@@ -539,9 +532,9 @@ public class JobManagerInfoServlet extends HttpServlet {
boolean first = true;
for(ExecutionGraph g : graphs){
- if (first) {
+ if (first) {
first = false;
- } else {
+ } else {
wrt.write(",");
}
@@ -559,7 +552,7 @@ public class JobManagerInfoServlet extends HttpServlet {
resultJob = Await.result(responseJob, timeout);
} catch (Exception ex){
throw new IOException("Could not retrieve the job with jobID " + jobId +
- "from the job manager.", ex);
+ "from the job manager.", ex);
}
if(!(resultJob instanceof JobResponse)) {
@@ -602,7 +595,7 @@ public class JobManagerInfoServlet extends HttpServlet {
wrt.write("]");
wrt.write("}");
- } else {
+ } else {
wrt.write("\"vertexevents\": [],");
wrt.write("\"jobevents\": [");
wrt.write("{");
@@ -615,14 +608,14 @@ public class JobManagerInfoServlet extends HttpServlet {
}
}
}
-
+
} catch (Exception exception) { // Connection closed by client
LOG.info("Info server for jobmanager: Failed to write json updates for job {}, " +
"because {}.", jobId, StringUtils.stringifyException(exception));
- }
-
+ }
+
}
-
+
/**
* Writes info about one particular archived JobVertex in a job, including all member execution vertices, their times and statuses.
*/
@@ -663,10 +656,10 @@ public class JobManagerInfoServlet extends HttpServlet {
}
wrt.write("}}");
}
-
+
/**
* Writes the version and the revision of Flink.
- *
+ *
* @param wrt
*/
private void writeJsonForVersion(PrintWriter wrt) {
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedValue.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedValue.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedValue.java
new file mode 100644
index 0000000..f5e897b
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializedValue.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.util;
+
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * This class is used to transfer (via serialization) objects whose classes are not available
+ * in the system class loader. When those objects are deserialized without access to their
+ * special class loader, the deserialization fails with a {@code ClassNotFoundException}.
+ *
+ * To work around that issue, the SerializedValue serialized data immediately into a byte array.
+ * When send through RPC or another service that uses serialization, the only the byte array is
+ * transferred. The object is deserialized later (upon access) and requires the accessor to
+ * provide the corresponding class loader.
+ *
+ * @param <T> The type of the value held.
+ */
+public class SerializedValue<T> implements java.io.Serializable {
+
+ private static final long serialVersionUID = -3564011643393683761L;
+
+ /** The serialized data */
+ private final byte[] serializedData;
+
+
+ public SerializedValue(T value) throws IOException {
+ this.serializedData = value == null ? null : InstantiationUtil.serializeObject(value);
+ }
+
+
+ @SuppressWarnings("unchecked")
+ public T deserializeValue(ClassLoader loader) throws IOException, ClassNotFoundException {
+ if (loader == null) {
+ throw new NullPointerException();
+ }
+
+ return serializedData == null ? null : (T) InstantiationUtil.deserializeObject(serializedData, loader);
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+
+ @Override
+ public int hashCode() {
+ return serializedData == null ? 0 : Arrays.hashCode(serializedData);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof SerializedValue) {
+ SerializedValue<?> other = (SerializedValue<?>) obj;
+ return this.serializedData == null ? other.serializedData == null :
+ (other.serializedData != null && Arrays.equals(this.serializedData, other.serializedData));
+ }
+ else {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "SerializedValue";
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/6774054a/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
index d153dcc..013fe4c 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/client/JobClient.scala
@@ -24,7 +24,6 @@ import java.net.{InetAddress, InetSocketAddress}
import akka.actor.Status.{Success, Failure}
import akka.actor._
import akka.pattern.{Patterns, ask}
-import org.apache.flink.api.common.JobExecutionResult
import org.apache.flink.configuration.{ConfigConstants, Configuration}
import org.apache.flink.runtime.ActorLogMessages
import org.apache.flink.runtime.akka.AkkaUtils
@@ -46,6 +45,7 @@ class JobClient(jobManager: ActorRef) extends
Actor with ActorLogMessages with ActorLogging {
override def receiveWithLogMessages: Receive = {
+
case SubmitJobDetached(jobGraph) =>
jobManager forward SubmitJob(jobGraph, registerForEvents = false)
@@ -53,7 +53,7 @@ Actor with ActorLogMessages with ActorLogging {
jobManager forward cancelJob
case SubmitJobAndWait(jobGraph, listen) =>
- val listener = context.actorOf(Props(classOf[JobClientListener], sender))
+ val listener = context.actorOf(Props(classOf[JobClientListener], sender()))
jobManager.tell(SubmitJob(jobGraph, registerForEvents = listen), listener)
case RequestBlobManagerPort =>
@@ -88,8 +88,8 @@ ActorLogging {
case Success(_) =>
- case JobResultSuccess(jobId, duration, accumulatorResults) =>
- jobSubmitter ! new JobExecutionResult(jobId, duration, accumulatorResults)
+ case JobResultSuccess(result) =>
+ jobSubmitter ! result
self ! PoisonPill
case msg =>
@@ -191,8 +191,8 @@ object JobClient {
/**
* Sends a [[JobGraph]] to the JobClient actor specified by jobClient which submits it then to
* the JobManager. The method blocks until the job has finished or the JobManager is no longer
- * alive. In the former case, the [[JobExecutionResult]] is returned and in the latter case a
- * [[JobExecutionException]] is thrown.
+ * alive. In the former case, the [[SerializedJobExecutionResult]] is returned and in the latter
+ * case a [[JobExecutionException]] is thrown.
*
* @param jobGraph JobGraph describing the Flink job
* @param listenToStatusEvents true if the JobClient shall print status events of the
@@ -204,14 +204,16 @@ object JobClient {
* @return The job execution result
*/
@throws(classOf[JobExecutionException])
- def submitJobAndWait(jobGraph: JobGraph, listenToStatusEvents: Boolean, jobClient: ActorRef)
- (implicit timeout: FiniteDuration): JobExecutionResult = {
+ def submitJobAndWait(jobGraph: JobGraph,
+ listenToStatusEvents: Boolean,
+ jobClient: ActorRef,
+ timeout: FiniteDuration): SerializedJobExecutionResult = {
var waitForAnswer = true
- var answer: JobExecutionResult = null
+ var answer: SerializedJobExecutionResult = null
val result = (jobClient ? SubmitJobAndWait(jobGraph, listenToEvents = listenToStatusEvents))(
- AkkaUtils.INF_TIMEOUT).mapTo[JobExecutionResult]
+ AkkaUtils.INF_TIMEOUT).mapTo[SerializedJobExecutionResult]
while (waitForAnswer) {
try {
@@ -241,12 +243,13 @@ object JobClient {
*
* @param jobGraph Flink job
* @param jobClient ActorRef to the JobClient
- * @param timeout Tiemout for futures
+ * @param timeout Timeout for futures
* @return The submission response
*/
@throws(classOf[JobExecutionException])
- def submitJobDetached(jobGraph: JobGraph, jobClient: ActorRef)(implicit timeout: FiniteDuration):
- Unit = {
+ def submitJobDetached(jobGraph: JobGraph,
+ jobClient: ActorRef,
+ timeout: FiniteDuration): Unit = {
val response = (jobClient ? SubmitJobDetached(jobGraph))(timeout)
@@ -255,7 +258,7 @@ object JobClient {
} catch {
case timeout: TimeoutException =>
throw new JobTimeoutException(jobGraph.getJobID,
- "Timeout while waiting for the submission result.", timeout);
+ "Timeout while submitting the job to the JobManager.", timeout);
}
}