You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2015/01/12 09:16:16 UTC
[08/18] flink git commit: [FLINK-986] [FLINK-25] [Distributed
runtime] Add initial support for intermediate results
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
index e29b149..bc815dc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/IterationEventWithAggregators.java
@@ -31,11 +31,11 @@ import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.core.memory.InputViewDataInputStreamWrapper;
import org.apache.flink.core.memory.OutputViewDataOutputStreamWrapper;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
+import org.apache.flink.runtime.event.task.TaskEvent;
import org.apache.flink.types.Value;
import org.apache.flink.util.InstantiationUtil;
-public abstract class IterationEventWithAggregators extends AbstractTaskEvent {
+public abstract class IterationEventWithAggregators extends TaskEvent {
protected static final String[] NO_STRINGS = new String[0];
protected static final Value[] NO_VALUES = new Value[0];
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
index 6cc1e10..9e74c34 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/event/TerminationEvent.java
@@ -23,12 +23,12 @@ import java.io.IOException;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
+import org.apache.flink.runtime.event.task.TaskEvent;
/**
* Signals that the iteration is completely executed, participating tasks must terminate now
*/
-public class TerminationEvent extends AbstractTaskEvent {
+public class TerminationEvent extends TaskEvent {
public static final TerminationEvent INSTANCE = new TerminationEvent();
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
index c73c1ea..3b2eda5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetFastUpdateOutputCollector.java
@@ -26,13 +26,12 @@ import org.apache.flink.util.Collector;
/**
* A {@link Collector} to update the solution set of a workset iteration.
- * <p/>
+ * <p>
* The records are written to a HashTable hash table to allow in-memory point updates.
- * <p/>
+ * <p>
* Assumption for fast updates: the build side iterator of the hash table is already positioned for the update. This
* is for example the case when a solution set update happens directly after a solution set join. If this assumption
* doesn't hold, use {@link SolutionSetUpdateOutputCollector}, which probes the hash table before updating.
-
*/
public class SolutionSetFastUpdateOutputCollector<T> implements Collector<T> {
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
index 502dfc9..21a9cc8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetObjectsUpdateOutputCollector.java
@@ -24,9 +24,9 @@ import org.apache.flink.util.Collector;
/**
* A {@link Collector} to update the solution set of a workset iteration.
- * <p/>
+ * <p>
* The records are written to a HashTable hash table to allow in-memory point updates.
- * <p/>
+ * <p>
* Records will only be collected, if there is a match after probing the hash table. If the build side iterator is
* already positioned for the update, use {@link SolutionSetFastUpdateOutputCollector} to the save re-probing.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
index 0bc2dd2..f400b7e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/SolutionSetUpdateOutputCollector.java
@@ -27,9 +27,9 @@ import org.apache.flink.util.Collector;
/**
* A {@link Collector} to update the solution set of a workset iteration.
- * <p/>
+ * <p>
* The records are written to a HashTable hash table to allow in-memory point updates.
- * <p/>
+ * <p>
* Records will only be collected, if there is a match after probing the hash table. If the build side iterator is
* already positioned for the update, use {@link SolutionSetFastUpdateOutputCollector} to the save re-probing.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
index 22bc46d..ad1d274 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/io/WorksetUpdateOutputCollector.java
@@ -27,7 +27,7 @@ import org.apache.flink.util.Collector;
/**
* A {@link Collector} to update the iteration workset (partial solution for bulk iterations).
- * <p/>
+ * <p>
* The records are written to a {@link DataOutputView} to allow in-memory data exchange.
*/
public class WorksetUpdateOutputCollector<T> implements Collector<T> {
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
index 1008256..4f10683 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/AbstractIterativePactTask.java
@@ -29,7 +29,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.io.network.api.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannelBroker;
import org.apache.flink.runtime.iterative.concurrent.Broker;
@@ -164,7 +164,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
@Override
public DistributedRuntimeUDFContext createRuntimeContext(String taskName) {
Environment env = getEnvironment();
- return new IterativeRuntimeUdfContext(taskName, env.getCurrentNumberOfSubtasks(),
+ return new IterativeRuntimeUdfContext(taskName, env.getNumberOfSubtasks(),
env.getIndexInSubtaskGroup(), getUserCodeClassLoader());
}
@@ -227,7 +227,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
for (int inputNum : this.iterativeInputs) {
MutableReader<?> reader = this.inputReaders[inputNum];
- if (!reader.isInputClosed()) {
+ if (!reader.isFinished()) {
if (reader.hasReachedEndOfSuperstep()) {
reader.startNextSuperstep();
}
@@ -238,7 +238,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
Object o = this.inputSerializers[inputNum].getSerializer().createInstance();
while ((o = inIter.next(o)) != null);
- if (!reader.isInputClosed()) {
+ if (!reader.isFinished()) {
// also reset the end-of-superstep state
reader.startNextSuperstep();
}
@@ -249,7 +249,7 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
for (int inputNum : this.iterativeBroadcastInputs) {
MutableReader<?> reader = this.broadcastInputReaders[inputNum];
- if (!reader.isInputClosed()) {
+ if (!reader.isFinished()) {
// sanity check that the BC input is at the end of the superstep
if (!reader.hasReachedEndOfSuperstep()) {
@@ -283,10 +283,10 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
/**
* Creates a new {@link WorksetUpdateOutputCollector}.
- * <p/>
+ * <p>
* This collector is used by {@link IterationIntermediatePactTask} or {@link IterationTailPactTask} to update the
* workset.
- * <p/>
+ * <p>
* If a non-null delegate is given, the new {@link Collector} will write to the solution set and also call
* collect(T) of the delegate.
*
@@ -305,12 +305,12 @@ public abstract class AbstractIterativePactTask<S extends Function, OT> extends
/**
* Creates a new solution set update output collector.
- * <p/>
+ * <p>
* This collector is used by {@link IterationIntermediatePactTask} or {@link IterationTailPactTask} to update the
* solution set of workset iterations. Depending on the task configuration, either a fast (non-probing)
* {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing)
* {@link SolutionSetUpdateOutputCollector} is created.
- * <p/>
+ * <p>
* If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call
* collect(T) of the delegate.
*
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
index ea8efb6..b8e4857 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadPactTask.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.flink.runtime.io.network.api.writer.BufferWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,7 +36,6 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.io.disk.InputViewIterator;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannelBroker;
import org.apache.flink.runtime.iterative.concurrent.Broker;
@@ -84,7 +85,7 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
private Collector<X> finalOutputCollector;
- private List<BufferWriter> finalOutputWriters;
+ private List<RecordWriter<?>> finalOutputWriters;
private TypeSerializerFactory<Y> feedbackTypeSerializer;
@@ -114,11 +115,11 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
// at this time, the outputs to the step function are created
// add the outputs for the final solution
- this.finalOutputWriters = new ArrayList<BufferWriter>();
+ this.finalOutputWriters = new ArrayList<RecordWriter<?>>();
final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
final ClassLoader userCodeClassLoader = getUserCodeClassLoader();
this.finalOutputCollector = RegularPactTask.getOutputCollector(this, finalOutConfig,
- userCodeClassLoader, this.finalOutputWriters, finalOutConfig.getNumOutputs());
+ userCodeClassLoader, this.finalOutputWriters, config.getNumOutputs(), finalOutConfig.getNumOutputs());
// sanity check the setup
final int writersIntoStepFunction = this.eventualOutputs.size();
@@ -129,7 +130,7 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates.");
}
// now, we can instantiate the sync gate
- this.toSync = new BufferWriter(this);
+ this.toSync = getEnvironment().getWriter(syncGateIndex);
}
/**
@@ -233,9 +234,6 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
@Override
public void run() throws Exception {
- // initialize the serializers (one per channel) of the record writers
- RegularPactTask.initOutputWriters(this.finalOutputWriters);
-
final String brokerKey = brokerKey();
final int workerIndex = getEnvironment().getIndexInSubtaskGroup();
@@ -445,6 +443,6 @@ public class IterationHeadPactTask<X, Y, S extends Function, OT> extends Abstrac
if (log.isInfoEnabled()) {
log.info(formatLogString("sending " + WorkerDoneEvent.class.getSimpleName() + " to sync"));
}
- this.toSync.broadcastEvent(event);
+ this.toSync.writeEventToAllChannels(event);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
index 4bdb3cf..e7801e4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationIntermediatePactTask.java
@@ -18,26 +18,26 @@
package org.apache.flink.runtime.iterative.task;
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
-import org.apache.flink.runtime.io.network.channels.EndOfSuperstepEvent;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
+import org.apache.flink.runtime.io.network.api.EndOfSuperstepEvent;
import org.apache.flink.runtime.iterative.concurrent.BlockingBackChannel;
import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatch;
import org.apache.flink.runtime.iterative.concurrent.SuperstepKickoffLatchBroker;
import org.apache.flink.runtime.iterative.event.TerminationEvent;
import org.apache.flink.runtime.iterative.io.WorksetUpdateOutputCollector;
import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
/**
* An intermediate iteration task, which runs a Driver}inside.
- * <p/>
+ * <p>
* It will propagate {@link EndOfSuperstepEvent}s and {@link TerminationEvent}s to it's connected tasks. Furthermore
* intermediate tasks can also update the iteration state, either the workset or the solution set.
- * <p/>
+ * <p>
* If the iteration state is updated, the output of this task will be send back to the {@link IterationHeadPactTask} via
* a {@link BlockingBackChannel} for the workset -XOR- a eHashTable for the solution set. In this case
* this task must be scheduled on the same instance as the head.
@@ -123,7 +123,7 @@ public class IterationIntermediatePactTask<S extends Function, OT> extends Abstr
}
private void sendEndOfSuperstep() throws IOException, InterruptedException {
- for (BufferWriter eventualOutput : this.eventualOutputs) {
+ for (RecordWriter eventualOutput : this.eventualOutputs) {
eventualOutput.sendEndOfSuperstep();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
index 91ae1e9..6ce615d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationSynchronizationSinkTask.java
@@ -24,13 +24,13 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.flink.runtime.event.task.TaskEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.aggregators.Aggregator;
import org.apache.flink.api.common.aggregators.AggregatorWithName;
import org.apache.flink.api.common.aggregators.ConvergenceCriterion;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
-import org.apache.flink.runtime.io.network.api.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
import org.apache.flink.runtime.iterative.event.AllWorkersDoneEvent;
import org.apache.flink.runtime.iterative.event.TerminationEvent;
import org.apache.flink.runtime.iterative.event.WorkerDoneEvent;
@@ -73,7 +73,7 @@ public class IterationSynchronizationSinkTask extends AbstractInvokable implemen
@Override
public void registerInputOutput() {
- this.headEventReader = new MutableRecordReader<IntegerRecord>(this);
+ this.headEventReader = new MutableRecordReader<IntegerRecord>(getEnvironment().getReader(0));
}
@Override
@@ -99,7 +99,7 @@ public class IterationSynchronizationSinkTask extends AbstractInvokable implemen
int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators,
getEnvironment().getUserClassLoader());
- headEventReader.subscribeToEvent(eventHandler, WorkerDoneEvent.class);
+ headEventReader.subscribeToTaskEvent(eventHandler, WorkerDoneEvent.class);
IntegerRecord dummy = new IntegerRecord();
@@ -199,8 +199,8 @@ public class IterationSynchronizationSinkTask extends AbstractInvokable implemen
}
}
- private void sendToAllWorkers(AbstractTaskEvent event) throws IOException, InterruptedException {
- headEventReader.publishEvent(event);
+ private void sendToAllWorkers(TaskEvent event) throws IOException, InterruptedException {
+ headEventReader.sendTaskEvent(event);
}
private String formatLogString(String message) {
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
index f7db9d1..159d3f2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationTailPactTask.java
@@ -30,12 +30,12 @@ import org.apache.flink.util.Collector;
/**
* An iteration tail, which runs a driver inside.
- * <p/>
+ * <p>
* If the iteration state is updated, the output of this task will be send back to the {@link IterationHeadPactTask} via
* a BackChannel for the workset -OR- a HashTable for the solution set. Therefore this
* task must be scheduled on the same instance as the head. It's also possible for the tail to update *both* the workset
* and the solution set.
- * <p/>
+ * <p>
* If there is a separate solution set tail, the iteration head has to make sure to wait for it to finish.
*/
public class IterationTailPactTask<S extends Function, OT> extends AbstractIterativePactTask<S, OT> {
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
index 780b34b..8cce6ef 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/SyncEventHandler.java
@@ -21,14 +21,14 @@ package org.apache.flink.runtime.iterative.task;
import java.util.Map;
import org.apache.flink.api.common.aggregators.Aggregator;
-import org.apache.flink.runtime.event.task.AbstractTaskEvent;
-import org.apache.flink.runtime.event.task.EventListener;
+import org.apache.flink.runtime.event.task.TaskEvent;
+import org.apache.flink.runtime.util.event.EventListener;
import org.apache.flink.runtime.iterative.event.WorkerDoneEvent;
import org.apache.flink.types.Value;
import com.google.common.base.Preconditions;
-public class SyncEventHandler implements EventListener {
+public class SyncEventHandler implements EventListener<TaskEvent> {
private final ClassLoader userCodeClassLoader;
@@ -49,7 +49,7 @@ public class SyncEventHandler implements EventListener {
}
@Override
- public void eventOccurred(AbstractTaskEvent event) {
+ public void onEvent(TaskEvent event) {
if (WorkerDoneEvent.class.equals(event.getClass())) {
onWorkerDoneEvent((WorkerDoneEvent) event);
return;
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
index 5a9f308..29a6581 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/DistributionPattern.java
@@ -18,19 +18,25 @@
package org.apache.flink.runtime.jobgraph;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+
/**
- * A distribution pattern determines which subtasks of a producing Nephele task a wired to which
- * subtasks of a consuming subtask.
+ * A distribution pattern determines, which sub tasks of a producing task are connected to which
+ * consuming sub tasks.
*/
public enum DistributionPattern {
/**
- * Each subtask of the producing Nephele task is wired to each subtask of the consuming Nephele task.
+ * Each producing sub task is connected to each sub task of the consuming task.
+ * <p>
+ * {@link ExecutionVertex#connectAllToAll(org.apache.flink.runtime.executiongraph.IntermediateResultPartition[], int)}
*/
- BIPARTITE,
+ ALL_TO_ALL,
/**
- * The i-th subtask of the producing Nephele task is wired to the i-th subtask of the consuming Nephele task.
+ * Each producing sub task is connected to one or more subtask(s) of the consuming task.
+ * <p>
+ * {@link ExecutionVertex#connectPointwise(org.apache.flink.runtime.executiongraph.IntermediateResultPartition[], int)}
*/
POINTWISE
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java
new file mode 100644
index 0000000..190845d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionID.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.flink.runtime.AbstractID;
+
+public class IntermediateResultPartitionID extends AbstractID {
+
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Creates an new random intermediate result partition ID.
+ */
+ public IntermediateResultPartitionID() {
+ super();
+ }
+
+ public IntermediateResultPartitionID(long lowerPart, long upperPart) {
+ super(lowerPart, upperPart);
+ }
+
+ public static IntermediateResultPartitionID fromByteBuf(ByteBuf buf) {
+ long lower = buf.readLong();
+ long upper = buf.readLong();
+ return new IntermediateResultPartitionID(lower, upper);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java
new file mode 100644
index 0000000..25d057e
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/IntermediateResultPartitionType.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph;
+
+public enum IntermediateResultPartitionType {
+
+ BLOCKING(true, false, false),
+
+ PIPELINED(false, true, true),
+ PIPELINED_PERSISTENT(true, true, true);
+
+ private final boolean isPersistent;
+
+ private final boolean isPipelined;
+
+ private final boolean hasBackPressure;
+
+ IntermediateResultPartitionType(boolean isPersistent, boolean isPipelined, boolean hasBackPressure) {
+ this.isPersistent = isPersistent;
+ this.isPipelined = isPipelined;
+ this.hasBackPressure = hasBackPressure;
+ }
+
+ public boolean hasBackPressure() {
+ return hasBackPressure;
+ }
+
+ public boolean isPipelined() {
+ return isPipelined;
+ }
+
+ public boolean isPersistent() {
+ return isPersistent;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
index 842fe62..dc5d9ae 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobID.java
@@ -55,12 +55,6 @@ public final class JobID extends AbstractID {
return new JobID(lower, upper);
}
- public static JobID fromByteBuffer(ByteBuffer buf, int offset) {
- long lower = buf.getLong(offset);
- long upper = buf.getLong(offset + 8);
- return new JobID(lower, upper);
- }
-
public static JobID fromHexString(String hexString) {
return new JobID(DatatypeConverter.parseHexBinary(hexString));
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
index 1666788..ff1b272 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
@@ -80,7 +80,7 @@ public abstract class AbstractInvokable {
* @return the current number of subtasks the respective task is split into
*/
public final int getCurrentNumberOfSubtasks() {
- return this.environment.getCurrentNumberOfSubtasks();
+ return this.environment.getNumberOfSubtasks();
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
index 8d48975..103c1be 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/web/JsonFactory.java
@@ -18,18 +18,17 @@
package org.apache.flink.runtime.jobmanager.web;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.ExecutionVertex;
import org.apache.flink.runtime.executiongraph.IntermediateResult;
import org.apache.flink.runtime.instance.AllocatedSlot;
-import org.apache.flink.runtime.io.network.channels.ChannelType;
import org.apache.flink.util.StringUtils;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
public class JsonFactory {
public static String toJson(ExecutionVertex vertex) {
@@ -92,8 +91,7 @@ public class JsonFactory {
json.append("{");
json.append("\"groupvertexid\": \"" + input.getJobVertexId() + "\",");
- json.append("\"groupvertexname\": \"" + StringUtils.escapeHtml(jobVertex.getJobVertex().getName()) + "\",");
- json.append("\"channelType\": \"" + ChannelType.NETWORK + "\"");
+ json.append("\"groupvertexname\": \"" + StringUtils.escapeHtml(jobVertex.getJobVertex().getName()) + "\"");
json.append("}");
// print delimiter
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
index e3ffdba..be4453d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSinkTask.java
@@ -16,14 +16,9 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators;
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.runtime.plugable.DeserializationDelegate;
-import org.apache.flink.util.InstantiationUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.io.CleanupWhenUnsuccessful;
import org.apache.flink.api.common.io.OutputFormat;
import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
@@ -32,16 +27,20 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.IOReadableWritable;
import org.apache.flink.runtime.execution.CancelTaskException;
-import org.apache.flink.runtime.io.network.api.MutableReader;
-import org.apache.flink.runtime.io.network.api.MutableRecordReader;
-import org.apache.flink.runtime.io.network.api.MutableUnionRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
import org.apache.flink.runtime.operators.sort.UnilateralSortMerger;
import org.apache.flink.runtime.operators.util.CloseableInputProvider;
import org.apache.flink.runtime.operators.util.ReaderIterator;
import org.apache.flink.runtime.operators.util.TaskConfig;
+import org.apache.flink.runtime.plugable.DeserializationDelegate;
+import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.MutableObjectIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
@@ -59,13 +58,12 @@ public class DataSinkTask<IT> extends AbstractInvokable {
// OutputFormat instance. volatile, because the asynchronous canceller may access it
private volatile OutputFormat<IT> format;
-
+
+ private MutableReader<?> inputReader;
+
// input reader
private MutableObjectIterator<IT> reader;
-
- // input iterator
- private MutableObjectIterator<IT> input;
-
+
// The serializer for the input type
private TypeSerializerFactory<IT> inputTypeSerializerFactory;
@@ -129,11 +127,12 @@ public class DataSinkTask<IT> extends AbstractInvokable {
try {
// initialize local strategies
+ MutableObjectIterator<IT> input1;
switch (this.config.getInputLocalStrategy(0)) {
case NONE:
// nothing to do
localStrategy = null;
- input = reader;
+ input1 = reader;
break;
case SORT:
// initialize sort local strategy
@@ -154,7 +153,7 @@ public class DataSinkTask<IT> extends AbstractInvokable {
this.config.getSpillingThresholdInput(0));
this.localStrategy = sorter;
- this.input = sorter.getIterator();
+ input1 = sorter.getIterator();
} catch (Exception e) {
throw new RuntimeException("Initializing the input processing failed" +
e.getMessage() == null ? "." : ": " + e.getMessage(), e);
@@ -167,7 +166,7 @@ public class DataSinkTask<IT> extends AbstractInvokable {
// read the reader and write it to the output
final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer();
- final MutableObjectIterator<IT> input = this.input;
+ final MutableObjectIterator<IT> input = input1;
final OutputFormat<IT> format = this.format;
@@ -181,7 +180,7 @@ public class DataSinkTask<IT> extends AbstractInvokable {
}
// open
- format.open(this.getEnvironment().getIndexInSubtaskGroup(), this.getEnvironment().getCurrentNumberOfSubtasks());
+ format.open(this.getEnvironment().getIndexInSubtaskGroup(), this.getEnvironment().getNumberOfSubtasks());
if (objectReuseEnabled) {
IT record = serializer.createInstance();
@@ -253,6 +252,8 @@ public class DataSinkTask<IT> extends AbstractInvokable {
LOG.error("Error closing local strategy", t);
}
}
+
+ RegularPactTask.clearReaders(new MutableReader[]{inputReader});
}
if (!this.taskCanceled) {
@@ -337,9 +338,6 @@ public class DataSinkTask<IT> extends AbstractInvokable {
*/
@SuppressWarnings("unchecked")
private void initInputReaders() throws Exception {
-
- MutableReader<?> inputReader;
-
int numGates = 0;
// ---------------- create the input readers ---------------------
// in case where a logical input unions multiple physical inputs, create a union reader
@@ -347,25 +345,20 @@ public class DataSinkTask<IT> extends AbstractInvokable {
numGates += groupSize;
if (groupSize == 1) {
// non-union case
- inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(this);
+ inputReader = new MutableRecordReader<DeserializationDelegate<IT>>(getEnvironment().getReader(0));
} else if (groupSize > 1){
// union case
-
- MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
- for (int j = 0; j < groupSize; ++j) {
- readers[j] = new MutableRecordReader<IOReadableWritable>(this);
- }
- inputReader = new MutableUnionRecordReader<IOReadableWritable>(readers);
+ UnionBufferReader reader = new UnionBufferReader(getEnvironment().getAllReaders());
+ inputReader = new MutableRecordReader<IOReadableWritable>(reader);
} else {
throw new Exception("Illegal input group size in task configuration: " + groupSize);
}
-
+
this.inputTypeSerializerFactory = this.config.getInputSerializer(0, getUserCodeClassLoader());
-
@SuppressWarnings({ "rawtypes" })
final MutableObjectIterator<?> iter = new ReaderIterator(inputReader, this.inputTypeSerializerFactory.getSerializer());
this.reader = (MutableObjectIterator<IT>)iter;
-
+
// final sanity check
if (numGates != this.config.getNumInputs()) {
throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
index b50bf36..330f438 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/DataSourceTask.java
@@ -18,17 +18,7 @@
package org.apache.flink.runtime.operators;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.util.InstantiationUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.api.common.typeutils.TypeSerializer;
@@ -36,13 +26,23 @@ import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.InputSplit;
import org.apache.flink.runtime.execution.CancelTaskException;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.operators.chaining.ChainedDriver;
import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
import org.apache.flink.runtime.operators.util.TaskConfig;
import org.apache.flink.util.Collector;
+import org.apache.flink.util.InstantiationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
/**
* DataSourceTask which is executed by a task manager. The task reads data and uses an
@@ -51,11 +51,10 @@ import org.apache.flink.util.Collector;
* @see org.apache.flink.api.common.io.InputFormat
*/
public class DataSourceTask<OT> extends AbstractInvokable {
-
+
private static final Logger LOG = LoggerFactory.getLogger(DataSourceTask.class);
-
- private List<BufferWriter> eventualOutputs;
+ private List<RecordWriter<?>> eventualOutputs;
// Output collector
private Collector<OT> output;
@@ -145,9 +144,6 @@ public class DataSourceTask<OT> extends AbstractInvokable {
final TypeSerializer<OT> serializer = this.serializerFactory.getSerializer();
try {
- // initialize the serializers (one per channel) of the record writers
- RegularPactTask.initOutputWriters(this.eventualOutputs);
-
// start all chained tasks
RegularPactTask.openChainedTasks(this.chainedTasks, this);
@@ -159,7 +155,7 @@ public class DataSourceTask<OT> extends AbstractInvokable {
{
// get start and end
final InputSplit split = splitIterator.next();
-
+
if (LOG.isDebugEnabled()) {
LOG.debug(getLogString("Opening input split " + split.toString()));
}
@@ -174,7 +170,6 @@ public class DataSourceTask<OT> extends AbstractInvokable {
}
try {
-
final Collector<OT> output = this.output;
if (objectReuseEnabled) {
@@ -222,7 +217,7 @@ public class DataSourceTask<OT> extends AbstractInvokable {
// close the input, but do not report any exceptions, since we already have another root cause
try {
this.format.close();
- } catch (Throwable t) {}
+ } catch (Throwable ignored) {}
RegularPactTask.cancelChainedTasks(this.chainedTasks);
@@ -236,6 +231,8 @@ public class DataSourceTask<OT> extends AbstractInvokable {
// drop exception, if the task was canceled
RegularPactTask.logAndThrowException(ex, this);
}
+ } finally {
+ RegularPactTask.clearWriters(eventualOutputs);
}
if (!this.taskCanceled) {
@@ -304,7 +301,7 @@ public class DataSourceTask<OT> extends AbstractInvokable {
*/
private void initOutputs(ClassLoader cl) throws Exception {
this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
- this.eventualOutputs = new ArrayList<BufferWriter>();
+ this.eventualOutputs = new ArrayList<RecordWriter<?>>();
this.output = RegularPactTask.initOutputs(this, cl, this.config, this.chainedTasks, this.eventualOutputs, getExecutionConfig());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index 34aaced..2d36c1e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -19,10 +19,7 @@
package org.apache.flink.runtime.operators;
import akka.actor.ActorRef;
-import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.api.common.ExecutionConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.api.common.accumulators.AccumulatorHelper;
import org.apache.flink.api.common.distributions.DataDistribution;
@@ -40,14 +37,15 @@ import org.apache.flink.runtime.broadcast.BroadcastVariableMaterialization;
import org.apache.flink.runtime.execution.CancelTaskException;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.network.api.BufferWriter;
-import org.apache.flink.runtime.io.network.api.ChannelSelector;
-import org.apache.flink.runtime.io.network.api.MutableReader;
-import org.apache.flink.runtime.io.network.api.MutableRecordReader;
-import org.apache.flink.runtime.io.network.api.MutableUnionRecordReader;
-import org.apache.flink.runtime.io.network.api.RecordWriter;
+import org.apache.flink.runtime.io.network.api.reader.BufferReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableRecordReader;
+import org.apache.flink.runtime.io.network.api.reader.UnionBufferReader;
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.memorymanager.MemoryManager;
+import org.apache.flink.runtime.messages.JobManagerMessages;
import org.apache.flink.runtime.operators.chaining.ChainedDriver;
import org.apache.flink.runtime.operators.chaining.ExceptionInChainedStubException;
import org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator;
@@ -69,6 +67,8 @@ import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
import org.apache.flink.util.InstantiationUtil;
import org.apache.flink.util.MutableObjectIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
@@ -111,7 +111,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
* The output writers for the data that this task forwards to the next task. The latest driver (the central, if no chained
* drivers exist, otherwise the last chained driver) produces its output to these writers.
*/
- protected List<BufferWriter> eventualOutputs;
+ protected List<RecordWriter<?>> eventualOutputs;
/**
* The input readers to this task.
@@ -278,9 +278,6 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// so that the thread that creates them catches its own errors that may happen in that process.
// this is especially important, since there may be asynchronous closes (such as through canceling).
try {
- // initialize the serializers (one per channel) of the record writers
- initOutputWriters(this.eventualOutputs);
-
// initialize the remaining data structures on the input and trigger the local processing
// the local processing includes building the dams / caches
try {
@@ -302,7 +299,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
throw new IllegalArgumentException();
}
else if (numberOfEventsUntilInterrupt > 0) {
- this.inputReaders[i].setIterative(numberOfEventsUntilInterrupt);
+ this.inputReaders[i].setIterativeReader();
iterativeInputs.add(i);
if (LOG.isDebugEnabled()) {
@@ -324,7 +321,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
throw new IllegalArgumentException();
}
else if (numberOfEventsUntilInterrupt > 0) {
- this.broadcastInputReaders[i].setIterative(numberOfEventsUntilInterrupt);
+ this.broadcastInputReaders[i].setIterativeReader();
iterativeBcInputs.add(i);
if (LOG.isDebugEnabled()) {
@@ -365,6 +362,9 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
finally {
// clean up in any case!
closeLocalStrategiesAndCaches();
+
+ clearReaders(inputReaders);
+ clearWriters(eventualOutputs);
}
if (this.running) {
@@ -564,14 +564,12 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
* Each chained task might have accumulators which will be merged
* with the accumulators of the stub.
*/
- protected static void reportAndClearAccumulators(Environment env, Map<String, Accumulator<?, ?>> accumulators,
- ArrayList<ChainedDriver<?, ?>> chainedTasks) {
+ protected static void reportAndClearAccumulators(
+ Environment env, Map<String, Accumulator<?, ?>> accumulators, ArrayList<ChainedDriver<?, ?>> chainedTasks) {
// We can merge here the accumulators from the stub and the chained
// tasks. Type conflicts can occur here if counters with same name but
// different type were used.
-
-
for (ChainedDriver<?, ?> chainedTask : chainedTasks) {
if (FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null) != null) {
Map<String, Accumulator<?, ?>> chainedAccumulators = FunctionUtils.getFunctionRuntimeContext(chainedTask.getStub(), null).getAllAccumulators();
@@ -587,7 +585,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// Report accumulators to JobManager
JobManagerMessages.ReportAccumulatorResult accResult = new JobManagerMessages.ReportAccumulatorResult(new
AccumulatorEvent(env.getJobID(), AccumulatorHelper.copy(accumulators)));
- env.getAccumulator().tell(accResult, ActorRef.noSender());
+ env.getJobManager().tell(accResult, ActorRef.noSender());
// We also clear the accumulators, since stub instances might be reused
// (e.g. in iterations) and we don't want to count twice. This may not be
@@ -664,7 +662,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
/**
* Sets the last output {@link Collector} of the collector chain of this {@link RegularPactTask}.
- * <p/>
+ * <p>
* In case of chained tasks, the output collector of the last {@link ChainedDriver} is set. Otherwise it is the
* single collector of the {@link RegularPactTask}.
*
@@ -713,31 +711,34 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
final int numInputs = getNumTaskInputs();
final MutableReader<?>[] inputReaders = new MutableReader[numInputs];
- int numGates = 0;
+ int currentReaderOffset = 0;
for (int i = 0; i < numInputs; i++) {
// ---------------- create the input readers ---------------------
// in case where a logical input unions multiple physical inputs, create a union reader
final int groupSize = this.config.getGroupSize(i);
- numGates += groupSize;
+
if (groupSize == 1) {
// non-union case
- inputReaders[i] = new MutableRecordReader<IOReadableWritable>(this);
+ inputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getReader(currentReaderOffset));
} else if (groupSize > 1){
// union case
- MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
+ BufferReader[] readers = new BufferReader[groupSize];
for (int j = 0; j < groupSize; ++j) {
- readers[j] = new MutableRecordReader<IOReadableWritable>(this);
+ readers[j] = getEnvironment().getReader(currentReaderOffset + j);
}
- inputReaders[i] = new MutableUnionRecordReader<IOReadableWritable>(readers);
+ UnionBufferReader reader = new UnionBufferReader(readers);
+ inputReaders[i] = new MutableRecordReader<IOReadableWritable>(reader);
} else {
throw new Exception("Illegal input group size in task configuration: " + groupSize);
}
+
+ currentReaderOffset += groupSize;
}
this.inputReaders = inputReaders;
// final sanity check
- if (numGates != this.config.getNumInputs()) {
+ if (currentReaderOffset != this.config.getNumInputs()) {
throw new Exception("Illegal configuration: Number of input gates and group sizes are not consistent.");
}
}
@@ -751,24 +752,29 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
protected void initBroadcastInputReaders() throws Exception {
final int numBroadcastInputs = this.config.getNumBroadcastInputs();
final MutableReader<?>[] broadcastInputReaders = new MutableReader[numBroadcastInputs];
-
+
+ int currentReaderOffset = config.getNumInputs();
+
for (int i = 0; i < this.config.getNumBroadcastInputs(); i++) {
// ---------------- create the input readers ---------------------
// in case where a logical input unions multiple physical inputs, create a union reader
final int groupSize = this.config.getBroadcastGroupSize(i);
if (groupSize == 1) {
// non-union case
- broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(this);
+ broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(getEnvironment().getReader(currentReaderOffset));
} else if (groupSize > 1){
// union case
- MutableRecordReader<IOReadableWritable>[] readers = new MutableRecordReader[groupSize];
+ BufferReader[] readers = new BufferReader[groupSize];
for (int j = 0; j < groupSize; ++j) {
- readers[j] = new MutableRecordReader<IOReadableWritable>(this);
+ readers[j] = getEnvironment().getReader(currentReaderOffset + j);
}
- broadcastInputReaders[i] = new MutableUnionRecordReader<IOReadableWritable>(readers);
+ UnionBufferReader reader = new UnionBufferReader(readers);
+ broadcastInputReaders[i] = new MutableRecordReader<IOReadableWritable>(reader);
} else {
throw new Exception("Illegal input group size in task configuration: " + groupSize);
}
+
+ currentReaderOffset += groupSize;
}
this.broadcastInputReaders = broadcastInputReaders;
}
@@ -853,8 +859,8 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
for (int i = 0; i < numInputs; i++) {
final int memoryPages;
- final boolean cached = this.config.isInputCached(i);
final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
+ final boolean cached = this.config.isInputCached(i);
this.inputIsAsyncMaterialized[i] = async;
this.inputIsCached[i] = cached;
@@ -886,16 +892,16 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
}
protected void resetAllInputs() throws Exception {
-
+
// first we need to make sure that caches consume remaining data
// NOTE: we need to do this before closing the local strategies
for (int i = 0; i < this.inputs.length; i++) {
-
+
if (this.inputIsCached[i] && this.resettableInputs[i] != null) {
this.resettableInputs[i].consumeAndCacheRemainingData();
}
}
-
+
// close all local-strategies. they will either get re-initialized, or we have
// read them now and their data is cached
for (int i = 0; i < this.localStrategies.length; i++) {
@@ -1054,7 +1060,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*/
protected void initOutputs() throws Exception {
this.chainedTasks = new ArrayList<ChainedDriver<?, ?>>();
- this.eventualOutputs = new ArrayList<BufferWriter>();
+ this.eventualOutputs = new ArrayList<RecordWriter<?>>();
ClassLoader userCodeClassLoader = getUserCodeClassLoader();
@@ -1063,7 +1069,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
public DistributedRuntimeUDFContext createRuntimeContext(String taskName) {
Environment env = getEnvironment();
- return new DistributedRuntimeUDFContext(taskName, env.getCurrentNumberOfSubtasks(),
+ return new DistributedRuntimeUDFContext(taskName, env.getNumberOfSubtasks(),
env.getIndexInSubtaskGroup(), getUserCodeClassLoader(), env.getCopyTask());
}
@@ -1208,7 +1214,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*/
public static String constructLogString(String message, String taskName, AbstractInvokable parent) {
return message + ": " + taskName + " (" + (parent.getEnvironment().getIndexInSubtaskGroup() + 1) +
- '/' + parent.getEnvironment().getCurrentNumberOfSubtasks() + ')';
+ '/' + parent.getEnvironment().getNumberOfSubtasks() + ')';
}
/**
@@ -1255,7 +1261,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*
* @return The OutputCollector that data produced in this task is submitted to.
*/
- public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl, List<BufferWriter> eventualOutputs, int numOutputs)
+ public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl, List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs)
throws Exception
{
if (numOutputs == 0) {
@@ -1289,7 +1295,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
oe = new RecordOutputEmitter(strategy, comparator, partitioner, distribution);
}
- writers.add(new RecordWriter<Record>(task, oe));
+ writers.add(new RecordWriter<Record>(task.getEnvironment().getWriter(outputOffset + i), oe));
}
if (eventualOutputs != null) {
eventualOutputs.addAll(writers);
@@ -1321,7 +1327,8 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
final TypeComparator<T> comparator = compFactory.createComparator();
oe = new OutputEmitter<T>(strategy, comparator, partitioner, dataDist);
}
- writers.add(new RecordWriter<SerializationDelegate<T>>(task, oe));
+
+ writers.add(new RecordWriter<SerializationDelegate<T>>(task.getEnvironment().getWriter(outputOffset + i), oe));
}
if (eventualOutputs != null) {
eventualOutputs.addAll(writers);
@@ -1336,7 +1343,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
*/
@SuppressWarnings("unchecked")
public static <T> Collector<T> initOutputs(AbstractInvokable nepheleTask, ClassLoader cl, TaskConfig config,
- List<ChainedDriver<?, ?>> chainedTasksTarget, List<BufferWriter> eventualOutputs, ExecutionConfig executionConfig)
+ List<ChainedDriver<?, ?>> chainedTasksTarget, List<RecordWriter<?>> eventualOutputs, ExecutionConfig executionConfig)
throws Exception
{
final int numOutputs = config.getNumOutputs();
@@ -1370,7 +1377,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
if (i == numChained -1) {
// last in chain, instantiate the output collector for this task
- previous = getOutputCollector(nepheleTask, chainedStubConf, cl, eventualOutputs, chainedStubConf.getNumOutputs());
+ previous = getOutputCollector(nepheleTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs());
}
ct.setup(chainedStubConf, taskName, previous, nepheleTask, cl, executionConfig);
@@ -1384,13 +1391,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// else
// instantiate the output collector the default way from this configuration
- return getOutputCollector(nepheleTask , config, cl, eventualOutputs, numOutputs);
- }
-
- public static void initOutputWriters(List<BufferWriter> writers) {
- for (BufferWriter writer : writers) {
- ((RecordWriter<?>) writer).initializeSerializers();
- }
+ return getOutputCollector(nepheleTask , config, cl, eventualOutputs, 0, numOutputs);
}
// --------------------------------------------------------------------------------------------
@@ -1528,4 +1529,16 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
}
return a;
}
+
+ public static void clearWriters(List<RecordWriter<?>> writers) {
+ for (RecordWriter<?> writer : writers) {
+ writer.clearBuffers();
+ }
+ }
+
+ public static void clearReaders(MutableReader<?>[] readers) {
+ for (MutableReader<?> reader : readers) {
+ reader.clearBuffers();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
index c36e742..838286e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
@@ -61,7 +61,7 @@ public abstract class ChainedDriver<IT, OT> implements Collector<IT> {
this.udfContext = ((RegularPactTask<?, ?>) parent).createRuntimeContext(taskName);
} else {
Environment env = parent.getEnvironment();
- this.udfContext = new DistributedRuntimeUDFContext(taskName, env.getCurrentNumberOfSubtasks(),
+ this.udfContext = new DistributedRuntimeUDFContext(taskName, env.getNumberOfSubtasks(),
env.getIndexInSubtaskGroup(), userCodeClassLoader, env.getCopyTask());
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
index 5b0cdf1..56723df 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputCollector.java
@@ -25,7 +25,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.io.network.api.RecordWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.runtime.plugable.SerializationDelegate;
import org.apache.flink.util.Collector;
@@ -103,8 +103,6 @@ public class OutputCollector<T> implements Collector<T>
writer.flush();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
- } catch (InterruptedException e) {
- throw new RuntimeException(e.getMessage(), e);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
index ec92e3f..ff1a148 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.shipping;
import org.apache.flink.api.common.distributions.DataDistribution;
import org.apache.flink.api.common.functions.Partitioner;
import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.runtime.io.network.api.ChannelSelector;
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
import org.apache.flink.runtime.plugable.SerializationDelegate;
public class OutputEmitter<T> implements ChannelSelector<SerializationDelegate<T>> {
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
index 845591b..dfd58dc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputCollector.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.apache.flink.runtime.io.network.api.RecordWriter;
+import org.apache.flink.runtime.io.network.api.writer.RecordWriter;
import org.apache.flink.types.Record;
import org.apache.flink.util.Collector;
@@ -97,8 +97,6 @@ public class RecordOutputCollector implements Collector<Record>
writer.flush();
} catch (IOException e) {
throw new RuntimeException(e.getMessage(), e);
- } catch (InterruptedException e) {
- throw new RuntimeException(e.getMessage(), e);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
index 455ebbf..300d73f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/RecordOutputEmitter.java
@@ -22,7 +22,7 @@ package org.apache.flink.runtime.operators.shipping;
import org.apache.flink.api.common.distributions.DataDistribution;
import org.apache.flink.api.common.functions.Partitioner;
import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.runtime.io.network.api.ChannelSelector;
+import org.apache.flink.runtime.io.network.api.writer.ChannelSelector;
import org.apache.flink.types.Key;
import org.apache.flink.types.Record;
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
index 85e36a4..7c00c53 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/ReaderIterator.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.util;
import java.io.IOException;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.runtime.io.network.api.MutableReader;
+import org.apache.flink.runtime.io.network.api.reader.MutableReader;
import org.apache.flink.runtime.plugable.DeserializationDelegate;
import org.apache.flink.runtime.plugable.NonReusingDeserializationDelegate;
import org.apache.flink.runtime.plugable.ReusingDeserializationDelegate;
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
index 226abf0..e4a131d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/SimpleCloseableInputProvider.java
@@ -16,17 +16,14 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.util;
import java.io.IOException;
import org.apache.flink.util.MutableObjectIterator;
-
/**
* A simple iterator provider that returns a supplied iterator and does nothing when closed.
- *
*/
public class SimpleCloseableInputProvider<E> implements CloseableInputProvider<E>
{
@@ -34,8 +31,7 @@ public class SimpleCloseableInputProvider<E> implements CloseableInputProvider<E
* The iterator returned by this class.
*/
private final MutableObjectIterator<E> iterator;
-
-
+
/**
* Creates a new simple input provider that will return the given iterator.
*
@@ -44,15 +40,12 @@ public class SimpleCloseableInputProvider<E> implements CloseableInputProvider<E
public SimpleCloseableInputProvider(MutableObjectIterator<E> iterator) {
this.iterator = iterator;
}
-
-
@Override
public void close() throws IOException {
// do nothing
}
-
@Override
public MutableObjectIterator<E> getIterator() {
return this.iterator;
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 8570992..5b6259a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -18,46 +18,48 @@
package org.apache.flink.runtime.taskmanager;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-
import akka.actor.ActorRef;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.execution.RuntimeEnvironment;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.io.network.api.reader.BufferReader;
+import org.apache.flink.runtime.io.network.api.writer.BufferWriter;
+import org.apache.flink.runtime.io.network.partition.IntermediateResultPartition;
import org.apache.flink.runtime.jobgraph.JobID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.memorymanager.MemoryManager;
import org.apache.flink.runtime.messages.ExecutionGraphMessages;
import org.apache.flink.runtime.profiling.TaskManagerProfiler;
import org.apache.flink.util.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
public final class Task {
/** For atomic state updates */
- private static final AtomicReferenceFieldUpdater<Task, ExecutionState> STATE_UPDATER =
+ private static final AtomicReferenceFieldUpdater<Task, ExecutionState> STATE_UPDATER =
AtomicReferenceFieldUpdater.newUpdater(Task.class, ExecutionState.class, "executionState");
-
+
/** The log object used for debugging. */
private static final Logger LOG = LoggerFactory.getLogger(Task.class);
// --------------------------------------------------------------------------------------------
-
+
private final JobID jobId;
-
+
private final JobVertexID vertexId;
-
+
private final int subtaskIndex;
-
+
private final int numberOfSubtasks;
-
+
private final ExecutionAttemptID executionId;
-
+
private final String taskName;
private final TaskManager taskManager;
@@ -66,15 +68,15 @@ public final class Task {
/** The environment (with the invokable) executed by this task */
private volatile RuntimeEnvironment environment;
-
+
/** The current execution state of the task */
private volatile ExecutionState executionState = ExecutionState.DEPLOYING;
// --------------------------------------------------------------------------------------------
-
- public Task(JobID jobId, JobVertexID vertexId, int taskIndex, int parallelism,
- ExecutionAttemptID executionId, String taskName, TaskManager taskManager)
- {
+
+ public Task(JobID jobId, JobVertexID vertexId, int taskIndex, int parallelism,
+ ExecutionAttemptID executionId, String taskName, TaskManager taskManager) {
+
this.jobId = jobId;
this.vertexId = vertexId;
this.subtaskIndex = taskIndex;
@@ -84,11 +86,8 @@ public final class Task {
this.taskManager = taskManager;
}
-
/**
* Returns the ID of the job this task belongs to.
- *
- * @return the ID of the job this task belongs to
*/
public JobID getJobID() {
return this.jobId;
@@ -96,8 +95,6 @@ public final class Task {
/**
* Returns the ID of this task vertex.
- *
- * @return the ID of this task vertex.
*/
public JobVertexID getVertexID() {
return this.vertexId;
@@ -105,71 +102,63 @@ public final class Task {
/**
* Gets the index of the parallel subtask [0, parallelism).
- *
- * @return The task index of the parallel subtask.
*/
public int getSubtaskIndex() {
return subtaskIndex;
}
-
+
/**
* Gets the total number of subtasks of the task that this subtask belongs to.
- *
- * @return The total number of this task's subtasks.
*/
public int getNumberOfSubtasks() {
return numberOfSubtasks;
}
-
+
/**
* Gets the ID of the execution attempt.
- *
- * @return The ID of the execution attempt.
*/
public ExecutionAttemptID getExecutionId() {
return executionId;
}
-
+
/**
* Returns the current execution state of the task.
- *
- * @return the current execution state of the task
*/
public ExecutionState getExecutionState() {
return this.executionState;
}
-
+
public void setEnvironment(RuntimeEnvironment environment) {
this.environment = environment;
}
-
+
public RuntimeEnvironment getEnvironment() {
return environment;
}
-
+
public boolean isCanceledOrFailed() {
return executionState == ExecutionState.CANCELING ||
executionState == ExecutionState.CANCELED ||
executionState == ExecutionState.FAILED;
}
-
+
public String getTaskName() {
return taskName;
}
-
+
public String getTaskNameWithSubtasks() {
return this.taskName + " (" + (this.subtaskIndex + 1) + "/" + this.numberOfSubtasks + ")";
}
-
+
// ----------------------------------------------------------------------------------------------------------------
// States and Transitions
// ----------------------------------------------------------------------------------------------------------------
-
+
/**
* Marks the task as finished. This succeeds, if the task was previously in the state
* "RUNNING", otherwise it fails. Failure indicates that the task was either
* canceled, or set to failed.
- *
+ *
* @return True, if the task correctly enters the state FINISHED.
*/
public boolean markAsFinished() {
@@ -177,20 +166,21 @@ public final class Task {
notifyObservers(ExecutionState.FINISHED, null);
taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.FINISHED, null);
return true;
- } else {
+ }
+ else {
return false;
}
}
-
+
public void markFailed(Throwable error) {
while (true) {
ExecutionState current = this.executionState;
-
+
// if canceled, fine. we are done, and the jobmanager has been told
if (current == ExecutionState.CANCELED) {
return;
}
-
+
// if canceling, we are done, but we cannot be sure that the jobmanager has been told.
// after all, we may have recognized our failure state before the cancelling and never sent a canceled
// message back
@@ -201,23 +191,22 @@ public final class Task {
}
}
}
-
+
public void cancelExecution() {
while (true) {
ExecutionState current = this.executionState;
-
+
// if the task is already canceled (or canceling) or finished or failed,
// then we need not do anything
if (current == ExecutionState.FINISHED || current == ExecutionState.CANCELED ||
- current == ExecutionState.CANCELING || current == ExecutionState.FAILED)
- {
+ current == ExecutionState.CANCELING || current == ExecutionState.FAILED) {
return;
}
-
+
if (current == ExecutionState.DEPLOYING) {
// directly set to canceled
if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELED)) {
-
+
notifyObservers(ExecutionState.CANCELED, null);
taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.CANCELED, null);
return;
@@ -226,14 +215,15 @@ public final class Task {
else if (current == ExecutionState.RUNNING) {
// go to canceling and perform the actual task canceling
if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELING)) {
-
+
notifyObservers(ExecutionState.CANCELING, null);
try {
this.environment.cancelExecution();
- } catch (Throwable e) {
+ }
+ catch (Throwable e) {
LOG.error("Error while cancelling the task.", e);
}
-
+
return;
}
}
@@ -242,30 +232,33 @@ public final class Task {
}
}
}
-
+
/**
* Sets the tasks to be cancelled and reports a failure back to the master.
- * This method is important if a failure needs to be reported to the master, because
- * a simple canceled m
- *
- * @param cause The exception to report in the error message
*/
public void failExternally(Throwable cause) {
while (true) {
ExecutionState current = this.executionState;
-
+
// if the task is already canceled (or canceling) or finished or failed,
// then we need not do anything
- if (current == ExecutionState.FINISHED || current == ExecutionState.CANCELED ||
- current == ExecutionState.CANCELING || current == ExecutionState.FAILED)
- {
+ if (current == ExecutionState.CANCELED || current == ExecutionState.CANCELING || current == ExecutionState.FAILED) {
return;
}
-
+
+ if (current == ExecutionState.FINISHED) {
+ // Set state to failed in order to correctly unregister task from network environment
+ if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
+ notifyObservers(ExecutionState.FAILED, null);
+
+ return;
+ }
+ }
+
if (current == ExecutionState.DEPLOYING) {
// directly set to canceled
if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
-
+
notifyObservers(ExecutionState.FAILED, null);
taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.FAILED, cause);
return;
@@ -276,13 +269,14 @@ public final class Task {
if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
try {
this.environment.cancelExecution();
- } catch (Throwable e) {
+ }
+ catch (Throwable e) {
LOG.error("Error while cancelling the task.", e);
}
-
+
notifyObservers(ExecutionState.FAILED, null);
taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.FAILED, cause);
-
+
return;
}
}
@@ -291,18 +285,18 @@ public final class Task {
}
}
}
-
+
public void cancelingDone() {
while (true) {
ExecutionState current = this.executionState;
-
+
if (current == ExecutionState.CANCELED || current == ExecutionState.FAILED) {
return;
}
if (!(current == ExecutionState.RUNNING || current == ExecutionState.CANCELING)) {
LOG.error(String.format("Unexpected state transition in Task: %s -> %s", current, ExecutionState.CANCELED));
}
-
+
if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELED)) {
notifyObservers(ExecutionState.CANCELED, null);
taskManager.notifyExecutionStateChange(jobId, executionId, ExecutionState.CANCELED, null);
@@ -319,16 +313,14 @@ public final class Task {
final Thread thread = this.environment.getExecutingThread();
thread.start();
return true;
- } else {
+ }
+ else {
return false;
}
}
-
+
/**
* Unregisters the task from the central memory manager.
- *
- * @param memoryManager
- * the central memory manager
*/
public void unregisterMemoryManager(MemoryManager memoryManager) {
RuntimeEnvironment env = this.environment;
@@ -336,18 +328,13 @@ public final class Task {
memoryManager.releaseAll(env.getInvokable());
}
}
-
+
// -----------------------------------------------------------------------------------------------------------------
// Task Profiling
// -----------------------------------------------------------------------------------------------------------------
/**
* Registers the task manager profiler with the task.
- *
- * @param taskManagerProfiler
- * the task manager profiler
- * @param jobConfiguration
- * the configuration attached to the job
*/
public void registerProfiler(TaskManagerProfiler taskManagerProfiler, Configuration jobConfiguration) {
taskManagerProfiler.registerTask(this, jobConfiguration);
@@ -355,45 +342,58 @@ public final class Task {
/**
* Unregisters the task from the task manager profiler.
- *
- * @param taskManagerProfiler
- * the task manager profiler
*/
public void unregisterProfiler(TaskManagerProfiler taskManagerProfiler) {
if (taskManagerProfiler != null) {
taskManagerProfiler.unregisterTask(this.executionId);
}
}
-
+
+ // ------------------------------------------------------------------------
+ // Intermediate result partitions
+ // ------------------------------------------------------------------------
+
+ public BufferReader[] getReaders() {
+ return environment != null ? environment.getAllReaders() : null;
+ }
+
+ public BufferWriter[] getWriters() {
+ return environment != null ? environment.getAllWriters() : null;
+ }
+
+ public IntermediateResultPartition[] getProducedPartitions() {
+ return environment != null ? environment.getProducedPartitions() : null;
+ }
+
// --------------------------------------------------------------------------------------------
// State Listeners
// --------------------------------------------------------------------------------------------
-
- public void registerExecutionListener(ActorRef listener){
+
+ public void registerExecutionListener(ActorRef listener) {
executionListenerActors.add(listener);
}
- public void unregisterExecutionListener(ActorRef listener){
+ public void unregisterExecutionListener(ActorRef listener) {
executionListenerActors.remove(listener);
}
-
+
private void notifyObservers(ExecutionState newState, String message) {
if (LOG.isInfoEnabled()) {
LOG.info(getTaskNameWithSubtasks() + " switched to " + newState + (message == null ? "" : " : " + message));
}
-
- for(ActorRef listener: executionListenerActors){
+
+ for (ActorRef listener : executionListenerActors) {
listener.tell(new ExecutionGraphMessages.ExecutionStateChanged(
jobId, vertexId, taskName, numberOfSubtasks, subtaskIndex,
- executionId, newState,System.currentTimeMillis(), message),
+ executionId, newState, System.currentTimeMillis(), message),
ActorRef.noSender());
}
}
-
+
// --------------------------------------------------------------------------------------------
// Utilities
// --------------------------------------------------------------------------------------------
-
+
@Override
public String toString() {
return getTaskNameWithSubtasks() + " [" + executionState + ']';
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
index b0f1089..8ca1544 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/AtomicDisposableReferenceCounter.java
@@ -68,4 +68,4 @@ public class AtomicDisposableReferenceCounter {
return isDisposed;
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/d908ca19/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java
deleted file mode 100644
index 87d030e..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/BufferPoolConnector.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.util;
-
-import java.util.Queue;
-
-import org.apache.flink.core.memory.MemorySegment;
-import org.apache.flink.runtime.io.network.BufferRecycler;
-
-public class BufferPoolConnector implements BufferRecycler {
-
- /**
- * Reference to the memory pool the byte buffer was originally taken from.
- */
- private final Queue<MemorySegment> memoryPool;
-
- /**
- * Constructs a new buffer pool connector
- *
- * @param bufferPool
- * a reference to the memory pool the byte buffer was originally taken from
- */
- public BufferPoolConnector(final Queue<MemorySegment> bufferPool) {
- this.memoryPool = bufferPool;
- }
-
- @Override
- public void recycle(final MemorySegment buffer) {
- synchronized (this.memoryPool) {
- this.memoryPool.add(buffer);
- this.memoryPool.notify();
- }
- }
-}