You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/05/12 23:03:12 UTC
[01/10] flink git commit: [FLINK-1953] [runtime] Integrate new
snapshot checkpoint coordinator with jobgraph and execution graph
Repository: flink
Updated Branches:
refs/heads/master ff750e61a -> e79ff4ebf
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
index 1321336..5ab0150 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
@@ -35,6 +35,9 @@ import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.memorymanager.MemoryManager;
import org.apache.flink.runtime.messages.accumulators.ReportAccumulatorResult;
+import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
+import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
import java.io.IOException;
import java.util.Map;
@@ -224,7 +227,25 @@ public class RuntimeEnvironment implements Environment {
}
@Override
- public ActorRef getJobManager() {
- return jobManagerActor;
+ public void acknowledgeCheckpoint(long checkpointId) {
+ acknowledgeCheckpoint(checkpointId, null);
+ }
+
+ @Override
+ public void acknowledgeCheckpoint(long checkpointId, StateHandle<?> state) {
+ // try and create a serialized version of the state handle
+ SerializedValue<StateHandle<?>> serializedState;
+ if (state == null) {
+ serializedState = null;
+ } else {
+ try {
+ serializedState = new SerializedValue<StateHandle<?>>(state);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to serialize state handle during checkpoint confirmation", e);
+ }
+ }
+
+ AcknowledgeCheckpoint message = new AcknowledgeCheckpoint(jobId, executionId, checkpointId, serializedState);
+ jobManagerActor.tell(message, ActorRef.noSender());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index f12344b..1578e4b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -46,6 +46,8 @@ import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobgraph.tasks.BarrierTransceiver;
+import org.apache.flink.runtime.jobgraph.tasks.CheckpointCommittingOperator;
+import org.apache.flink.runtime.jobgraph.tasks.CheckpointedOperator;
import org.apache.flink.runtime.jobgraph.tasks.OperatorStateCarrier;
import org.apache.flink.runtime.memorymanager.MemoryManager;
import org.apache.flink.runtime.messages.TaskMessages;
@@ -53,6 +55,8 @@ import org.apache.flink.runtime.messages.TaskMessages.TaskInFinalState;
import org.apache.flink.runtime.messages.TaskManagerMessages.FatalError;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.state.StateUtils;
+import org.apache.flink.runtime.util.SerializedValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -139,9 +143,6 @@ public class Task implements Runnable {
/** The name of the class that holds the invokable code */
private final String nameOfInvokableClass;
- /** The handle to the state that the operator was initialized with */
- private final StateHandle operatorState;
-
/** The memory manager to be used by this task */
private final MemoryManager memoryManager;
@@ -171,10 +172,13 @@ public class Task implements Runnable {
/** The timeout for all ask operations on actors */
private final Timeout actorAskTimeout;
+ /** The library cache, from which the task can request its required JAR files */
private final LibraryCacheManager libraryCache;
+ /** The cache for user-defined files that the invokable requires */
private final FileCache fileCache;
+ /** The gateway to the network stack, which handles inputs and produced results */
private final NetworkEnvironment network;
/** The thread that executes the task */
@@ -182,10 +186,12 @@ public class Task implements Runnable {
// ------------------------------------------------------------------------
- // Fields that control the task execution
+ // Fields that control the task execution. All these fields are volatile
+ // (which means that they introduce memory barriers), to establish
+ // proper happens-before semantics on parallel modification
// ------------------------------------------------------------------------
- private final AtomicBoolean invokableHasBeenCanceled = new AtomicBoolean(false);
+ private final AtomicBoolean invokableHasBeenCanceled;
/** The invokable of this task, if initialized */
private volatile AbstractInvokable invokable;
@@ -196,6 +202,10 @@ public class Task implements Runnable {
/** The observed exception, in case the task execution failed */
private volatile Throwable failureCause;
+ /** The handle to the state that the operator was initialized with. Will be set to null after the
+ * initialization, to be memory friendly */
+ private volatile SerializedValue<StateHandle<?>> operatorState;
+
/**
* <p><b>IMPORTANT:</b> This constructor may not start any work that would need to
@@ -227,7 +237,7 @@ public class Task implements Runnable {
this.taskConfiguration = checkNotNull(tdd.getTaskConfiguration());
this.requiredJarFiles = checkNotNull(tdd.getRequiredJarFiles());
this.nameOfInvokableClass = checkNotNull(tdd.getInvokableClassName());
- this.operatorState = tdd.getOperatorStates();
+ this.operatorState = tdd.getOperatorState();
this.memoryManager = checkNotNull(memManager);
this.ioManager = checkNotNull(ioManager);
@@ -286,7 +296,9 @@ public class Task implements Runnable {
}
// finally, create the executing thread, but do not start it
- executingThread = new Thread(TASK_THREADS_GROUP, this, taskNameWithSubtask);
+ executingThread = new Thread(TASK_THREADS_GROUP, this, taskNameWithSubtask);
+
+ invokableHasBeenCanceled = new AtomicBoolean(false);
}
// ------------------------------------------------------------------------
@@ -423,9 +435,7 @@ public class Task implements Runnable {
// all resource acquisitions and registrations from here on
// need to be undone in the end
-
Map<String, Future<Path>> distributedCacheEntries = new HashMap<String, Future<Path>>();
-
AbstractInvokable invokable = null;
try {
@@ -500,15 +510,32 @@ public class Task implements Runnable {
// the very last thing before the actual execution starts running is to inject
// the state into the task. the state is non-empty if this is an execution
// of a task that failed but had backuped state from a checkpoint
+
+ // get our private reference onto the stack (be safe against concurrent changes)
+ SerializedValue<StateHandle<?>> operatorState = this.operatorState;
+
if (operatorState != null) {
if (invokable instanceof OperatorStateCarrier) {
- ((OperatorStateCarrier) invokable).injectState(operatorState);
+ try {
+ StateHandle<?> state = operatorState.deserializeValue(userCodeClassLoader);
+ OperatorStateCarrier<?> op = (OperatorStateCarrier<?>) invokable;
+ StateUtils.setOperatorState(op, state);
+ }
+ catch (Exception e) {
+ throw new Exception("Failed to deserialize state handle and setup initial operator state");
+ }
}
else {
throw new IllegalStateException("Found operator state for a non-stateful task invokable");
}
}
+ // be memory and GC friendly - since the code stays in invoke() for a potentially long time,
+ // we clear the reference to the state handle
+ //noinspection UnusedAssignment
+ operatorState = null;
+ this.operatorState = null;
+
// ----------------------------------------------------------------
// actual task core work
// ----------------------------------------------------------------
@@ -568,7 +595,7 @@ public class Task implements Runnable {
// ----------------------------------------------------------------
try {
- // transition into our final state. we should be either in RUNNING, CANCELING, or FAILED
+ // transition into our final state. we should be either in DEPLOYING, RUNNING, CANCELING, or FAILED
// loop for multiple retries during concurrent state changes via calls to cancel() or
// to failExternally()
while (true) {
@@ -708,6 +735,14 @@ public class Task implements Runnable {
// Canceling / Failing the task from the outside
// ----------------------------------------------------------------------------------------------------------------
+ /**
+ * Cancels the task execution. If the task is already in a terminal state
+ * (such as FINISHED, CANCELED, FAILED), or if the task is already canceling this does nothing.
+ * Otherwise it sets the state to CANCELING, and, if the invokable code is running,
+ * starts an asynchronous thread that aborts that code.
+ *
+ * <p>This method never blocks.</p>
+ */
public void cancelExecution() {
LOG.info("Attempting to cancel task " + taskNameWithSubtask);
if (cancelOrFailAndCancelInvokable(ExecutionState.CANCELING)) {
@@ -716,7 +751,13 @@ public class Task implements Runnable {
}
/**
- * Sets the tasks to be cancelled and reports a failure back to the master.
+ * Marks task execution failed for an external reason (a reason other than th task code itself
+ * throwing an exception). If the task is already in a terminal state
+ * (such as FINISHED, CANCELED, FAILED), or if the task is already canceling this does nothing.
+ * Otherwise it sets the state to FAILED, and, if the invokable code is running,
+ * starts an asynchronous thread that aborts that code.
+ *
+ * <p>This method never blocks.</p>
*/
public void failExternally(Throwable cause) {
LOG.info("Attempting to fail task externally " + taskNameWithSubtask);
@@ -751,6 +792,8 @@ public class Task implements Runnable {
LOG.info("Triggering cancellation of task code {} ({}).", taskNameWithSubtask, executionId);
// because the canceling may block on user code, we cancel from a separate thread
+ // we do not reuse the async call handler, because that one may be blocked, in which
+ // case the canceling could not continue
Runnable canceler = new TaskCanceler(LOG, invokable, executingThread, taskNameWithSubtask);
Thread cancelThread = new Thread(executingThread.getThreadGroup(), canceler,
"Canceler for " + taskNameWithSubtask);
@@ -798,15 +841,42 @@ public class Task implements Runnable {
// Notifications on the invokable
// ------------------------------------------------------------------------
- public void triggerCheckpointBarrier(final long checkpointID) {
- AbstractInvokable invokabe = this.invokable;
+ /**
+ * Calls the invokable to trigger a checkpoint, if the invokable implements the interface
+ * {@link org.apache.flink.runtime.jobgraph.tasks.CheckpointedOperator}.
+ *
+ * @param checkpointID The ID identifying the checkpoint.
+ * @param checkpointTimestamp The timestamp associated with the checkpoint.
+ */
+ public void triggerCheckpointBarrier(final long checkpointID, final long checkpointTimestamp) {
+ AbstractInvokable invokable = this.invokable;
- if (executionState == ExecutionState.RUNNING && invokabe != null) {
- if (invokabe instanceof BarrierTransceiver) {
- final BarrierTransceiver barrierTransceiver = (BarrierTransceiver) invokabe;
+ if (executionState == ExecutionState.RUNNING && invokable != null) {
+ if (invokable instanceof CheckpointedOperator) {
+
+ // build a local closure
+ final CheckpointedOperator checkpointer = (CheckpointedOperator) invokable;
+ final Logger logger = LOG;
+ final String taskName = taskNameWithSubtask;
+
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ checkpointer.triggerCheckpoint(checkpointID, checkpointTimestamp);
+ }
+ catch (Throwable t) {
+ logger.error("Error while triggering checkpoint for " + taskName, t);
+ }
+ }
+ };
+ executeAsyncCallRunnable(runnable, "Checkpoint Trigger");
+ }
+ else if (invokable instanceof BarrierTransceiver) {
+ final BarrierTransceiver barrierTransceiver = (BarrierTransceiver) invokable;
final Logger logger = LOG;
- Thread caller = new Thread("Barrier emitter") {
+ Runnable runnable = new Runnable() {
@Override
public void run() {
try {
@@ -817,8 +887,7 @@ public class Task implements Runnable {
}
}
};
- caller.setDaemon(true);
- caller.start();
+ executeAsyncCallRunnable(runnable, "Checkpoint Trigger");
}
else {
LOG.error("Task received a checkpoint request, but is not a checkpointing task - "
@@ -826,10 +895,49 @@ public class Task implements Runnable {
}
}
else {
- LOG.debug("Ignoring request to trigger a checkpoint barrier");
+ LOG.debug("Ignoring request to trigger a checkpoint for non-running task.");
}
}
+ public void confirmCheckpoint(final long checkpointID, final long checkpointTimestamp) {
+ AbstractInvokable invokable = this.invokable;
+
+ if (executionState == ExecutionState.RUNNING && invokable != null) {
+ if (invokable instanceof CheckpointCommittingOperator) {
+
+ // build a local closure
+ final CheckpointCommittingOperator checkpointer = (CheckpointCommittingOperator) invokable;
+ final Logger logger = LOG;
+ final String taskName = taskNameWithSubtask;
+
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ checkpointer.confirmCheckpoint(checkpointID, checkpointTimestamp);
+ }
+ catch (Throwable t) {
+ logger.error("Error while confirming checkpoint for " + taskName, t);
+ }
+ }
+ };
+ executeAsyncCallRunnable(runnable, "Checkpoint Confirmation");
+ }
+ else {
+ LOG.error("Task received a checkpoint commit notification, but is not a checkpoint committing task - "
+ + taskNameWithSubtask);
+ }
+ }
+ else {
+ LOG.debug("Ignoring checkpoint commit notification for non-running task.");
+ }
+ }
+
+ private void executeAsyncCallRunnable(Runnable runnable, String callName) {
+ Thread thread = new Thread(runnable, callName);
+ thread.setDaemon(true);
+ thread.start();
+ }
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializableObject.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializableObject.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializableObject.java
new file mode 100644
index 0000000..af6fa16
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/SerializableObject.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.util;
+
+/**
+ * A simple object that only implements {@link java.io.Serializable}, so it can be used
+ * in serializable classes.
+ */
+public class SerializableObject implements java.io.Serializable {
+
+ private static final long serialVersionUID = -7322636177391854669L;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 75463fd..4745fb6 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -33,11 +33,11 @@ import org.apache.flink.runtime.client._
import org.apache.flink.runtime.executiongraph.{ExecutionJobVertex, ExecutionGraph}
import org.apache.flink.runtime.jobmanager.web.WebInfoServer
import org.apache.flink.runtime.messages.ArchiveMessages.ArchiveExecutionGraph
-import org.apache.flink.runtime.messages.CheckpointingMessages.{StateBarrierAck, BarrierAck}
import org.apache.flink.runtime.messages.ExecutionGraphMessages.JobStatusChanged
import org.apache.flink.runtime.messages.Messages.{Disconnect, Acknowledge}
import org.apache.flink.runtime.messages.TaskMessages.UpdateTaskExecutionState
import org.apache.flink.runtime.messages.accumulators._
+import org.apache.flink.runtime.messages.checkpoint.{AcknowledgeCheckpoint, AbstractCheckpointMessage}
import org.apache.flink.runtime.process.ProcessReaper
import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.security.SecurityUtils.FlinkSecuredRunner
@@ -47,7 +47,7 @@ import org.apache.flink.runtime.{ActorSynchronousLogging, ActorLogMessages}
import org.apache.flink.runtime.akka.AkkaUtils
import org.apache.flink.runtime.execution.librarycache.BlobLibraryCacheManager
import org.apache.flink.runtime.instance.InstanceManager
-import org.apache.flink.runtime.jobgraph.{JobGraph, JobStatus}
+import org.apache.flink.runtime.jobgraph.{JobVertexID, JobGraph, JobStatus}
import org.apache.flink.runtime.jobmanager.accumulators.AccumulatorManager
import org.apache.flink.runtime.jobmanager.scheduler.{Scheduler => FlinkScheduler}
import org.apache.flink.runtime.messages.JobManagerMessages._
@@ -89,19 +89,19 @@ import scala.collection.JavaConverters._
* - [[JobStatusChanged]] indicates that the status of job (RUNNING, CANCELING, FINISHED, etc.) has
* changed. This message is sent by the ExecutionGraph.
*/
-class JobManager(val flinkConfiguration: Configuration,
- val instanceManager: InstanceManager,
- val scheduler: FlinkScheduler,
- val libraryCacheManager: BlobLibraryCacheManager,
- val archive: ActorRef,
- val accumulatorManager: AccumulatorManager,
- val defaultExecutionRetries: Int,
- val delayBetweenRetries: Long,
- val timeout: FiniteDuration)
+class JobManager(protected val flinkConfiguration: Configuration,
+ protected val instanceManager: InstanceManager,
+ protected val scheduler: FlinkScheduler,
+ protected val libraryCacheManager: BlobLibraryCacheManager,
+ protected val archive: ActorRef,
+ protected val accumulatorManager: AccumulatorManager,
+ protected val defaultExecutionRetries: Int,
+ protected val delayBetweenRetries: Long,
+ protected val timeout: FiniteDuration)
extends Actor with ActorLogMessages with ActorSynchronousLogging {
/** List of current jobs running jobs */
- val currentJobs = scala.collection.mutable.HashMap[JobID, (ExecutionGraph, JobInfo)]()
+ protected val currentJobs = scala.collection.mutable.HashMap[JobID, (ExecutionGraph, JobInfo)]()
/**
@@ -278,6 +278,9 @@ class JobManager(val flinkConfiguration: Configuration,
sender ! NextInputSplit(serializedInputSplit)
+ case checkpointMessage : AbstractCheckpointMessage =>
+ handleCheckpointMessage(checkpointMessage)
+
case JobStatusChanged(jobID, newJobStatus, timeStamp, error) =>
currentJobs.get(jobID) match {
case Some((executionGraph, jobInfo)) => executionGraph.getJobName
@@ -323,19 +326,6 @@ class JobManager(val flinkConfiguration: Configuration,
removeJob(jobID)
}
- case msg: BarrierAck =>
- currentJobs.get(msg.jobID) match {
- case Some(jobExecution) =>
- jobExecution._1.getStateCheckpointerActor forward msg
- case None =>
- }
- case msg: StateBarrierAck =>
- currentJobs.get(msg.jobID) match {
- case Some(jobExecution) =>
- jobExecution._1.getStateCheckpointerActor forward msg
- case None =>
- }
-
case ScheduleOrUpdateConsumers(jobId, partitionId) =>
currentJobs.get(jobId) match {
case Some((executionGraph, _)) =>
@@ -486,10 +476,7 @@ class JobManager(val flinkConfiguration: Configuration,
executionGraph.setDelayBeforeRetrying(delayBetweenRetries)
executionGraph.setScheduleMode(jobGraph.getScheduleMode)
executionGraph.setQueuedSchedulingAllowed(jobGraph.getAllowQueuedScheduling)
-
- executionGraph.setCheckpointingEnabled(jobGraph.isCheckpointingEnabled)
- executionGraph.setCheckpointingInterval(jobGraph.getCheckpointingInterval)
-
+
// initialize the vertices that have a master initialization hook
// file output formats create directories here, input formats create splits
if (log.isDebugEnabled) {
@@ -531,8 +518,33 @@ class JobManager(val flinkConfiguration: Configuration,
log.debug(s"Successfully created execution graph from job graph ${jobId} (${jobName}).")
}
- // give an actorContext
- executionGraph.setParentContext(context)
+ // configure the state checkpointing
+ val snapshotSettings = jobGraph.getSnapshotSettings
+ if (snapshotSettings != null) {
+
+ val idToVertex: JobVertexID => ExecutionJobVertex = id => {
+ val vertex = executionGraph.getJobVertex(id)
+ if (vertex == null) {
+ throw new JobSubmissionException(jobId,
+ "The snapshot checkpointing settings refer to non-existent vertex " + id)
+ }
+ vertex
+ }
+
+ val triggerVertices: java.util.List[ExecutionJobVertex] =
+ snapshotSettings.getVerticesToTrigger.asScala.map(idToVertex).asJava
+
+ val ackVertices: java.util.List[ExecutionJobVertex] =
+ snapshotSettings.getVerticesToAcknowledge.asScala.map(idToVertex).asJava
+
+ val confirmVertices: java.util.List[ExecutionJobVertex] =
+ snapshotSettings.getVerticesToConfirm.asScala.map(idToVertex).asJava
+
+ executionGraph.enableSnaphotCheckpointing(
+ snapshotSettings.getCheckpointInterval, snapshotSettings.getCheckpointTimeout,
+ triggerVertices, ackVertices, confirmVertices,
+ context.system)
+ }
// get notified about job status changes
executionGraph.registerJobStatusListener(self)
@@ -588,6 +600,40 @@ class JobManager(val flinkConfiguration: Configuration,
}
/**
+ * Dedicated handler for checkpoint messages.
+ *
+ * @param actorMessage The checkpoint actor message.
+ */
+ private def handleCheckpointMessage(actorMessage: AbstractCheckpointMessage): Unit = {
+ actorMessage match {
+ case ackMessage: AcknowledgeCheckpoint =>
+ val jid = ackMessage.getJob()
+ currentJobs.get(jid) match {
+ case Some((graph, _)) =>
+ val coordinator = graph.getCheckpointCoordinator()
+ if (coordinator != null) {
+ try {
+ coordinator.receiveAcknowledgeMessage(ackMessage)
+ }
+ catch {
+ case t: Throwable =>
+ log.error(s"Error in CheckpointCoordinator while processing $ackMessage", t)
+ }
+ }
+ else {
+ log.error(
+ s"Received ConfirmCheckpoint message for job $jid with no CheckpointCoordinator")
+ }
+
+ case None => log.error(s"Received ConfirmCheckpoint for unavailable job $jid")
+ }
+
+ // unknown checkpoint message
+ case _ => unhandled(actorMessage)
+ }
+ }
+
+ /**
* Handle unmatched messages with an exception.
*/
override def unhandled(message: Any): Unit = {
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/StreamCheckpointCoordinator.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/StreamCheckpointCoordinator.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/StreamCheckpointCoordinator.scala
deleted file mode 100644
index 8bb1274..0000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/StreamCheckpointCoordinator.scala
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmanager
-
-import java.lang.{Long => JLong}
-
-import akka.actor._
-import org.apache.flink.runtime.{ActorSynchronousLogging, ActorLogMessages}
-import org.apache.flink.runtime.execution.ExecutionState
-import org.apache.flink.runtime.executiongraph.{ExecutionGraph, ExecutionVertex}
-import org.apache.flink.runtime.jobgraph.JobStatus._
-import org.apache.flink.runtime.jobgraph.JobVertexID
-import org.apache.flink.runtime.messages.CheckpointingMessages._
-import org.apache.flink.runtime.state.StateHandle
-
-import scala.collection.JavaConversions._
-import scala.collection.immutable.TreeMap
-import scala.concurrent.duration.{FiniteDuration, _}
-
-/**
- * The StreamCheckpointCoordinator is responsible for operator state management and checkpoint
- * coordination in streaming jobs. It periodically sends checkpoint barriers to the sources of a
- * running job and constantly collects acknowledgements from operators while the barriers are being
- * disseminated throughout the execution graph. Upon time intervals it finds the last globally
- * acknowledged checkpoint barrier to be used for a consistent recovery and loads all associated
- * state handles to the respected execution vertices.
- *
- * The following messages describe this actor's expected behavior:
- *
- * - [[InitBarrierScheduler]] initiates the actor and schedules the periodic [[BarrierTimeout]]
- * and [[CompactAndUpdate]] messages that are used for maintaining the state checkpointing logic.
- *
- * - [[BarrierTimeout]] is periodically triggered upon initiation in order to start a new
- * checkpoint barrier. That is when the barriers are being disseminated to the source vertices.
- *
- * - [[BarrierAck]] is being sent by each operator upon the completion of a state checkpoint. All
- * such acknowledgements are being collected and inspected upon [[CompactAndUpdate]] handling in
- * order to find out the last consistent checkpoint.
- *
- * - [[StateBarrierAck]] describes an acknowledgement such as the case of a [[BarrierAck]] that
- * additionally carries operatorState with it.
- *
- * - [[CompactAndUpdate]] marks the last globally consistent checkpoint barrier to be used for
- * recovery purposes and removes all older states and acknowledgements up to that barrier.
- * Furthermore, it updates the current ExecutionGraph with the current operator state handles
- *
- */
-
-class StreamCheckpointCoordinator(val executionGraph: ExecutionGraph,
- val vertices: Iterable[ExecutionVertex],
- var acks: Map[(JobVertexID,Int),List[JLong]],
- var states: Map[(JobVertexID, Integer, JLong), StateHandle],
- val interval: FiniteDuration,
- var curId: JLong,
- var ackId: JLong)
-extends Actor with ActorLogMessages with ActorSynchronousLogging {
-
- implicit private val executor = context.dispatcher
-
- override def receiveWithLogMessages: Receive = {
-
- case InitBarrierScheduler =>
- context.system.scheduler.schedule(interval,interval,self,BarrierTimeout)
- context.system.scheduler.schedule(2 * interval,2 * interval,self,CompactAndUpdate)
- log.info("Started Stream State Monitor for job " +
- s"${executionGraph.getJobID}${executionGraph.getJobName}")
-
- case BarrierTimeout =>
- executionGraph.getState match {
- case FAILED | CANCELED | FINISHED =>
- log.info(s"Stopping monitor for terminated job ${executionGraph.getJobID}.")
- self ! PoisonPill
- case RUNNING =>
- curId += 1
- log.debug("Sending Barrier to vertices of Job " + executionGraph.getJobName)
- vertices.filter(v => v.getJobVertex.getJobVertex.isInputVertex &&
- v.getExecutionState == ExecutionState.RUNNING).foreach(vertex
- => vertex.getCurrentAssignedResource.getInstance.getTaskManager
- ! BarrierReq(vertex.getCurrentExecutionAttempt.getAttemptId,curId))
- case _ =>
- log.debug("Omitting sending barrier since graph is in " +
- s"${executionGraph.getState} state for job ${executionGraph.getJobID}.")
- }
-
- case StateBarrierAck(jobID, jobVertexID, instanceID, checkpointID, opState) =>
- states += (jobVertexID, instanceID, checkpointID) -> opState
- self ! BarrierAck(jobID, jobVertexID, instanceID, checkpointID)
-
- case BarrierAck(jobID, jobVertexID,instanceID,checkpointID) =>
- acks.get(jobVertexID,instanceID) match {
- case Some(acklist) =>
- acks += (jobVertexID,instanceID) -> (checkpointID :: acklist)
- case None =>
- }
- log.debug(acks.toString())
-
- case CompactAndUpdate =>
- val barrierCount =
- acks.values.foldLeft(TreeMap[JLong,Int]().withDefaultValue(0))((dict,myList)
- => myList.foldLeft(dict)((dict2,elem) => dict2.updated(elem,dict2(elem) + 1)))
- val keysToKeep = barrierCount.filter(_._2 == acks.size).keys
- ackId = if(keysToKeep.nonEmpty) keysToKeep.max else ackId
- acks.keys.foreach(x => acks = acks.updated(x,acks(x).filter(_ >= ackId)))
- states = states.filterKeys(_._3 >= ackId)
- log.debug("[FT-MONITOR] Last global barrier is " + ackId)
- executionGraph.loadOperatorStates(states)
-
- }
-}
-
-object StreamCheckpointCoordinator {
-
- def spawn(context: ActorContext,executionGraph: ExecutionGraph,
- interval: FiniteDuration = 5 seconds): ActorRef = {
-
- val vertices: Iterable[ExecutionVertex] = getExecutionVertices(executionGraph)
- val monitor = context.system.actorOf(Props(new StreamCheckpointCoordinator(executionGraph,
- vertices,vertices.map(x => ((x.getJobVertex.getJobVertexId,x.getParallelSubtaskIndex),
- List.empty[JLong])).toMap, Map() ,interval,0L,-1L)))
- monitor ! InitBarrierScheduler
- monitor
- }
-
- private def getExecutionVertices(executionGraph: ExecutionGraph): Iterable[ExecutionVertex] = {
- for((_,execJobVertex) <- executionGraph.getAllVertices;
- execVertex: ExecutionVertex <- execJobVertex.getTaskVertices)
- yield execVertex
- }
-}
-
-case class BarrierTimeout()
-
-case class InitBarrierScheduler()
-
-case class CompactAndUpdate()
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/CheckpointingMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/CheckpointingMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/CheckpointingMessages.scala
deleted file mode 100644
index 9f6f51a..0000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/CheckpointingMessages.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.messages
-
-import org.apache.flink.api.common.JobID
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
-import org.apache.flink.runtime.jobgraph.JobVertexID
-import org.apache.flink.runtime.state.StateHandle
-
-/**
- * Actor messages specific to checkpoints (triggering, acknowledging,
- * state transfer, ...)
- */
-object CheckpointingMessages {
-
- /**
- * Abstract base trait for all checkpoint messages.
- */
- trait CheckpointingMessage
-
- // --------------------------------------------------------------------------
-
- case class BarrierReq(attemptID: ExecutionAttemptID,
- checkpointID: Long) extends CheckpointingMessage
-
- case class BarrierAck(jobID: JobID,
- jobVertexID:JobVertexID,
- instanceID: Int,
- checkpointID: Long) extends CheckpointingMessage
-
- case class StateBarrierAck(jobID: JobID,
- jobVertexID: JobVertexID,
- instanceID: Integer,
- checkpointID: java.lang.Long,
- states: StateHandle) extends CheckpointingMessage
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 2e580cc..ed63db0 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper
import grizzled.slf4j.Logger
import org.apache.flink.configuration._
+import org.apache.flink.runtime.messages.checkpoint.{ConfirmCheckpoint, TriggerCheckpoint, AbstractCheckpointMessage}
import org.apache.flink.runtime.{ActorSynchronousLogging, ActorLogMessages}
import org.apache.flink.runtime.akka.AkkaUtils
import org.apache.flink.runtime.blob.{BlobService, BlobCache}
@@ -53,7 +54,6 @@ import org.apache.flink.runtime.io.network.netty.NettyConfig
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID
import org.apache.flink.runtime.jobmanager.JobManager
import org.apache.flink.runtime.memorymanager.{MemoryManager, DefaultMemoryManager}
-import org.apache.flink.runtime.messages.CheckpointingMessages.{CheckpointingMessage, BarrierReq}
import org.apache.flink.runtime.messages.Messages._
import org.apache.flink.runtime.messages.RegistrationMessages._
import org.apache.flink.runtime.messages.TaskManagerMessages._
@@ -241,7 +241,7 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
case message: TaskMessage => handleTaskMessage(message)
// messages for coordinating checkpoints
- case message: CheckpointingMessage => handleCheckpointingMessage(message)
+ case message: AbstractCheckpointMessage => handleCheckpointingMessage(message)
// registration messages for connecting and disconnecting from / to the JobManager
case message: RegistrationMessage => handleRegistrationMessage(message)
@@ -345,7 +345,7 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
currentJobManager foreach {
jobManager => {
val futureResponse = (jobManager ? updateMsg)(askTimeout)
-
+
val executionID = taskExecutionState.getID
futureResponse.mapTo[Boolean].onComplete {
@@ -399,25 +399,43 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
/**
* Handler for messages related to checkpoints.
*
- * @param message The checkpoint message.
+ * @param actorMessage The checkpoint message.
*/
- private def handleCheckpointingMessage(message: CheckpointingMessage): Unit = {
+ private def handleCheckpointingMessage(actorMessage: AbstractCheckpointMessage): Unit = {
- message match {
+ actorMessage match {
- case BarrierReq(attemptID, checkpointID) =>
- log.debug(s"[FT-TaskManager] Barrier $checkpointID request received " +
- s"for attempt $attemptID.")
+ case message: TriggerCheckpoint =>
+ val taskExecutionId = message.getTaskExecutionId
+ val checkpointId = message.getCheckpointId
+ val timestamp = message.getTimestamp
+
+ log.debug(s"Receiver TriggerCheckpoint ${checkpointId}@${timestamp} for $taskExecutionId.")
- val task = runningTasks.get(attemptID)
+ val task = runningTasks.get(taskExecutionId)
if (task != null) {
- task.triggerCheckpointBarrier(checkpointID)
+ task.triggerCheckpointBarrier(checkpointId, timestamp)
} else {
- log.debug(s"Taskmanager received a checkpoint request for unknown task $attemptID.")
+ log.debug(s"Taskmanager received a checkpoint request for unknown task $taskExecutionId.")
+ }
+
+ case message: ConfirmCheckpoint =>
+ val taskExecutionId = message.getTaskExecutionId
+ val checkpointId = message.getCheckpointId
+ val timestamp = message.getTimestamp
+
+ log.debug(s"Receiver ConfirmCheckpoint ${checkpointId}@${timestamp} for $taskExecutionId.")
+
+ val task = runningTasks.get(taskExecutionId)
+ if (task != null) {
+ task.confirmCheckpoint(checkpointId, timestamp)
+ } else {
+ log.debug(
+ s"Taskmanager received a checkpoint confirmation for unknown task $taskExecutionId.")
}
// unknown checkpoint message
- case _ => unhandled(message)
+ case _ => unhandled(actorMessage)
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
index aee0e63..10c8074 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorTest.java
@@ -197,8 +197,8 @@ public class CheckpointCoordinatorTest {
// validate that the relevant tasks got a confirmation message
{
- ConfirmCheckpoint confirmMessage1 = new ConfirmCheckpoint(jid, attemptID1, checkpointId);
- ConfirmCheckpoint confirmMessage2 = new ConfirmCheckpoint(jid, attemptID2, checkpointId);
+ ConfirmCheckpoint confirmMessage1 = new ConfirmCheckpoint(jid, attemptID1, checkpointId, timestamp);
+ ConfirmCheckpoint confirmMessage2 = new ConfirmCheckpoint(jid, attemptID2, checkpointId, timestamp);
verify(vertex1, times(1)).sendMessageToCurrentExecution(eq(confirmMessage1), eq(attemptID1));
verify(vertex2, times(1)).sendMessageToCurrentExecution(eq(confirmMessage2), eq(attemptID2));
}
@@ -235,8 +235,8 @@ public class CheckpointCoordinatorTest {
verify(vertex1, times(1)).sendMessageToCurrentExecution(eq(expectedMessage1), eq(attemptID1));
verify(vertex2, times(1)).sendMessageToCurrentExecution(eq(expectedMessage2), eq(attemptID2));
- ConfirmCheckpoint confirmMessage1 = new ConfirmCheckpoint(jid, attemptID1, checkpointIdNew);
- ConfirmCheckpoint confirmMessage2 = new ConfirmCheckpoint(jid, attemptID2, checkpointIdNew);
+ ConfirmCheckpoint confirmMessage1 = new ConfirmCheckpoint(jid, attemptID1, checkpointIdNew, timestampNew);
+ ConfirmCheckpoint confirmMessage2 = new ConfirmCheckpoint(jid, attemptID2, checkpointIdNew, timestampNew);
verify(vertex1, times(1)).sendMessageToCurrentExecution(eq(confirmMessage1), eq(attemptID1));
verify(vertex2, times(1)).sendMessageToCurrentExecution(eq(confirmMessage2), eq(attemptID2));
}
@@ -341,7 +341,7 @@ public class CheckpointCoordinatorTest {
// the first confirm message should be out
verify(commitVertex, times(1)).sendMessageToCurrentExecution(
- new ConfirmCheckpoint(jid, commitAttemptID, checkpointId1), commitAttemptID);
+ new ConfirmCheckpoint(jid, commitAttemptID, checkpointId1, timestamp1), commitAttemptID);
// send the last remaining ack for the second checkpoint
coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId2));
@@ -353,7 +353,7 @@ public class CheckpointCoordinatorTest {
// the second commit message should be out
verify(commitVertex, times(1)).sendMessageToCurrentExecution(
- new ConfirmCheckpoint(jid, commitAttemptID, checkpointId2), commitAttemptID);
+ new ConfirmCheckpoint(jid, commitAttemptID, checkpointId2, timestamp2), commitAttemptID);
// validate the committed checkpoints
List<SuccessfulCheckpoint> scs = coord.getSuccessfulCheckpoints();
@@ -480,7 +480,7 @@ public class CheckpointCoordinatorTest {
// the first confirm message should be out
verify(commitVertex, times(1)).sendMessageToCurrentExecution(
- new ConfirmCheckpoint(jid, commitAttemptID, checkpointId2), commitAttemptID);
+ new ConfirmCheckpoint(jid, commitAttemptID, checkpointId2, timestamp2), commitAttemptID);
// send the last remaining ack for the first checkpoint. This should not do anything
coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, ackAttemptID3, checkpointId1));
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
new file mode 100644
index 0000000..51c4890
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.execution.ExecutionState;
+import org.apache.flink.runtime.executiongraph.Execution;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
+import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
+import org.apache.flink.runtime.state.LocalStateHandle;
+import org.apache.flink.runtime.state.OperatorState;
+import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
+
+import org.junit.Test;
+
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.*;
+
+/**
+ * Tests concerning the restoring of state from a checkpoint to the task executions.
+ */
+public class CheckpointStateRestoreTest {
+
+ @Test
+ public void testSetState() {
+ try {
+ final SerializedValue<StateHandle<?>> serializedState = new SerializedValue<StateHandle<?>>(
+ new LocalStateHandle(Collections.<String,OperatorState<?>>emptyMap()));
+
+ final JobID jid = new JobID();
+ final JobVertexID statefulId = new JobVertexID();
+ final JobVertexID statelessId = new JobVertexID();
+
+ Execution statefulExec1 = mockExecution();
+ Execution statefulExec2 = mockExecution();
+ Execution statefulExec3 = mockExecution();
+ Execution statelessExec1 = mockExecution();
+ Execution statelessExec2 = mockExecution();
+
+ ExecutionVertex stateful1 = mockExecutionVertex(statefulExec1, statefulId, 0);
+ ExecutionVertex stateful2 = mockExecutionVertex(statefulExec2, statefulId, 1);
+ ExecutionVertex stateful3 = mockExecutionVertex(statefulExec3, statefulId, 2);
+ ExecutionVertex stateless1 = mockExecutionVertex(statelessExec1, statelessId, 0);
+ ExecutionVertex stateless2 = mockExecutionVertex(statelessExec2, statelessId, 1);
+
+ ExecutionJobVertex stateful = mockExecutionJobVertex(statefulId,
+ new ExecutionVertex[] { stateful1, stateful2, stateful3 });
+ ExecutionJobVertex stateless = mockExecutionJobVertex(statelessId,
+ new ExecutionVertex[] { stateless1, stateless2 });
+
+ Map<JobVertexID, ExecutionJobVertex> map = new HashMap<JobVertexID, ExecutionJobVertex>();
+ map.put(statefulId, stateful);
+ map.put(statelessId, stateless);
+
+
+ CheckpointCoordinator coord = new CheckpointCoordinator(jid, 1, 200000L,
+ new ExecutionVertex[] { stateful1, stateful2, stateful3, stateless1, stateless2 },
+ new ExecutionVertex[] { stateful1, stateful2, stateful3, stateless1, stateless2 },
+ new ExecutionVertex[0]);
+
+ // create ourselves a checkpoint with state
+ final long timestamp = 34623786L;
+ coord.triggerCheckpoint(timestamp);
+
+ PendingCheckpoint pending = coord.getPendingCheckpoints().values().iterator().next();
+ final long checkpointId = pending.getCheckpointId();
+
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec1.getAttemptId(), checkpointId, serializedState));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec2.getAttemptId(), checkpointId, serializedState));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec3.getAttemptId(), checkpointId, serializedState));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statelessExec1.getAttemptId(), checkpointId));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statelessExec2.getAttemptId(), checkpointId));
+
+ assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
+ assertEquals(0, coord.getNumberOfPendingCheckpoints());
+
+ // let the coordinator inject the state
+ coord.restoreLatestCheckpointedState(map, true, false);
+
+ // verify that each stateful vertex got the state
+ verify(statefulExec1, times(1)).setInitialState(serializedState);
+ verify(statefulExec2, times(1)).setInitialState(serializedState);
+ verify(statefulExec3, times(1)).setInitialState(serializedState);
+ verify(statelessExec1, times(0)).setInitialState(Mockito.<SerializedValue<StateHandle<?>>>any());
+ verify(statelessExec2, times(0)).setInitialState(Mockito.<SerializedValue<StateHandle<?>>>any());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testStateOnlyPartiallyAvailable() {
+ try {
+ final SerializedValue<StateHandle<?>> serializedState = new SerializedValue<StateHandle<?>>(
+ new LocalStateHandle(Collections.<String,OperatorState<?>>emptyMap()));
+
+ final JobID jid = new JobID();
+ final JobVertexID statefulId = new JobVertexID();
+ final JobVertexID statelessId = new JobVertexID();
+
+ Execution statefulExec1 = mockExecution();
+ Execution statefulExec2 = mockExecution();
+ Execution statefulExec3 = mockExecution();
+ Execution statelessExec1 = mockExecution();
+ Execution statelessExec2 = mockExecution();
+
+ ExecutionVertex stateful1 = mockExecutionVertex(statefulExec1, statefulId, 0);
+ ExecutionVertex stateful2 = mockExecutionVertex(statefulExec2, statefulId, 1);
+ ExecutionVertex stateful3 = mockExecutionVertex(statefulExec3, statefulId, 2);
+ ExecutionVertex stateless1 = mockExecutionVertex(statelessExec1, statelessId, 0);
+ ExecutionVertex stateless2 = mockExecutionVertex(statelessExec2, statelessId, 1);
+
+ ExecutionJobVertex stateful = mockExecutionJobVertex(statefulId,
+ new ExecutionVertex[] { stateful1, stateful2, stateful3 });
+ ExecutionJobVertex stateless = mockExecutionJobVertex(statelessId,
+ new ExecutionVertex[] { stateless1, stateless2 });
+
+ Map<JobVertexID, ExecutionJobVertex> map = new HashMap<JobVertexID, ExecutionJobVertex>();
+ map.put(statefulId, stateful);
+ map.put(statelessId, stateless);
+
+
+ CheckpointCoordinator coord = new CheckpointCoordinator(jid, 1, 200000L,
+ new ExecutionVertex[] { stateful1, stateful2, stateful3, stateless1, stateless2 },
+ new ExecutionVertex[] { stateful1, stateful2, stateful3, stateless1, stateless2 },
+ new ExecutionVertex[0]);
+
+ // create ourselves a checkpoint with state
+ final long timestamp = 34623786L;
+ coord.triggerCheckpoint(timestamp);
+
+ PendingCheckpoint pending = coord.getPendingCheckpoints().values().iterator().next();
+ final long checkpointId = pending.getCheckpointId();
+
+ // the difference to the test "testSetState" is that one stateful subtask does not report state
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec1.getAttemptId(), checkpointId, serializedState));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec2.getAttemptId(), checkpointId));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statefulExec3.getAttemptId(), checkpointId, serializedState));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statelessExec1.getAttemptId(), checkpointId));
+ coord.receiveAcknowledgeMessage(new AcknowledgeCheckpoint(jid, statelessExec2.getAttemptId(), checkpointId));
+
+ assertEquals(1, coord.getNumberOfRetainedSuccessfulCheckpoints());
+ assertEquals(0, coord.getNumberOfPendingCheckpoints());
+
+ // let the coordinator inject the state
+ try {
+ coord.restoreLatestCheckpointedState(map, true, true);
+ fail("this should fail with an exception");
+ }
+ catch (IllegalStateException e) {
+ // swish!
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testNoCheckpointAvailable() {
+ try {
+ CheckpointCoordinator coord = new CheckpointCoordinator(new JobID(), 1, 200000L,
+ new ExecutionVertex[] { mock(ExecutionVertex.class) },
+ new ExecutionVertex[] { mock(ExecutionVertex.class) },
+ new ExecutionVertex[0]);
+
+ try {
+ coord.restoreLatestCheckpointedState(new HashMap<JobVertexID, ExecutionJobVertex>(), true, false);
+ fail("this should throw an exception");
+ }
+ catch (IllegalStateException e) {
+ // expected
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ // ------------------------------------------------------------------------
+
+ private Execution mockExecution() {
+ Execution mock = mock(Execution.class);
+ when(mock.getAttemptId()).thenReturn(new ExecutionAttemptID());
+ when(mock.getState()).thenReturn(ExecutionState.CREATED);
+ return mock;
+ }
+
+ private ExecutionVertex mockExecutionVertex(Execution execution, JobVertexID vertexId, int subtask) {
+ ExecutionVertex mock = mock(ExecutionVertex.class);
+ when(mock.getJobvertexId()).thenReturn(vertexId);
+ when(mock.getParallelSubtaskIndex()).thenReturn(subtask);
+ when(mock.getCurrentExecutionAttempt()).thenReturn(execution);
+ return mock;
+ }
+
+ private ExecutionJobVertex mockExecutionJobVertex(JobVertexID id, ExecutionVertex[] vertices) {
+ ExecutionJobVertex vertex = mock(ExecutionJobVertex.class);
+ when(vertex.getParallelism()).thenReturn(vertices.length);
+ when(vertex.getJobVertexId()).thenReturn(id);
+ when(vertex.getTaskVertices()).thenReturn(vertices);
+ return vertex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
new file mode 100644
index 0000000..3d9a155
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CoordinatorShutdownTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import akka.actor.ActorRef;
+import akka.pattern.Patterns;
+
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.executiongraph.ExecutionGraph;
+import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
+import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
+import org.apache.flink.runtime.jobmanager.Tasks;
+
+import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster;
+import org.junit.Test;
+
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.*;
+
+public class CoordinatorShutdownTest {
+
+ @Test
+ public void testCoordinatorShutsDownOnFailure() {
+ LocalFlinkMiniCluster cluster = null;
+ try {
+ Configuration noTaskManagerConfig = new Configuration();
+ noTaskManagerConfig.setInteger(ConfigConstants.LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER, 0);
+ cluster = new LocalFlinkMiniCluster(noTaskManagerConfig, true);
+
+ // build a test graph with snapshotting enabled
+ AbstractJobVertex vertex = new AbstractJobVertex("Test Vertex");
+ vertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
+
+ JobGraph testGraph = new JobGraph("test job", vertex);
+ testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 5000));
+
+ ActorRef jobManager = cluster.getJobManager();
+
+ FiniteDuration timeout = new FiniteDuration(60, TimeUnit.SECONDS);
+ JobManagerMessages.SubmitJob submitMessage = new JobManagerMessages.SubmitJob(testGraph, false);
+
+ // submit is successful, but then the job dies because no TaskManager / slot is available
+ Future<Object> submitFuture = Patterns.ask(jobManager, submitMessage, timeout.toMillis());
+ Await.result(submitFuture, timeout);
+
+ // get the execution graph and make sure the coordinator is properly shut down
+ Future<Object> jobRequestFuture = Patterns.ask(jobManager,
+ new JobManagerMessages.RequestJob(testGraph.getJobID()), timeout.toMillis());
+
+ ExecutionGraph graph = ((JobManagerMessages.JobFound) Await.result(jobRequestFuture, timeout)).executionGraph();
+
+ assertNotNull(graph);
+ graph.waitUntilFinished();
+
+ CheckpointCoordinator coord = graph.getCheckpointCoordinator();
+ assertTrue(coord == null || coord.isShutdown());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster.awaitTermination();
+ }
+ }
+ }
+
+ @Test
+ public void testCoordinatorShutsDownOnSuccess() {
+ LocalFlinkMiniCluster cluster = null;
+ try {
+ cluster = new LocalFlinkMiniCluster(new Configuration(), true);
+
+ // build a test graph with snapshotting enabled
+ AbstractJobVertex vertex = new AbstractJobVertex("Test Vertex");
+ vertex.setInvokableClass(Tasks.NoOpInvokable.class);
+ List<JobVertexID> vertexIdList = Collections.singletonList(vertex.getID());
+
+ JobGraph testGraph = new JobGraph("test job", vertex);
+ testGraph.setSnapshotSettings(new JobSnapshottingSettings(vertexIdList, vertexIdList, vertexIdList, 5000));
+
+ ActorRef jobManager = cluster.getJobManager();
+
+ FiniteDuration timeout = new FiniteDuration(60, TimeUnit.SECONDS);
+ JobManagerMessages.SubmitJob submitMessage = new JobManagerMessages.SubmitJob(testGraph, false);
+
+ // submit is successful, but then the job dies because no TaskManager / slot is available
+ Future<Object> submitFuture = Patterns.ask(jobManager, submitMessage, timeout.toMillis());
+ Await.result(submitFuture, timeout);
+
+ // get the execution graph and make sure the coordinator is properly shut down
+ Future<Object> jobRequestFuture = Patterns.ask(jobManager,
+ new JobManagerMessages.RequestJob(testGraph.getJobID()), timeout.toMillis());
+
+ ExecutionGraph graph = ((JobManagerMessages.JobFound) Await.result(jobRequestFuture, timeout)).executionGraph();
+
+ assertNotNull(graph);
+ graph.waitUntilFinished();
+
+ CheckpointCoordinator coord = graph.getCheckpointCoordinator();
+ assertTrue(coord == null || coord.isShutdown());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster.awaitTermination();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/test/java/org/apache/flink/runtime/messages/CheckpointMessagesTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/messages/CheckpointMessagesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/messages/CheckpointMessagesTest.java
index fe1a598..9a9f486 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/messages/CheckpointMessagesTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/messages/CheckpointMessagesTest.java
@@ -22,32 +22,28 @@ import static org.junit.Assert.*;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.runtime.messages.checkpoint.AbortCheckpoint;
import org.apache.flink.runtime.messages.checkpoint.ConfirmCheckpoint;
import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
import org.apache.flink.runtime.messages.checkpoint.TriggerCheckpoint;
-import org.apache.flink.runtime.state.OperatorState;
import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.runtime.testutils.CommonTestUtils;
+import org.apache.flink.runtime.util.SerializedValue;
import org.junit.Test;
import java.io.IOException;
import java.io.Serializable;
-import java.util.Map;
public class CheckpointMessagesTest {
@Test
public void testTriggerAndConfirmCheckpoint() {
try {
- ConfirmCheckpoint cc = new ConfirmCheckpoint(new JobID(), new ExecutionAttemptID(), 45287698767345L);
+ ConfirmCheckpoint cc = new ConfirmCheckpoint(new JobID(), new ExecutionAttemptID(), 45287698767345L, 467L);
testSerializabilityEqualsHashCode(cc);
TriggerCheckpoint tc = new TriggerCheckpoint(new JobID(), new ExecutionAttemptID(), 347652734L, 7576752L);
testSerializabilityEqualsHashCode(tc);
-
- AbortCheckpoint ac = new AbortCheckpoint(new JobID(), new ExecutionAttemptID(), 1365762983745L);
- testSerializabilityEqualsHashCode(ac);
+
}
catch (Exception e) {
e.printStackTrace();
@@ -62,7 +58,8 @@ public class CheckpointMessagesTest {
new JobID(), new ExecutionAttemptID(), 569345L);
AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
- new JobID(), new ExecutionAttemptID(), 87658976143L, new MyHandle());
+ new JobID(), new ExecutionAttemptID(), 87658976143L,
+ new SerializedValue<StateHandle<?>>(new MyHandle()));
testSerializabilityEqualsHashCode(noState);
testSerializabilityEqualsHashCode(withState);
@@ -81,12 +78,12 @@ public class CheckpointMessagesTest {
assertNotNull(copy.toString());
}
- private static class MyHandle implements StateHandle {
+ private static class MyHandle implements StateHandle<Serializable> {
private static final long serialVersionUID = 8128146204128728332L;
@Override
- public Map<String, OperatorState<?>> getState(ClassLoader userClassloader) {
+ public Serializable getState() {
return null;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
index 735f67e..0f62b27 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.operators.testutils;
-import akka.actor.ActorRef;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
@@ -41,6 +40,7 @@ import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.memorymanager.DefaultMemoryManager;
import org.apache.flink.runtime.memorymanager.MemoryManager;
+import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.types.Record;
import org.apache.flink.util.MutableObjectIterator;
import org.mockito.invocation.InvocationOnMock;
@@ -264,7 +264,12 @@ public class MockEnvironment implements Environment {
}
@Override
- public ActorRef getJobManager() {
- return ActorRef.noSender();
+ public void acknowledgeCheckpoint(long checkpointId) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void acknowledgeCheckpoint(long checkpointId, StateHandle<?> state) {
+ throw new UnsupportedOperationException();
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
index 1016fa0..e8acccc 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java
@@ -30,8 +30,10 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
import org.apache.flink.runtime.jobgraph.DistributionPattern;
import org.apache.flink.runtime.jobgraph.JobGraph;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.ScheduleMode;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.streaming.api.graph.StreamGraph.StreamLoop;
@@ -77,19 +79,10 @@ public class StreamingJobGraphGenerator {
public JobGraph createJobGraph(String jobName) {
jobGraph = new JobGraph(jobName);
- // Turn lazy scheduling off
+ // make sure that all vertices start immediately
jobGraph.setScheduleMode(ScheduleMode.ALL);
- jobGraph.setCheckpointingEnabled(streamGraph.isCheckpointingEnabled());
- jobGraph.setCheckpointingInterval(streamGraph.getCheckpointingInterval());
-
- if (jobGraph.isCheckpointingEnabled()) {
- int executionRetries = streamGraph.getExecutionConfig().getNumberOfExecutionRetries();
- if (executionRetries != -1) {
- jobGraph.setNumberOfExecutionRetries(executionRetries);
- } else {
- jobGraph.setNumberOfExecutionRetries(Integer.MAX_VALUE);
- }
- }
+
+
init();
setChaining();
@@ -97,6 +90,8 @@ public class StreamingJobGraphGenerator {
setPhysicalEdges();
setSlotSharing();
+
+ configureCheckpointing();
return jobGraph;
}
@@ -356,4 +351,55 @@ public class StreamingJobGraphGenerator {
ccg.addVertex(tail);
}
}
+
+ private void configureCheckpointing() {
+
+ if (streamGraph.isCheckpointingEnabled()) {
+ long interval = streamGraph.getCheckpointingInterval();
+ if (interval < 1) {
+ throw new IllegalArgumentException("The checkpoint interval must be positive");
+ }
+
+ // gather source and sink IDs
+ HashSet<JobVertexID> sourceIds = new HashSet<JobVertexID>();
+ HashSet<JobVertexID> sinkIds = new HashSet<JobVertexID>();
+ for (AbstractJobVertex vertex : jobVertices.values()) {
+ if (vertex.isInputVertex()) {
+ sourceIds.add(vertex.getID());
+ }
+ if (vertex.isOutputVertex()) {
+ sinkIds.add(vertex.getID());
+ }
+ }
+
+ HashSet<JobVertexID> sourceorSink = new HashSet<JobVertexID>();
+ sourceorSink.addAll(sourceIds);
+ sourceorSink.addAll(sinkIds);
+
+ // collect the vertices that receive "trigger checkpoint" messages.
+ // currently, these are all the sources
+ List<JobVertexID> triggerVertices = new ArrayList<JobVertexID>(sourceIds);
+
+ // collect the vertices that need to acknowledge the checkpoint
+ // currently, these are the sources and sinks
+ // the sources acknowledge their state backup, the sinks the arrival of the barriers
+ List<JobVertexID> ackVertices = new ArrayList<JobVertexID>(sourceorSink);
+
+ // collect the vertices that receive "commit checkpoint" messages
+ // currently, these are only the sources
+ List<JobVertexID> commitVertices = new ArrayList<JobVertexID>(sourceIds);
+
+ JobSnapshottingSettings settings = new JobSnapshottingSettings(
+ triggerVertices, ackVertices, commitVertices, interval);
+
+ jobGraph.setSnapshotSettings(settings);
+
+ int executionRetries = streamGraph.getExecutionConfig().getNumberOfExecutionRetries();
+ if (executionRetries != -1) {
+ jobGraph.setNumberOfExecutionRetries(executionRetries);
+ } else {
+ jobGraph.setNumberOfExecutionRetries(Integer.MAX_VALUE);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 82486e8..b4a11aa 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -25,11 +25,11 @@ import org.apache.flink.runtime.event.task.TaskEvent;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
import org.apache.flink.runtime.jobgraph.tasks.BarrierTransceiver;
+import org.apache.flink.runtime.jobgraph.tasks.CheckpointCommittingOperator;
+import org.apache.flink.runtime.jobgraph.tasks.CheckpointedOperator;
import org.apache.flink.runtime.jobgraph.tasks.OperatorStateCarrier;
-import org.apache.flink.runtime.messages.CheckpointingMessages;
import org.apache.flink.runtime.state.LocalStateHandle;
import org.apache.flink.runtime.state.OperatorState;
-import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.runtime.util.event.EventListener;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.ChainableStreamOperator;
@@ -43,10 +43,10 @@ import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import akka.actor.ActorRef;
public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTaskContext<OUT>,
- BarrierTransceiver, OperatorStateCarrier {
+ OperatorStateCarrier<LocalStateHandle>, CheckpointedOperator, CheckpointCommittingOperator,
+ BarrierTransceiver {
private static final Logger LOG = LoggerFactory.getLogger(StreamTask.class);
@@ -110,18 +110,12 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
*/
@Override
public void confirmBarrier(long barrierID) throws IOException {
-
if (configuration.getStateMonitoring() && !states.isEmpty()) {
- getEnvironment().getJobManager().tell(
- new CheckpointingMessages.StateBarrierAck(getEnvironment().getJobID(), getEnvironment()
- .getJobVertexId(), context.getIndexOfThisSubtask(), barrierID,
- new LocalStateHandle(states)), ActorRef.noSender());
- } else {
- getEnvironment().getJobManager().tell(
- new CheckpointingMessages.BarrierAck(getEnvironment().getJobID(), getEnvironment().getJobVertexId(),
- context.getIndexOfThisSubtask(), barrierID), ActorRef.noSender());
+ getEnvironment().acknowledgeCheckpoint(barrierID, new LocalStateHandle(states));
+ }
+ else {
+ getEnvironment().acknowledgeCheckpoint(barrierID);
}
-
}
public void setInputsOutputs() {
@@ -304,17 +298,30 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
@Override
public String toString() {
- return configuration.getOperatorName() + " (" + context.getIndexOfThisSubtask() + ")";
+ return getEnvironment().getTaskNameWithSubtasks();
}
/**
* Re-injects the user states into the map
*/
@Override
- public void injectState(StateHandle stateHandle) {
- this.states.putAll(stateHandle.getState(userClassLoader));
+ public void setInitialState(LocalStateHandle stateHandle) {
+ this.states.putAll(stateHandle.getState());
}
+ @Override
+ public void triggerCheckpoint(long checkpointId, long timestamp) {
+ broadcastBarrierFromSource(checkpointId);
+ }
+
+ @Override
+ public void confirmCheckpoint(long checkpointId, long timestamp) {
+ // we do nothing here so far. this should call commit on the source function, for example
+ }
+
+
+
+
private class SuperstepEventListener implements EventListener<TaskEvent> {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
index c4bd095..fd70bfb 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/StreamCheckpointingITCase.java
@@ -51,7 +51,7 @@ public class StreamCheckpointingITCase {
private static final int NUM_TASK_SLOTS = 3;
private static final int PARALLELISM = NUM_TASK_MANAGERS * NUM_TASK_SLOTS;
- private static final long NUM_STRINGS = 4000000;
+ private static final long NUM_STRINGS = 10000000L;
private static ForkableFlinkMiniCluster cluster;
@@ -99,7 +99,7 @@ public class StreamCheckpointingITCase {
StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment(
"localhost", cluster.getJobManagerRPCPort());
env.setParallelism(PARALLELISM);
- env.enableCheckpointing(200);
+ env.enableCheckpointing(1000);
env.getConfig().disableSysoutLogging();
DataStream<String> stream = env.addSource(new RichParallelSourceFunction<String>() {
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java
index 361621b..877893f 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/TaskManagerFailureRecoveryITCase.java
@@ -72,7 +72,7 @@ public class TaskManagerFailureRecoveryITCase {
config.setInteger(ConfigConstants.LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER, 2);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 16);
-
+
config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "500 ms");
config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "20 s");
config.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 20);
[05/10] flink git commit: [streaming] Integrate new checkpointed
interface with StreamTask, StreamOperator, and PersistentKafkaSource
Posted by se...@apache.org.
[streaming] Integrate new checkpointed interface with StreamTask, StreamOperator, and PersistentKafkaSource
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/3b2ee23b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/3b2ee23b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/3b2ee23b
Branch: refs/heads/master
Commit: 3b2ee23b65544a5f517e77cb55911bacae4117c6
Parents: ededb6b
Author: Stephan Ewen <se...@apache.org>
Authored: Tue May 5 16:08:50 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:58 2015 +0200
----------------------------------------------------------------------
.../jobgraph/tasks/BarrierTransceiver.java | 43 ------
.../tasks/CheckpointCommittingOperator.java | 2 +-
.../jobgraph/tasks/CheckpointedOperator.java | 2 +-
.../jobgraph/tasks/OperatorStateCarrier.java | 2 +-
.../flink/runtime/state/LocalStateHandle.java | 14 +-
.../apache/flink/runtime/state/StateUtils.java | 4 +-
.../apache/flink/runtime/taskmanager/Task.java | 19 +--
.../checkpoint/CheckpointStateRestoreTest.java | 7 +-
.../kafka/api/simple/PersistentKafkaSource.java | 27 ++--
.../streaming/api/checkpoint/Checkpointed.java | 20 ++-
.../streaming/api/operators/StreamOperator.java | 53 ++++++-
.../streaming/runtime/tasks/OutputHandler.java | 4 +-
.../streaming/runtime/tasks/StreamTask.java | 150 ++++++++++---------
.../runtime/tasks/StreamingRuntimeContext.java | 60 +-------
.../runtime/tasks/StreamingSuperstep.java | 37 ++++-
.../streaming/runtime/io/BarrierBufferTest.java | 4 +-
.../ProcessFailureStreamingRecoveryITCase.java | 31 ++--
17 files changed, 229 insertions(+), 250 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/BarrierTransceiver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/BarrierTransceiver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/BarrierTransceiver.java
deleted file mode 100644
index 1dd6a90..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/BarrierTransceiver.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobgraph.tasks;
-
-
-import java.io.IOException;
-
-/**
- * A BarrierTransceiver describes an operator's barrier checkpointing behavior used for
- * fault tolerance. In the most common case [[broadcastBarrier]] is being expected to be called
- * periodically upon receiving a checkpoint barrier. Furthermore, a [[confirmBarrier]] method should
- * be implemented and used for acknowledging a specific checkpoint checkpoint.
- */
-public interface BarrierTransceiver {
-
- /**
- * A callback for notifying an operator of a new checkpoint barrier.
- * @param barrierID
- */
- public void broadcastBarrierFromSource(long barrierID);
-
- /**
- * A callback for confirming that a barrier checkpoint is complete
- * @param barrierID
- */
- public void confirmBarrier(long barrierID) throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java
index 69cb1f8..be203d2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java
@@ -20,5 +20,5 @@ package org.apache.flink.runtime.jobgraph.tasks;
public interface CheckpointCommittingOperator {
- void confirmCheckpoint(long checkpointId, long timestamp);
+ void confirmCheckpoint(long checkpointId, long timestamp) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java
index d07b07e..17ba947 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java
@@ -20,5 +20,5 @@ package org.apache.flink.runtime.jobgraph.tasks;
public interface CheckpointedOperator {
- void triggerCheckpoint(long checkpointId, long timestamp);
+ void triggerCheckpoint(long checkpointId, long timestamp) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
index 576edb6..fb5e63f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
@@ -33,6 +33,6 @@ public interface OperatorStateCarrier<T extends StateHandle<?>> {
*
* @param stateHandle The handle to the state.
*/
- public void setInitialState(T stateHandle);
+ public void setInitialState(T stateHandle) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
index f47b054..fa0c515 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
@@ -18,23 +18,23 @@
package org.apache.flink.runtime.state;
-import java.util.Map;
+import java.io.Serializable;
/**
* A StateHandle that includes a map of operator states directly.
*/
-public class LocalStateHandle implements StateHandle<Map<String, OperatorState<?>>> {
+public class LocalStateHandle implements StateHandle<Serializable> {
private static final long serialVersionUID = 2093619217898039610L;
- private final Map<String, OperatorState<?>> stateMap;
+ private final Serializable state;
- public LocalStateHandle(Map<String,OperatorState<?>> state) {
- this.stateMap = state;
+ public LocalStateHandle(Serializable state) {
+ this.state = state;
}
@Override
- public Map<String,OperatorState<?>> getState() {
- return stateMap;
+ public Serializable getState() {
+ return state;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java
index 2cdfef3..fbd76ba 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java
@@ -37,7 +37,9 @@ public class StateUtils {
* @param state The state handle.
* @param <T> Type bound for the
*/
- public static <T extends StateHandle<?>> void setOperatorState(OperatorStateCarrier<?> op, StateHandle<?> state) {
+ public static <T extends StateHandle<?>> void setOperatorState(OperatorStateCarrier<?> op, StateHandle<?> state)
+ throws Exception
+ {
@SuppressWarnings("unchecked")
OperatorStateCarrier<T> typedOp = (OperatorStateCarrier<T>) op;
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index 1578e4b..8630edf 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -45,7 +45,6 @@ import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobgraph.tasks.BarrierTransceiver;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointCommittingOperator;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointedOperator;
import org.apache.flink.runtime.jobgraph.tasks.OperatorStateCarrier;
@@ -57,6 +56,7 @@ import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.runtime.state.StateUtils;
import org.apache.flink.runtime.util.SerializedValue;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -872,23 +872,6 @@ public class Task implements Runnable {
};
executeAsyncCallRunnable(runnable, "Checkpoint Trigger");
}
- else if (invokable instanceof BarrierTransceiver) {
- final BarrierTransceiver barrierTransceiver = (BarrierTransceiver) invokable;
- final Logger logger = LOG;
-
- Runnable runnable = new Runnable() {
- @Override
- public void run() {
- try {
- barrierTransceiver.broadcastBarrierFromSource(checkpointID);
- }
- catch (Throwable t) {
- logger.error("Error while triggering checkpoint barriers", t);
- }
- }
- };
- executeAsyncCallRunnable(runnable, "Checkpoint Trigger");
- }
else {
LOG.error("Task received a checkpoint request, but is not a checkpointing task - "
+ taskNameWithSubtask);
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
index 51c4890..e1cf061 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStateRestoreTest.java
@@ -27,15 +27,14 @@ import org.apache.flink.runtime.executiongraph.ExecutionVertex;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
import org.apache.flink.runtime.state.LocalStateHandle;
-import org.apache.flink.runtime.state.OperatorState;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializableObject;
import org.apache.flink.runtime.util.SerializedValue;
import org.junit.Test;
import org.mockito.Mockito;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -52,7 +51,7 @@ public class CheckpointStateRestoreTest {
public void testSetState() {
try {
final SerializedValue<StateHandle<?>> serializedState = new SerializedValue<StateHandle<?>>(
- new LocalStateHandle(Collections.<String,OperatorState<?>>emptyMap()));
+ new LocalStateHandle(new SerializableObject()));
final JobID jid = new JobID();
final JobVertexID statefulId = new JobVertexID();
@@ -121,7 +120,7 @@ public class CheckpointStateRestoreTest {
public void testStateOnlyPartiallyAvailable() {
try {
final SerializedValue<StateHandle<?>> serializedState = new SerializedValue<StateHandle<?>>(
- new LocalStateHandle(Collections.<String,OperatorState<?>>emptyMap()));
+ new LocalStateHandle(new SerializableObject()));
final JobID jid = new JobID();
final JobVertexID statefulId = new JobVertexID();
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java
index 2af8fb2..a842160 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java
@@ -28,6 +28,7 @@ import com.google.common.base.Preconditions;
import kafka.consumer.ConsumerConfig;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.OperatorState;
+import org.apache.flink.streaming.api.checkpoint.Checkpointed;
import org.apache.flink.streaming.runtime.tasks.StreamingRuntimeContext;
import org.apache.flink.streaming.connectors.ConnectorSource;
import org.apache.flink.streaming.connectors.kafka.api.simple.iterator.KafkaConsumerIterator;
@@ -51,7 +52,7 @@ import org.slf4j.LoggerFactory;
* @param <OUT>
* Type of the messages on the topic.
*/
-public class PersistentKafkaSource<OUT> extends ConnectorSource<OUT> {
+public class PersistentKafkaSource<OUT> extends ConnectorSource<OUT> implements Checkpointed<HashMap<Integer, KafkaOffset>> {
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(PersistentKafkaSource.class);
@@ -202,13 +203,13 @@ public class PersistentKafkaSource<OUT> extends ConnectorSource<OUT> {
if (indexOfSubtask >= numberOfPartitions) {
LOG.info("Creating idle consumer because this subtask ({}) is higher than the number partitions ({})", indexOfSubtask + 1, numberOfPartitions);
iterator = new KafkaIdleConsumerIterator();
- } else {
- if (context.containsState("kafka")) {
+ }
+ else {
+ if (partitionOffsets != null) {
+ // we have restored state
LOG.info("Initializing PersistentKafkaSource from existing state.");
- kafkaOffSetOperatorState = (OperatorState<Map<Integer, KafkaOffset>>) context.getState("kafka");
-
- partitionOffsets = kafkaOffSetOperatorState.getState();
- } else {
+ }
+ else {
LOG.info("No existing state found. Creating new");
partitionOffsets = new HashMap<Integer, KafkaOffset>();
@@ -217,8 +218,6 @@ public class PersistentKafkaSource<OUT> extends ConnectorSource<OUT> {
}
kafkaOffSetOperatorState = new OperatorState<Map<Integer, KafkaOffset>>(partitionOffsets);
-
- context.registerState("kafka", kafkaOffSetOperatorState);
}
iterator = new KafkaMultiplePartitionsIterator(topicId, partitionOffsets, kafkaTopicUtils, this.consumerConfig);
@@ -272,4 +271,14 @@ public class PersistentKafkaSource<OUT> extends ConnectorSource<OUT> {
Properties props = (Properties) in.readObject();
consumerConfig = new ConsumerConfig(props);
}
+
+ @Override
+ public HashMap<Integer, KafkaOffset> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+ return new HashMap<Integer, KafkaOffset>(this.partitionOffsets);
+ }
+
+ @Override
+ public void restoreState(HashMap<Integer, KafkaOffset> state) {
+ this.partitionOffsets = state;
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
index f491dd3..2cab7a3 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
@@ -18,7 +18,7 @@
package org.apache.flink.streaming.api.checkpoint;
-import org.apache.flink.runtime.state.OperatorState;
+import java.io.Serializable;
/**
* This method must be implemented by functions that have state that needs to be
@@ -31,12 +31,14 @@ import org.apache.flink.runtime.state.OperatorState;
* continue to work and mutate the state, even while the state snapshot is being accessed,
* can implement the {@link org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously}
* interface.</p>
+ *
+ * @param <T> The type of the operator state.
*/
-public interface Checkpointed {
+public interface Checkpointed<T extends Serializable> {
/**
- * Gets the current operator state as a checkpoint. The state must reflect all operations
- * from all prior operations if this function.
+ * Gets the current state of the function of operator. The state must reflect the result of all
+ * prior invocations to this function.
*
* @param checkpointId The ID of the checkpoint.
* @param checkpointTimestamp The timestamp of the checkpoint, as derived by
@@ -49,5 +51,13 @@ public interface Checkpointed {
* recovery), or to discard this checkpoint attempt and to continue running
* and to try again with the next checkpoint attempt.
*/
- OperatorState<?> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception;
+ T snapshotState(long checkpointId, long checkpointTimestamp) throws Exception;
+
+ /**
+ * Restores the state of the function or operator to that of a previous checkpoint.
+ * This method is invoked when a function is executed as part of a recovery run.
+ * *
+ * @param state The state to be restored.
+ */
+ void restoreState(T state);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index a361b7a..f66e394 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -26,6 +26,8 @@ import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.functions.util.FunctionUtils;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.checkpoint.CheckpointCommitter;
+import org.apache.flink.streaming.api.checkpoint.Checkpointed;
import org.apache.flink.streaming.runtime.io.IndexedReaderIterator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecordSerializer;
@@ -107,16 +109,14 @@ public abstract class StreamOperator<IN, OUT> implements Serializable {
return nextRecord;
} catch (IOException e) {
if (isRunning) {
- throw new RuntimeException("Could not read next record due to: "
- + StringUtils.stringifyException(e));
+ throw new RuntimeException("Could not read next record", e);
} else {
// Task already cancelled do nothing
return null;
}
} catch (IllegalStateException e) {
if (isRunning) {
- throw new RuntimeException("Could not read next record due to: "
- + StringUtils.stringifyException(e));
+ throw new RuntimeException("Could not read next record", e);
} else {
// Task already cancelled do nothing
return null;
@@ -215,4 +215,49 @@ public abstract class StreamOperator<IN, OUT> implements Serializable {
public Function getUserFunction() {
return userFunction;
}
+
+ // ------------------------------------------------------------------------
+ // Checkpoints and Checkpoint Confirmations
+ // ------------------------------------------------------------------------
+
+ // NOTE - ALL OF THIS CODE WORKS ONLY FOR THE FIRST OPERATOR IN THE CHAIN
+ // IT NEEDS TO BE EXTENDED TO SUPPORT CHAINS
+
+ public void restoreInitialState(Serializable state) throws Exception {
+ if (userFunction instanceof Checkpointed) {
+ setStateOnFunction(state, userFunction);
+ }
+ else {
+ throw new IllegalStateException("Trying to restore state of a non-checkpointed function");
+ }
+ }
+
+ public Serializable getStateSnapshotFromFunction(long checkpointId, long timestamp) throws Exception {
+ if (userFunction instanceof Checkpointed) {
+ return ((Checkpointed<?>) userFunction).snapshotState(checkpointId, timestamp);
+ }
+ else {
+ return null;
+ }
+ }
+
+ public void confirmCheckpointCompleted(long checkpointId, long timestamp) throws Exception {
+ if (userFunction instanceof CheckpointCommitter) {
+ try {
+ ((CheckpointCommitter) userFunction).commitCheckpoint(checkpointId);
+ }
+ catch (Exception e) {
+ throw new Exception("Error while confirming checkpoint " + checkpointId + " to the stream function", e);
+ }
+ }
+ }
+
+ private static <T extends Serializable> void setStateOnFunction(Serializable state, Function function) {
+ @SuppressWarnings("unchecked")
+ T typedState = (T) state;
+ @SuppressWarnings("unchecked")
+ Checkpointed<T> typedFunction = (Checkpointed<T>) function;
+
+ typedFunction.restoreState(typedState);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
index c579c3a..55c0551 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
@@ -88,8 +88,8 @@ public class OutputHandler<OUT> {
this.outerCollector = createChainedCollector(configuration);
}
- public void broadcastBarrier(long id) throws IOException, InterruptedException {
- StreamingSuperstep barrier = new StreamingSuperstep(id);
+ public void broadcastBarrier(long id, long timestamp) throws IOException, InterruptedException {
+ StreamingSuperstep barrier = new StreamingSuperstep(id, timestamp);
for (StreamOutput<?> streamOutput : outputMap.values()) {
streamOutput.broadcastEvent(barrier);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index b4a11aa..930c9b4 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -18,18 +18,15 @@
package org.apache.flink.streaming.runtime.tasks;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.Serializable;
import org.apache.flink.runtime.event.task.TaskEvent;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobgraph.tasks.BarrierTransceiver;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointCommittingOperator;
import org.apache.flink.runtime.jobgraph.tasks.CheckpointedOperator;
import org.apache.flink.runtime.jobgraph.tasks.OperatorStateCarrier;
import org.apache.flink.runtime.state.LocalStateHandle;
-import org.apache.flink.runtime.state.OperatorState;
import org.apache.flink.runtime.util.event.EventListener;
import org.apache.flink.streaming.api.graph.StreamConfig;
import org.apache.flink.streaming.api.operators.ChainableStreamOperator;
@@ -40,16 +37,18 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecordSerializer;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
import org.apache.flink.util.StringUtils;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTaskContext<OUT>,
- OperatorStateCarrier<LocalStateHandle>, CheckpointedOperator, CheckpointCommittingOperator,
- BarrierTransceiver {
+ OperatorStateCarrier<LocalStateHandle>, CheckpointedOperator, CheckpointCommittingOperator {
private static final Logger LOG = LoggerFactory.getLogger(StreamTask.class);
+ private final Object checkpointLock = new Object();
+
private static int numTasks;
protected StreamConfig configuration;
@@ -62,7 +61,6 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
protected volatile boolean isRunning = false;
private StreamingRuntimeContext context;
- private Map<String, OperatorState<?>> states;
protected ClassLoader userClassLoader;
@@ -90,32 +88,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
protected void initialize() {
this.userClassLoader = getUserCodeClassLoader();
this.configuration = new StreamConfig(getTaskConfiguration());
- this.states = new HashMap<String, OperatorState<?>>();
- this.context = createRuntimeContext(getEnvironment().getTaskName(), this.states);
- }
-
- @Override
- public void broadcastBarrierFromSource(long id) {
- // Only called at input vertices
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received barrier from jobmanager: " + id);
- }
- actOnBarrier(id);
- }
-
- /**
- * This method is called to confirm that a barrier has been fully processed.
- * It sends an acknowledgment to the jobmanager. In the current version if
- * there is user state it also checkpoints the state to the jobmanager.
- */
- @Override
- public void confirmBarrier(long barrierID) throws IOException {
- if (configuration.getStateMonitoring() && !states.isEmpty()) {
- getEnvironment().acknowledgeCheckpoint(barrierID, new LocalStateHandle(states));
- }
- else {
- getEnvironment().acknowledgeCheckpoint(barrierID);
- }
+ this.context = createRuntimeContext(getEnvironment().getTaskName());
}
public void setInputsOutputs() {
@@ -136,11 +109,10 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
return instanceID;
}
- public StreamingRuntimeContext createRuntimeContext(String taskName,
- Map<String, OperatorState<?>> states) {
+ public StreamingRuntimeContext createRuntimeContext(String taskName) {
Environment env = getEnvironment();
return new StreamingRuntimeContext(taskName, env, getUserCodeClassLoader(),
- getExecutionConfig(), states);
+ getExecutionConfig());
}
@Override
@@ -272,62 +244,98 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
return this.superstepListener;
}
+ // ------------------------------------------------------------------------
+ // Checkpoint and Restore
+ // ------------------------------------------------------------------------
+
/**
- * Method to be called when a barrier is received from all the input
- * channels. It should broadcast the barrier to the output operators,
- * checkpoint the state and send an ack.
- *
- * @param id
+ * Re-injects the user states into the map. Also set the state on the functions.
*/
- private synchronized void actOnBarrier(long id) {
- if (isRunning) {
- try {
- outputHandler.broadcastBarrier(id);
- confirmBarrier(id);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Superstep " + id + " processed: " + StreamTask.this);
- }
- } catch (Exception e) {
- // Only throw any exception if the vertex is still running
- if (isRunning) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
@Override
- public String toString() {
- return getEnvironment().getTaskNameWithSubtasks();
+ public void setInitialState(LocalStateHandle stateHandle) throws Exception {
+ // here, we later resolve the state handle into the actual state by
+ // loading the state described by the handle from the backup store
+ Serializable state = stateHandle.getState();
+ streamOperator.restoreInitialState(state);
}
/**
- * Re-injects the user states into the map
+ * This method is either called directly by the checkpoint coordinator, or called
+ * when all incoming channels have reported a barrier
+ *
+ * @param checkpointId
+ * @param timestamp
+ * @throws Exception
*/
@Override
- public void setInitialState(LocalStateHandle stateHandle) {
- this.states.putAll(stateHandle.getState());
+ public void triggerCheckpoint(long checkpointId, long timestamp) throws Exception {
+
+ synchronized (checkpointLock) {
+ if (isRunning) {
+ try {
+ LOG.info("Starting checkpoint " + checkpointId);
+
+ // first draw the state that should go into checkpoint
+ LocalStateHandle state;
+ try {
+ Serializable userState = streamOperator.getStateSnapshotFromFunction(checkpointId, timestamp);
+ state = userState == null ? null : new LocalStateHandle(userState);
+ }
+ catch (Exception e) {
+ throw new Exception("Error while drawing snapshot of the user state.");
+ }
+
+ // now emit the checkpoint barriers
+ outputHandler.broadcastBarrier(checkpointId, timestamp);
+
+ // now confirm the checkpoint
+ if (state == null) {
+ getEnvironment().acknowledgeCheckpoint(checkpointId);
+ } else {
+ getEnvironment().acknowledgeCheckpoint(checkpointId, state);
+ }
+ }
+ catch (Exception e) {
+ if (isRunning) {
+ throw e;
+ }
+ }
+ }
+ }
}
@Override
- public void triggerCheckpoint(long checkpointId, long timestamp) {
- broadcastBarrierFromSource(checkpointId);
+ public void confirmCheckpoint(long checkpointId, long timestamp) throws Exception {
+ // we do nothing here so far. this should call commit on the source function, for example
+ synchronized (checkpointLock) {
+ streamOperator.confirmCheckpointCompleted(checkpointId, timestamp);
+ }
}
+
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
+
@Override
- public void confirmCheckpoint(long checkpointId, long timestamp) {
- // we do nothing here so far. this should call commit on the source function, for example
+ public String toString() {
+ return getEnvironment().getTaskNameWithSubtasks();
}
-
-
+ // ------------------------------------------------------------------------
private class SuperstepEventListener implements EventListener<TaskEvent> {
@Override
public void onEvent(TaskEvent event) {
- actOnBarrier(((StreamingSuperstep) event).getId());
+ try {
+ StreamingSuperstep sStep = (StreamingSuperstep) event;
+ triggerCheckpoint(sStep.getId(), sStep.getTimestamp());
+ }
+ catch (Exception e) {
+ throw new RuntimeException(
+ "Error triggering a checkpoint as the result of receiving checkpoint barrier", e);
+ }
}
-
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
index 8ae2ced..6112e03 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
@@ -18,17 +18,13 @@
package org.apache.flink.streaming.runtime.tasks;
-import java.util.Map;
-
import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.functions.RichFunction;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.operators.util.TaskConfig;
-import org.apache.flink.runtime.state.OperatorState;
/**
* Implementation of the {@link RuntimeContext}, created by runtime stream UDF
@@ -37,65 +33,13 @@ import org.apache.flink.runtime.state.OperatorState;
public class StreamingRuntimeContext extends RuntimeUDFContext {
private final Environment env;
- private final Map<String, OperatorState<?>> operatorStates;
+
public StreamingRuntimeContext(String name, Environment env, ClassLoader userCodeClassLoader,
- ExecutionConfig executionConfig, Map<String, OperatorState<?>> operatorStates) {
+ ExecutionConfig executionConfig) {
super(name, env.getNumberOfSubtasks(), env.getIndexInSubtaskGroup(), userCodeClassLoader,
executionConfig, env.getDistributedCacheEntries());
this.env = env;
- this.operatorStates = operatorStates;
- }
-
- /**
- * Returns the operator state registered by the given name for the operator.
- *
- * @param name
- * Name of the operator state to be returned.
- * @return The operator state.
- */
- public OperatorState<?> getState(String name) {
- if (operatorStates == null) {
- throw new RuntimeException("No state has been registered for this operator.");
- } else {
- OperatorState<?> state = operatorStates.get(name);
- if (state != null) {
- return state;
- } else {
- throw new RuntimeException("No state has been registered for the name: " + name);
- }
- }
- }
-
- /**
- * Returns whether there is a state stored by the given name
- */
- public boolean containsState(String name) {
- return operatorStates.containsKey(name);
- }
-
- /**
- * This is a beta feature </br></br> Register an operator state for this
- * operator by the given name. This name can be used to retrieve the state
- * during runtime using {@link StreamingRuntimeContext#getState(String)}. To
- * obtain the {@link StreamingRuntimeContext} from the user-defined function
- * use the {@link RichFunction#getRuntimeContext()} method.
- *
- * @param name
- * The name of the operator state.
- * @param state
- * The state to be registered for this name.
- */
- public void registerState(String name, OperatorState<?> state) {
- if (state == null) {
- throw new RuntimeException("Cannot register null state");
- } else {
- if (operatorStates.containsKey(name)) {
- throw new RuntimeException("State is already registered");
- } else {
- operatorStates.put(name, state);
- }
- }
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingSuperstep.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingSuperstep.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingSuperstep.java
index 4b3419b..f749773 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingSuperstep.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingSuperstep.java
@@ -27,34 +27,57 @@ import org.apache.flink.runtime.event.task.TaskEvent;
public class StreamingSuperstep extends TaskEvent {
protected long id;
+ protected long timestamp;
- public StreamingSuperstep() {
+ public StreamingSuperstep() {}
+ public StreamingSuperstep(long id, long timestamp) {
+ this.id = id;
+ this.timestamp = timestamp;
}
- public StreamingSuperstep(long id) {
- this.id = id;
+ public long getId() {
+ return id;
}
+ public long getTimestamp() {
+ return id;
+ }
+
+ // ------------------------------------------------------------------------
+
@Override
public void write(DataOutputView out) throws IOException {
out.writeLong(id);
+ out.writeLong(timestamp);
}
@Override
public void read(DataInputView in) throws IOException {
id = in.readLong();
+ timestamp = in.readLong();
}
+
+ // ------------------------------------------------------------------------
- public long getId() {
- return id;
+ @Override
+ public int hashCode() {
+ return (int) (id ^ (id >>> 32) ^ timestamp ^(timestamp >>> 32));
}
+ @Override
public boolean equals(Object other) {
if (other == null || !(other instanceof StreamingSuperstep)) {
return false;
- } else {
- return ((StreamingSuperstep) other).id == this.id;
}
+ else {
+ StreamingSuperstep that = (StreamingSuperstep) other;
+ return that.id == this.id && that.timestamp == this.timestamp;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format("StreamingSuperstep %d @ %d", id, timestamp);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java
index b8af4ed..0afe8b5 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/runtime/io/BarrierBufferTest.java
@@ -33,8 +33,8 @@ import org.apache.flink.runtime.io.network.buffer.BufferRecycler;
import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
import org.apache.flink.runtime.util.event.EventListener;
-import org.apache.flink.streaming.runtime.io.BarrierBuffer;
import org.apache.flink.streaming.runtime.tasks.StreamingSuperstep;
+
import org.junit.Test;
public class BarrierBufferTest {
@@ -201,7 +201,7 @@ public class BarrierBufferTest {
}
protected static BufferOrEvent createSuperstep(long id, int channel) {
- return new BufferOrEvent(new StreamingSuperstep(id), channel);
+ return new BufferOrEvent(new StreamingSuperstep(id, System.currentTimeMillis()), channel);
}
protected static BufferOrEvent createBuffer(int channel) {
http://git-wip-us.apache.org/repos/asf/flink/blob/3b2ee23b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
index aeaad5c..65e39a2 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
@@ -22,7 +22,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.state.OperatorState;
+import org.apache.flink.streaming.api.checkpoint.Checkpointed;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
@@ -145,12 +145,15 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
}
}
- public static class SleepyDurableGenerateSequence extends RichParallelSourceFunction<Long> {
+ public static class SleepyDurableGenerateSequence extends RichParallelSourceFunction<Long>
+ implements Checkpointed<Long> {
private static final long SLEEP_TIME = 50;
private final File coordinateDir;
private final long end;
+
+ private long collected;
public SleepyDurableGenerateSequence(File coordinateDir, long end) {
this.coordinateDir = coordinateDir;
@@ -162,23 +165,10 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
public void run(Collector<Long> collector) throws Exception {
StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
- OperatorState<Long> collectedState;
- if (context.containsState("collected")) {
- collectedState = (OperatorState<Long>) context.getState("collected");
-
-// if (collected == 0) {
-// throw new RuntimeException("The state did not capture a completed checkpoint");
-// }
- }
- else {
- collectedState = new OperatorState<Long>(0L);
- context.registerState("collected", collectedState);
- }
final long stepSize = context.getNumberOfParallelSubtasks();
final long congruence = context.getIndexOfThisSubtask();
final long toCollect = (end % stepSize > congruence) ? (end / stepSize + 1) : (end / stepSize);
- long collected = collectedState.getState();
final File proceedFile = new File(coordinateDir, PROCEED_MARKER_FILE);
boolean checkForProceedFile = true;
@@ -196,13 +186,22 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
}
collector.collect(collected * stepSize + congruence);
- collectedState.update(collected);
collected++;
}
}
@Override
public void cancel() {}
+
+ @Override
+ public Long snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+ return collected;
+ }
+
+ @Override
+ public void restoreState(Long state) {
+ collected = state;
+ }
}
[03/10] flink git commit: [tests] Fix ProcessFailureRecovery tests by
aligning restart delay parameter with heartbeat pause.
Posted by se...@apache.org.
[tests] Fix ProcessFailureRecovery tests by aligning restart delay parameter with heartbeat pause.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/959d2278
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/959d2278
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/959d2278
Branch: refs/heads/master
Commit: 959d227837c3b0c919c467fa59f6d1ec305bc4ad
Parents: c8de656
Author: Stephan Ewen <se...@apache.org>
Authored: Tue May 12 15:03:34 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:58 2015 +0200
----------------------------------------------------------------------
.../flink/test/recovery/AbstractProcessFailureRecoveryTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/959d2278/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractProcessFailureRecoveryTest.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractProcessFailureRecoveryTest.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractProcessFailureRecoveryTest.java
index ddfbeed..d55aff4 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractProcessFailureRecoveryTest.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractProcessFailureRecoveryTest.java
@@ -115,7 +115,7 @@ public abstract class AbstractProcessFailureRecoveryTest {
jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "1000 ms");
jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "6 s");
jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 9);
- jmConfig.setString(ConfigConstants.DEFAULT_EXECUTION_RETRY_DELAY_KEY, "4 s");
+ jmConfig.setString(ConfigConstants.DEFAULT_EXECUTION_RETRY_DELAY_KEY, "10 s");
jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress));
ActorRef jmActor = JobManager.startJobManagerActors(jmConfig, jmActorSystem)._1();
[04/10] flink git commit: [FLINK-1973] [jobmanager] Task execution
state messages are logged on INFO level, rather than on DEBUG level
Posted by se...@apache.org.
[FLINK-1973] [jobmanager] Task execution state messages are logged on INFO level, rather than on DEBUG level
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ededb6b7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ededb6b7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ededb6b7
Branch: refs/heads/master
Commit: ededb6b74a2d388ff1c54ddf0f2733c65675c9a0
Parents: 9b7f8aa
Author: Stephan Ewen <se...@apache.org>
Authored: Tue May 5 00:26:05 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:58 2015 +0200
----------------------------------------------------------------------
.../org/apache/flink/runtime/executiongraph/Execution.java | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ededb6b7/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 731d70f..742a7c1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -875,10 +875,8 @@ public class Execution implements Serializable {
if (STATE_UPDATER.compareAndSet(this, currentState, targetState)) {
markTimestamp(targetState);
- if (LOG.isDebugEnabled()) {
- LOG.debug("{} ({}) switched from {} to {}.",
- getVertex().getTaskNameWithSubtaskIndex(), getAttemptId(), currentState, targetState);
- }
+ LOG.info(getVertex().getTaskNameWithSubtaskIndex() + " (" + getAttemptId() + ") switched from "
+ + currentState + " to " + targetState);
// make sure that the state transition completes normally.
// potential errors (in listeners may not affect the main logic)
[09/10] flink git commit: [tests] Adapted CoGroupITCase to use
"collect()" rather than writing temp files (increases robustness)
Posted by se...@apache.org.
[tests] Adapted CoGroupITCase to use "collect()" rather than writing temp files (increases robustness)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/e79ff4eb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/e79ff4eb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/e79ff4eb
Branch: refs/heads/master
Commit: e79ff4ebfc896c827ee5cfde57570ff42fb04e10
Parents: 4bd3525
Author: Stephan Ewen <se...@apache.org>
Authored: Tue May 12 18:47:57 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:58 2015 +0200
----------------------------------------------------------------------
.../test/javaApiOperators/CoGroupITCase.java | 204 ++++++++++++-------
1 file changed, 130 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/e79ff4eb/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
index 84c05d6..115ab0d 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.common.functions.RichCoGroupFunction;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.tuple.Tuple;
import org.apache.flink.api.java.tuple.Tuple1;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
@@ -34,19 +35,19 @@ import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
import org.apache.flink.test.util.MultipleProgramsTestBase;
import org.apache.flink.util.Collector;
-import org.junit.After;
+
import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import static org.junit.Assert.assertEquals;
+
@RunWith(Parameterized.class)
public class CoGroupITCase extends MultipleProgramsTestBase {
@@ -54,27 +55,12 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
super(mode);
}
- private String resultPath;
- private String expected;
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
-
- @Before
- public void before() throws Exception{
- resultPath = tempFolder.newFile().toURI().toString();
- }
-
- @After
- public void after() throws Exception{
- compareResultsByLinesInMemory(expected, resultPath);
- }
-
+ /*
+ * CoGroup on tuples with key field selector
+ */
@Test
public void testCoGroupTuplesWithKeyFieldSelector() throws Exception {
- /*
- * CoGroup on tuples with key field selector
- */
+
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@@ -82,14 +68,15 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
DataSet<Tuple2<Integer, Integer>> coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).with(new Tuple5CoGroup());
- coGroupDs.writeAsCsv(resultPath);
- env.execute();
+ List<Tuple2<Integer, Integer>> result = coGroupDs.collect();
- expected = "1,0\n" +
+ String expected = "1,0\n" +
"2,6\n" +
"3,24\n" +
"4,60\n" +
"5,120\n";
+
+ compareResultAsTuples(result, expected);
}
@Test
@@ -105,15 +92,16 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<CustomType> coGroupDs = ds.coGroup(ds2).where(new KeySelector4()).equalTo(new
KeySelector5()).with(new CustomTypeCoGroup());
- coGroupDs.writeAsText(resultPath);
- env.execute();
+ List<CustomType> result = coGroupDs.collect();
- expected = "1,0,test\n" +
+ String expected = "1,0,test\n" +
"2,6,test\n" +
"3,24,test\n" +
"4,60,test\n" +
"5,120,test\n" +
"6,210,test\n";
+
+ compareResultAsText(result, expected);
}
public static class KeySelector4 implements KeySelector<CustomType, Integer> {
@@ -144,14 +132,15 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).with(new Tuple3ReturnLeft());
- coGroupDs.writeAsCsv(resultPath);
- env.execute();
+ List<Tuple3<Integer, Long, String>> result = coGroupDs.collect();
- expected = "1,1,Hi\n" +
+ String expected = "1,1,Hi\n" +
"2,2,Hello\n" +
"3,2,Hello world\n" +
"4,3,Hello world, how are you?\n" +
"5,3,I am fine.\n";
+
+ compareResultAsTuples(result, expected);
}
@Test
@@ -165,16 +154,17 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
DataSet<Tuple5<Integer, Long, Integer, String, Long>> coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).with(new Tuple5ReturnRight());
+
+ List<Tuple5<Integer, Long, Integer, String, Long>> result = coGroupDs.collect();
- coGroupDs.writeAsCsv(resultPath);
- env.execute();
-
- expected = "1,1,0,Hallo,1\n" +
+ String expected = "1,1,0,Hallo,1\n" +
"2,2,1,Hallo Welt,2\n" +
"2,3,2,Hallo Welt wie,1\n" +
"3,4,3,Hallo Welt wie gehts?,2\n" +
"3,5,4,ABC,2\n" +
"3,6,5,BCD,3\n";
+
+ compareResultAsTuples(result, expected);
}
@Test
@@ -191,14 +181,15 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
DataSet<Tuple3<Integer, Integer, Integer>> coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).with(new Tuple5CoGroupBC()).withBroadcastSet(intDs, "ints");
- coGroupDs.writeAsCsv(resultPath);
- env.execute();
+ List<Tuple3<Integer, Integer, Integer>> result = coGroupDs.collect();
- expected = "1,0,55\n" +
+ String expected = "1,0,55\n" +
"2,6,55\n" +
"3,24,55\n" +
"4,60,55\n" +
"5,120,55\n";
+
+ compareResultAsTuples(result, expected);
}
@Test
@@ -215,10 +206,9 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple3<Integer, Long, String>> coGroupDs = ds.coGroup(ds2).where(2).equalTo(new
KeySelector2()).with(new MixedCoGroup());
- coGroupDs.writeAsCsv(resultPath);
- env.execute();
+ List<Tuple3<Integer, Long, String>> result = coGroupDs.collect();
- expected = "0,1,test\n" +
+ String expected = "0,1,test\n" +
"1,2,test\n" +
"2,5,test\n" +
"3,15,test\n" +
@@ -233,6 +223,8 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
"12,5,test\n" +
"13,5,test\n" +
"14,5,test\n";
+
+ compareResultAsTuples(result, expected);
}
public static class KeySelector2 implements KeySelector<CustomType, Integer> {
@@ -257,10 +249,9 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<CustomType> coGroupDs = ds2.coGroup(ds).where(new KeySelector3()).equalTo(2).with
(new MixedCoGroup2());
- coGroupDs.writeAsText(resultPath);
- env.execute();
+ List<CustomType> result = coGroupDs.collect();
- expected = "0,1,test\n" +
+ String expected = "0,1,test\n" +
"1,2,test\n" +
"2,5,test\n" +
"3,15,test\n" +
@@ -276,6 +267,7 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
"13,5,test\n" +
"14,5,test\n";
+ compareResultAsText(result, expected);
}
public static class KeySelector3 implements KeySelector<CustomType, Integer> {
@@ -300,15 +292,16 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple3<Integer, Long, String>> coGrouped = ds1.coGroup(ds2).
where(0,4).equalTo(0,1).with(new Tuple5Tuple3CoGroup());
- coGrouped.writeAsCsv(resultPath);
- env.execute();
-
- expected = "1,1,Hallo\n" +
+ List<Tuple3<Integer, Long, String>> result = coGrouped.collect();
+
+ String expected = "1,1,Hallo\n" +
"2,2,Hallo Welt\n" +
"3,2,Hallo Welt wie gehts?\n" +
"3,2,ABC\n" +
"5,3,HIJ\n" +
"5,3,IJK\n";
+
+ compareResultAsTuples(result, expected);
}
@Test
@@ -326,15 +319,16 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
where(new KeySelector7()).
equalTo(new KeySelector8()).with(new Tuple5Tuple3CoGroup());
- coGrouped.writeAsCsv(resultPath);
- env.execute();
-
- expected = "1,1,Hallo\n" +
+ List<Tuple3<Integer, Long, String>> result = coGrouped.collect();
+
+ String expected = "1,1,Hallo\n" +
"2,2,Hallo Welt\n" +
"3,2,Hallo Welt wie gehts?\n" +
"3,2,ABC\n" +
"5,3,HIJ\n" +
"5,3,IJK\n";
+
+ compareResultAsTuples(result, expected);
}
public static class KeySelector7 implements KeySelector<Tuple5<Integer,Long,Integer,String,Long>,
@@ -368,15 +362,16 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<CustomType> ds2 = CollectionDataSets.getCustomTypeDataSet(env);
DataSet<CustomType> coGroupDs = ds.coGroup(ds2).where("myInt").equalTo("myInt").with(new CustomTypeCoGroup());
- coGroupDs.writeAsText(resultPath);
- env.execute();
-
- expected = "1,0,test\n" +
+ List<CustomType> result = coGroupDs.collect();
+
+ String expected = "1,0,test\n" +
"2,6,test\n" +
"3,24,test\n" +
"4,60,test\n" +
"5,120,test\n" +
"6,210,test\n";
+
+ compareResultAsText(result, expected);
}
@Test
@@ -392,12 +387,14 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedDataSet(env);
DataSet<CustomType> coGroupDs = ds.coGroup(ds2)
.where("nestedPojo.longNumber").equalTo(6).with(new CoGroup1());
- coGroupDs.writeAsText(resultPath);
- env.execute();
- expected = "-1,20000,Flink\n" +
+ List<CustomType> result = coGroupDs.collect();
+
+ String expected = "-1,20000,Flink\n" +
"-1,10000,Flink\n" +
"-1,30000,Flink\n";
+
+ compareResultAsText(result, expected);
}
public static class CoGroup1 implements CoGroupFunction<POJO, Tuple7<Integer, String, Integer, Integer, Long, String, Long>, CustomType> {
@@ -430,13 +427,14 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedDataSet(env);
DataSet<CustomType> coGroupDs = ds.coGroup(ds2)
.where(new KeySelector6()).equalTo(6).with(new CoGroup3());
- coGroupDs.writeAsText(resultPath);
- env.execute();
- expected = "-1,20000,Flink\n" +
+ List<CustomType> result = coGroupDs.collect();
+
+ String expected = "-1,20000,Flink\n" +
"-1,10000,Flink\n" +
"-1,30000,Flink\n";
+ compareResultAsText(result, expected);
}
public static class KeySelector6 implements KeySelector<POJO, Tuple1<Long>> {
@@ -480,12 +478,14 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple7<Integer, String, Integer, Integer, Long, String, Long>> ds2 = CollectionDataSets.getSmallTuplebasedDataSet(env);
DataSet<CustomType> coGroupDs = ds.coGroup(ds2)
.where(new KeySelector1()).equalTo(6).with(new CoGroup2());
- coGroupDs.writeAsText(resultPath);
- env.execute();
- expected = "-1,20000,Flink\n" +
+ List<CustomType> result = coGroupDs.collect();
+
+ String expected = "-1,20000,Flink\n" +
"-1,10000,Flink\n" +
"-1,30000,Flink\n";
+
+ compareResultAsText(result, expected);
}
@Test
@@ -496,11 +496,12 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple3<Integer, Long, String>> coGroupDs = ds1.coGroup(ds2).where(0).equalTo("*").with(new CoGroupAtomic1());
- coGroupDs.writeAsText(resultPath);
- env.execute();
-
- expected = "(1,1,Hi)\n" +
+ List<Tuple3<Integer, Long, String>> result = coGroupDs.collect();
+
+ String expected = "(1,1,Hi)\n" +
"(2,2,Hello)";
+
+ compareResultAsText(result, expected);
}
@Test
@@ -510,14 +511,69 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> coGroupDs = ds1.coGroup(ds2).where("*").equalTo(0).with(new CoGroupAtomic2());
+
+ List<Tuple3<Integer, Long, String>> result = coGroupDs.collect();
+
+ String expected = "(1,1,Hi)\n" +
+ "(2,2,Hello)";
+
+ compareResultAsText(result, expected);
+ }
- coGroupDs.writeAsText(resultPath);
- env.execute();
- expected = "(1,1,Hi)\n" +
- "(2,2,Hello)";
+ // --------------------------------------------------------------------------------------------
+ // Utilities
+ // --------------------------------------------------------------------------------------------
+
+ private <T> void compareResultAsTuples(List<T> result, String expected) {
+ compareResult(result, expected, true);
+ }
+
+ private <T> void compareResultAsText(List<T> result, String expected) {
+ compareResult(result, expected, false);
}
+
+ private <T> void compareResult(List<T> result, String expected, boolean asTuples) {
+ String[] extectedStrings = expected.split("\n");
+ String[] resultStrings = new String[result.size()];
+
+ for (int i = 0; i < resultStrings.length; i++) {
+ T val = result.get(i);
+
+ if (asTuples) {
+ if (val instanceof Tuple) {
+ Tuple t = (Tuple) val;
+ Object first = t.getField(0);
+ StringBuilder bld = new StringBuilder(first == null ? "null" : first.toString());
+ for (int pos = 1; pos < t.getArity(); pos++) {
+ Object next = t.getField(pos);
+ bld.append(',').append(next == null ? "null" : next.toString());
+ }
+ resultStrings[i] = bld.toString();
+ }
+ else {
+ throw new IllegalArgumentException(val + " is no tuple");
+ }
+ }
+ else {
+ resultStrings[i] = (val == null) ? "null" : val.toString();
+ }
+ }
+
+ assertEquals("Wrong number of elements result", extectedStrings.length, resultStrings.length);
+ Arrays.sort(extectedStrings);
+ Arrays.sort(resultStrings);
+
+ for (int i = 0; i < extectedStrings.length; i++) {
+ assertEquals(extectedStrings[i], resultStrings[i]);
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
+ // UDF classes
+ // --------------------------------------------------------------------------------------------
+
public static class KeySelector1 implements KeySelector<POJO, Long> {
private static final long serialVersionUID = 1L;
[08/10] flink git commit: [FLINK-1935] Reimplement
PersistentKafkaSource using high level Kafka API
Posted by se...@apache.org.
[FLINK-1935] Reimplement PersistentKafkaSource using high level Kafka API
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/54e95761
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/54e95761
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/54e95761
Branch: refs/heads/master
Commit: 54e957614c38fed69baf726fc86059e9b11384cb
Parents: 3b2ee23
Author: Robert Metzger <rm...@apache.org>
Authored: Thu Apr 23 12:29:29 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:58 2015 +0200
----------------------------------------------------------------------
.../flink/api/java/typeutils/TypeExtractor.java | 3 +-
.../checkpoint/CheckpointCoordinator.java | 2 +-
.../runtime/operators/RegularPactTask.java | 2 +-
.../flink/runtime/state/OperatorState.java | 87 ----
.../flink/runtime/state/StateCheckpoint.java | 59 ---
.../streaming/connectors/ConnectorSource.java | 14 +-
.../connectors/kafka/KafkaConsumerExample.java | 1 -
.../kafka/KafkaSimpleConsumerExample.java | 58 ---
.../flink/streaming/connectors/kafka/Utils.java | 3 +-
.../connectors/kafka/api/KafkaSink.java | 53 ++-
.../connectors/kafka/api/KafkaSource.java | 1 -
.../api/persistent/PersistentKafkaSource.java | 346 ++++++++++++++++
.../kafka/api/simple/KafkaTopicUtils.java | 250 -----------
.../kafka/api/simple/MessageWithMetadata.java | 46 ---
.../kafka/api/simple/PersistentKafkaSource.java | 284 -------------
.../simple/iterator/KafkaConsumerIterator.java | 46 ---
.../iterator/KafkaIdleConsumerIterator.java | 57 ---
.../KafkaMultiplePartitionsIterator.java | 116 ------
.../iterator/KafkaSinglePartitionIterator.java | 392 ------------------
.../api/simple/offset/BeginningOffset.java | 32 --
.../kafka/api/simple/offset/CurrentOffset.java | 32 --
.../kafka/api/simple/offset/GivenOffset.java | 39 --
.../kafka/api/simple/offset/KafkaOffset.java | 87 ----
.../kafka/api/simple/offset/Offset.java | 32 --
.../streaming/connectors/kafka/KafkaITCase.java | 410 +++++++++++++------
.../connectors/kafka/KafkaTopicUtilsTest.java | 152 -------
.../src/test/resources/log4j-test.properties | 2 +-
.../checkpoint/CheckpointedAsynchronously.java | 4 +-
.../api/environment/LocalStreamEnvironment.java | 9 +-
.../environment/StreamExecutionEnvironment.java | 8 +-
.../functions/source/GenericSourceFunction.java | 25 --
.../flink/streaming/api/graph/StreamGraph.java | 19 +-
.../streaming/api/operators/StreamOperator.java | 9 +-
.../streaming/api/state/PartitionableState.java | 66 ---
.../streaming/runtime/tasks/StreamTask.java | 7 +-
.../serialization/DeserializationSchema.java | 4 +-
.../serialization/JavaDefaultStringSchema.java | 7 +
.../streaming/util/serialization/RawSchema.java | 8 +
.../util/serialization/SimpleStringSchema.java | 7 +
.../streaming/api/state/OperatorStateTest.java | 44 --
40 files changed, 711 insertions(+), 2112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
index 0fec67f..2dcb76f 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/typeutils/TypeExtractor.java
@@ -1445,12 +1445,11 @@ public class TypeExtractor {
public static <X> TypeInformation<X> getForObject(X value) {
return new TypeExtractor().privateGetForObject(value);
-
}
+
@SuppressWarnings({ "unchecked", "rawtypes" })
private <X> TypeInformation<X> privateGetForObject(X value) {
Validate.notNull(value);
-
// check if we can extract the types from tuples, otherwise work with the class
if (value instanceof Tuple) {
Tuple t = (Tuple) value;
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index b3f6587..be0fcf2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -147,7 +147,7 @@ public class CheckpointCoordinator {
return;
}
shutdown = true;
- LOG.info("Stopping checkpoint coordinator jor job " + job);
+ LOG.info("Stopping checkpoint coordinator for job " + job);
// shut down the thread that handles the timeouts
timer.cancel();
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index 2bee094..c844d8e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -507,7 +507,7 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
// Collect the accumulators of all involved UDFs and send them to the
// JobManager. close() has been called earlier for all involved UDFs
// (using this.stub.close() and closeChainedTasks()), so UDFs can no longer
- // modify accumulators.ll;
+ // modify accumulators;
if (this.stub != null) {
// collect the counters from the stub
if (FunctionUtils.getFunctionRuntimeContext(this.stub, this.runtimeUdfContext) != null) {
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorState.java
deleted file mode 100644
index 8dfd715..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/OperatorState.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.state;
-
-import java.io.Serializable;
-
-/**
- * Abstract class for representing operator states in Flink programs. By
- * implementing the methods declared in this abstraction the state of the
- * operator can be checkpointed by the fault tolerance mechanism.
- *
- * @param <T>
- * The type of the operator state.
- */
-public class OperatorState<T> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private T stateObject;
-
- /**
- * Initializes the state using the given state object.
- *
- * @param initialState
- * The initial state object
- */
- public OperatorState(T initialState) {
- stateObject = initialState;
- }
-
- /**
- * Returns the currently stored state object.
- *
- * @return The state.
- */
- public T getState() {
- return stateObject;
- }
-
- /**
- * Updates the current state object. States should be only updated using
- * this method to avoid concurrency issues.
- *
- * @param stateUpdate
- * The update applied.
- */
- @SuppressWarnings("unchecked")
- public synchronized void update(Object stateUpdate) {
- this.stateObject = (T) stateUpdate;
- }
-
- /**
- * Creates a {@link StateCheckpoint} that will be used to backup the state
- * for failure recovery. This method will be called by the state
- * checkpointer.
- *
- * @return The {@link StateCheckpoint} created.
- */
- public synchronized StateCheckpoint<T> checkpoint() {
- return new StateCheckpoint<T>(this);
- }
-
- @Override
- public String toString() {
- return stateObject.toString();
- }
-
- public boolean stateEquals(OperatorState<T> other) {
- return stateObject.equals(other.stateObject);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateCheckpoint.java
deleted file mode 100644
index a872e5a..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateCheckpoint.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.state;
-
-import java.io.Serializable;
-
-/**
- * Base class for creating checkpoints for {@link OperatorState}. This
- * checkpoints will be used to backup states in stateful Flink operators and
- * also to restore them in case of node failure. To allow incremental
- * checkpoints override the {@link #update(StateCheckpoint)} method.
- *
- * @param <T>
- * The type of the state.
- */
-public class StateCheckpoint<T> implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- public OperatorState<T> checkpointedState;
-
- /**
- * Creates a state checkpoint from the given {@link OperatorState}
- *
- * @param operatorState
- * The {@link OperatorState} to checkpoint.
- */
- public StateCheckpoint(OperatorState<T> operatorState) {
- this.checkpointedState = operatorState;
- }
-
- public OperatorState<T> restore() {
- return checkpointedState;
- }
-
- @Override
- public String toString() {
- return checkpointedState.toString();
- }
-
- public boolean stateEquals(StateCheckpoint<T> other) {
- return checkpointedState.equals(other.checkpointedState);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
index caabb21..fde4cdf 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/ConnectorSource.java
@@ -19,13 +19,10 @@ package org.apache.flink.streaming.connectors;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
-import org.apache.flink.api.java.typeutils.TypeExtractor;
-import org.apache.flink.streaming.api.functions.source.GenericSourceFunction;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements
- GenericSourceFunction<OUT> {
+public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OUT> implements ResultTypeQueryable<OUT>{
private static final long serialVersionUID = 1L;
protected DeserializationSchema<OUT> schema;
@@ -35,12 +32,7 @@ public abstract class ConnectorSource<OUT> extends RichParallelSourceFunction<OU
}
@Override
- public TypeInformation<OUT> getType() {
- if(schema instanceof ResultTypeQueryable) {
- return ((ResultTypeQueryable<OUT>) schema).getProducedType();
- }
- return TypeExtractor.createTypeInfo(DeserializationSchema.class, schema.getClass(), 0,
- null, null);
+ public TypeInformation<OUT> getProducedType() {
+ return schema.getProducedType();
}
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
index 1bf2962..fe6684d 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaConsumerExample.java
@@ -38,7 +38,6 @@ public class KafkaConsumerExample {
DataStream<String> kafkaStream = env
.addSource(new KafkaSource<String>(host + ":" + port, topic, new JavaDefaultStringSchema()));
-
kafkaStream.print();
env.execute();
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSimpleConsumerExample.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSimpleConsumerExample.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSimpleConsumerExample.java
deleted file mode 100644
index d43460b..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/KafkaSimpleConsumerExample.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.kafka.api.simple.PersistentKafkaSource;
-import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
-
-public class KafkaSimpleConsumerExample {
-
- private static String host;
- private static int port;
- private static String topic;
-
- public static void main(String[] args) throws Exception {
-
- if (!parseParameters(args)) {
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment().setParallelism(4);
- DataStream<String> kafkaStream = env
- .addSource(new PersistentKafkaSource<String>(host + ":" + port, topic, new JavaDefaultStringSchema()));
-
- kafkaStream.print();
-
- env.execute();
- }
-
- private static boolean parseParameters(String[] args) {
- if (args.length == 3) {
- host = args[0];
- port = Integer.parseInt(args[1]);
- topic = args[2];
- return true;
- } else {
- System.err.println("Usage: KafkaConsumerExample <host> <port> <topic>");
- return false;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
index a29ba4d..4286196 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/Utils.java
@@ -19,7 +19,6 @@ package org.apache.flink.streaming.connectors.kafka;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.api.java.typeutils.runtime.ByteArrayInputView;
import org.apache.flink.runtime.util.DataOutputSerializer;
@@ -30,7 +29,7 @@ import java.io.IOException;
public class Utils {
public static class TypeInformationSerializationSchema<T>
- implements DeserializationSchema<T>, SerializationSchema<T, byte[]>, ResultTypeQueryable<T> {
+ implements DeserializationSchema<T>, SerializationSchema<T, byte[]> {
private final TypeSerializer<T> serializer;
private final TypeInformation<T> ti;
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
index 85c4d35..0965b29 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSink.java
@@ -24,7 +24,6 @@ import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.connectors.kafka.api.config.PartitionerWrapper;
-import org.apache.flink.streaming.connectors.kafka.api.simple.KafkaTopicUtils;
import org.apache.flink.streaming.connectors.kafka.partitioner.SerializableKafkaPartitioner;
import org.apache.flink.streaming.util.serialization.SerializationSchema;
import org.apache.flink.util.NetUtils;
@@ -38,6 +37,7 @@ import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import kafka.serializer.DefaultEncoder;
+
/**
* Sink that emits its inputs to a Kafka topic.
*
@@ -53,7 +53,7 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
private Producer<IN, byte[]> producer;
private Properties userDefinedProperties;
private String topicId;
- private String zookeeperAddress;
+ private String brokerList;
private SerializationSchema<IN, byte[]> schema;
private SerializableKafkaPartitioner partitioner;
private Class<? extends SerializableKafkaPartitioner> partitionerClass = null;
@@ -62,16 +62,16 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
* Creates a KafkaSink for a given topic. The sink produces its input to
* the topic.
*
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
+ * @param brokerList
+ * Addresses of the brokers
* @param topicId
* ID of the Kafka topic.
* @param serializationSchema
* User defined serialization schema.
*/
- public KafkaSink(String zookeeperAddress, String topicId,
+ public KafkaSink(String brokerList, String topicId,
SerializationSchema<IN, byte[]> serializationSchema) {
- this(zookeeperAddress, topicId, new Properties(), serializationSchema);
+ this(brokerList, topicId, new Properties(), serializationSchema);
}
/**
@@ -79,8 +79,8 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
* If you use this constructor, the broker should be set with the "metadata.broker.list"
* configuration.
*
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
+ * @param brokerList
+ * Addresses of the brokers
* @param topicId
* ID of the Kafka topic.
* @param producerConfig
@@ -88,13 +88,15 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
* @param serializationSchema
* User defined serialization schema.
*/
- public KafkaSink(String zookeeperAddress, String topicId, Properties producerConfig,
+ public KafkaSink(String brokerList, String topicId, Properties producerConfig,
SerializationSchema<IN, byte[]> serializationSchema) {
- NetUtils.ensureCorrectHostnamePort(zookeeperAddress);
+ String[] elements = brokerList.split(",");
+ for(String broker: elements) {
+ NetUtils.ensureCorrectHostnamePort(broker);
+ }
Preconditions.checkNotNull(topicId, "TopicID not set");
- ClosureCleaner.ensureSerializable(partitioner);
- this.zookeeperAddress = zookeeperAddress;
+ this.brokerList = brokerList;
this.topicId = topicId;
this.schema = serializationSchema;
this.partitionerClass = null;
@@ -105,8 +107,7 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
* Creates a KafkaSink for a given topic. The sink produces its input to
* the topic.
*
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
+ * @param brokerList
* @param topicId
* ID of the Kafka topic.
* @param serializationSchema
@@ -114,15 +115,18 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
* @param partitioner
* User defined partitioner.
*/
- public KafkaSink(String zookeeperAddress, String topicId,
+ public KafkaSink(String brokerList, String topicId,
SerializationSchema<IN, byte[]> serializationSchema, SerializableKafkaPartitioner partitioner) {
- this(zookeeperAddress, topicId, serializationSchema);
+ this(brokerList, topicId, serializationSchema);
+ ClosureCleaner.ensureSerializable(partitioner);
this.partitioner = partitioner;
}
- public KafkaSink(String zookeeperAddress, String topicId,
- SerializationSchema<IN, byte[]> serializationSchema, Class<? extends SerializableKafkaPartitioner> partitioner) {
- this(zookeeperAddress, topicId, serializationSchema);
+ public KafkaSink(String brokerList,
+ String topicId,
+ SerializationSchema<IN, byte[]> serializationSchema,
+ Class<? extends SerializableKafkaPartitioner> partitioner) {
+ this(brokerList, topicId, serializationSchema);
this.partitionerClass = partitioner;
}
@@ -132,16 +136,9 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
@Override
public void open(Configuration configuration) {
- KafkaTopicUtils kafkaTopicUtils = new KafkaTopicUtils(zookeeperAddress);
- String listOfBrokers = kafkaTopicUtils.getBrokerList(topicId);
-
- if (LOG.isInfoEnabled()) {
- LOG.info("Broker list: {}", listOfBrokers);
- }
-
Properties properties = new Properties();
- properties.put("metadata.broker.list", listOfBrokers);
+ properties.put("metadata.broker.list", brokerList);
properties.put("request.required.acks", "-1");
properties.put("message.send.max.retries", "10");
@@ -168,7 +165,7 @@ public class KafkaSink<IN> extends RichSinkFunction<IN> {
try {
producer = new Producer<IN, byte[]>(config);
} catch (NullPointerException e) {
- throw new RuntimeException("Cannot connect to Kafka broker " + listOfBrokers, e);
+ throw new RuntimeException("Cannot connect to Kafka broker " + brokerList, e);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
index a0805c0..28e338c 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/KafkaSource.java
@@ -47,7 +47,6 @@ public class KafkaSource<OUT> extends ConnectorSource<OUT> {
private static final long serialVersionUID = 1L;
- @SuppressWarnings("unused")
private static final Logger LOG = LoggerFactory.getLogger(KafkaSource.class);
private final String zookeeperAddress;
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
new file mode 100644
index 0000000..50ee15a
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/persistent/PersistentKafkaSource.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.connectors.kafka.api.persistent;
+
+import com.google.common.base.Preconditions;
+import kafka.common.TopicAndPartition;
+import kafka.consumer.Consumer;
+import kafka.consumer.ConsumerConfig;
+import kafka.consumer.ConsumerIterator;
+import kafka.consumer.KafkaStream;
+import kafka.javaapi.consumer.ConsumerConnector;
+import kafka.message.MessageAndMetadata;
+import kafka.utils.ZKGroupTopicDirs;
+import kafka.utils.ZkUtils;
+import org.I0Itec.zkclient.ZkClient;
+import org.I0Itec.zkclient.exception.ZkMarshallingError;
+import org.I0Itec.zkclient.serialize.ZkSerializer;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.checkpoint.CheckpointCommitter;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.util.serialization.DeserializationSchema;
+import org.apache.flink.util.Collector;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Option;
+import scala.collection.JavaConversions;
+import scala.collection.Seq;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Source for reading from Kafka using Flink Streaming Fault Tolerance.
+ * This source is updating the committed offset in Zookeeper based on the internal checkpointing of Flink.
+ *
+ * Note that the autocommit feature of Kafka needs to be disabled for using this source.
+ */
+public class PersistentKafkaSource<OUT> extends RichParallelSourceFunction<OUT> implements
+ ResultTypeQueryable<OUT>,
+ CheckpointCommitter,
+ CheckpointedAsynchronously<long[]> {
+ private static final Logger LOG = LoggerFactory.getLogger(PersistentKafkaSource.class);
+
+ protected transient ConsumerConfig consumerConfig;
+ private transient ConsumerIterator<byte[], byte[]> iteratorToRead;
+ private transient ConsumerConnector consumer;
+
+ private String topicName;
+ private DeserializationSchema<OUT> deserializationSchema;
+ private boolean running = true;
+
+ private transient long[] lastOffsets;
+ private transient ZkClient zkClient;
+ private transient long[] commitedOffsets; // maintain committed offsets, to avoid committing the same over and over again.
+
+
+ /**
+ *
+ * For the @param consumerConfig, specify at least the "groupid" and "zookeeper.connect" string.
+ * The config will be passed into the Kafka High Level Consumer.
+ * For a full list of possible values, check this out: https://kafka.apache.org/documentation.html#consumerconfigs
+ */
+ public PersistentKafkaSource(String topicName, DeserializationSchema<OUT> deserializationSchema, ConsumerConfig consumerConfig) {
+ Preconditions.checkNotNull(topicName);
+ Preconditions.checkNotNull(deserializationSchema);
+ Preconditions.checkNotNull(consumerConfig);
+
+ this.topicName = topicName;
+ this.deserializationSchema = deserializationSchema;
+ this.consumerConfig = consumerConfig;
+ if(consumerConfig.autoCommitEnable()) {
+ throw new IllegalArgumentException("'auto.commit.enable' is set to 'true'. " +
+ "This source can only be used with auto commit disabled because the " +
+ "source is committing to zookeeper by itself (not using the KafkaConsumer).");
+ }
+ if(!consumerConfig.offsetsStorage().equals("zookeeper")) {
+ // we can currently only commit to ZK.
+ throw new IllegalArgumentException("The 'offsets.storage' has to be set to 'zookeeper' for this Source to work reliably");
+ }
+ }
+
+ // ---------------------- ParallelSourceFunction Lifecycle -----------------
+
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ ConsumerConnector consumer = Consumer.createJavaConsumerConnector(this.consumerConfig);
+ // we request only one stream per consumer instance. Kafka will make sure that each consumer group
+ // will see each message only once.
+ Map<String,Integer> topicCountMap = Collections.singletonMap(topicName, 1);
+ Map<String, List<KafkaStream<byte[], byte[]>>> streams = consumer.createMessageStreams(topicCountMap);
+ if(streams.size() != 1) {
+ throw new RuntimeException("Expected only one message stream but got "+streams.size());
+ }
+ List<KafkaStream<byte[], byte[]>> kafkaStreams = streams.get(topicName);
+ if(kafkaStreams == null) {
+ throw new RuntimeException("Requested stream not available. Available streams: "+streams.toString());
+ }
+ if(kafkaStreams.size() != 1) {
+ throw new RuntimeException("Requested 1 stream from Kafka, bot got "+kafkaStreams.size()+" streams");
+ }
+ LOG.info("Opening Consumer instance for topic '{}' on group '{}'", topicName, consumerConfig.groupId());
+ this.iteratorToRead = kafkaStreams.get(0).iterator();
+ this.consumer = consumer;
+
+ zkClient = new ZkClient(consumerConfig.zkConnect(),
+ consumerConfig.zkSessionTimeoutMs(),
+ consumerConfig.zkConnectionTimeoutMs(),
+ new KafkaZKStringSerializer());
+
+ // most likely the number of offsets we're going to store here will be lower than the number of partitions.
+ int numPartitions = getNumberOfPartitions();
+ LOG.debug("The topic {} has {} partitions", topicName, numPartitions);
+ this.lastOffsets = new long[numPartitions];
+ this.commitedOffsets = new long[numPartitions];
+ Arrays.fill(this.lastOffsets, -1);
+ Arrays.fill(this.commitedOffsets, 0); // just to make it clear
+ }
+
+
+ @Override
+ public void run(Collector<OUT> collector) throws Exception {
+ if(iteratorToRead == null) {
+ throw new RuntimeException("Stream to read not initialized properly. Has open() been called");
+ }
+ try {
+ while (iteratorToRead.hasNext()) {
+ if (!running) {
+ LOG.info("Source got stopped");
+ break;
+ }
+ MessageAndMetadata<byte[], byte[]> message = iteratorToRead.next();
+ if(lastOffsets[message.partition()] >= message.offset()) {
+ LOG.info("Skipping message with offset {} from partition {}", message.offset(), message.partition());
+ continue;
+ }
+ lastOffsets[message.partition()] = message.offset();
+
+ OUT out = deserializationSchema.deserialize(message.message());
+ if (deserializationSchema.isEndOfStream(out)) {
+ LOG.info("DeserializationSchema signaled end of stream for this source");
+ break;
+ }
+
+ collector.collect(out);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Processed record with offset {} from partition {}", message.offset(), message.partition());
+ }
+ }
+ } catch(Exception ie) {
+ // this exception is coming from Scala code.
+ if(ie instanceof InterruptedException) {
+ if(running) {
+ throw new RuntimeException("Error while reading kafka consumer", ie);
+ } else {
+ LOG.debug("Kafka consumer got interrupted because it has been cancelled. This is expected", ie);
+ }
+ } else {
+ throw ie;
+ }
+ }
+
+ LOG.info("Source has finished reading data from the KafkaStream");
+ }
+
+ @Override
+ public void cancel() {
+ LOG.info("Instructing source to stop reading data from Kafka");
+ running = false;
+ }
+
+ @Override
+ public void close() {
+ LOG.info("Closing Kafka consumer");
+ this.consumer.shutdown();
+ zkClient.close();
+ }
+
+
+ // ---------------------- State / Checkpoint handling -----------------
+ // this source is keeping the partition offsets in Zookeeper
+
+ private Map<Long, long[]> pendingCheckpoints = new HashMap<Long, long[]>();
+
+ @Override
+ public long[] snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+ if(lastOffsets == null) {
+ LOG.warn("State snapshot requested on not yet opened source. Returning null");
+ return null;
+ }
+ LOG.info("Snapshotting state. Offsets: {}, checkpoint id {}, timestamp {}", Arrays.toString(lastOffsets), checkpointId, checkpointTimestamp);
+
+ long[] currentOffsets = Arrays.copyOf(lastOffsets, lastOffsets.length);
+ pendingCheckpoints.put(checkpointId, currentOffsets);
+ return currentOffsets;
+ }
+
+ @Override
+ public void restoreState(long[] state) {
+ // we maintain the offsets in Kafka, so nothing to do.
+ }
+
+
+ /**
+ * Notification on completed checkpoints
+ * @param checkpointId The ID of the checkpoint that has been completed.
+ */
+ @Override
+ public void commitCheckpoint(long checkpointId) {
+ LOG.info("Commit checkpoint {}", checkpointId);
+ long[] checkpointOffsets = pendingCheckpoints.remove(checkpointId);
+ if(checkpointOffsets == null) {
+ LOG.warn("Unable to find pending checkpoint for id {}", checkpointId);
+ return;
+ }
+ LOG.info("Got corresponding offsets {}", Arrays.toString(checkpointOffsets));
+
+ for(int partition = 0; partition < checkpointOffsets.length; partition++) {
+ long offset = checkpointOffsets[partition];
+ if(offset != -1) {
+ setOffset(partition, offset);
+ }
+ }
+ }
+
+ // --------------------- Zookeeper / Offset handling -----------------------------
+
+ private int getNumberOfPartitions() {
+ scala.collection.immutable.List<String> scalaSeq = JavaConversions.asScalaBuffer(Collections.singletonList(topicName)).toList();
+ scala.collection.mutable.Map<String, Seq<Object>> list = ZkUtils.getPartitionsForTopics(zkClient, scalaSeq);
+ Option<Seq<Object>> topicOption = list.get(topicName);
+ if(topicOption.isEmpty()) {
+ throw new IllegalArgumentException("Unable to get number of partitions for topic "+topicName+" from "+list.toString());
+ }
+ Seq<Object> topic = topicOption.get();
+ return topic.size();
+ }
+
+ protected void setOffset(int partition, long offset) {
+ if(commitedOffsets[partition] < offset) {
+ setOffset(zkClient, consumerConfig.groupId(), topicName, partition, offset);
+ commitedOffsets[partition] = offset;
+ } else {
+ LOG.debug("Ignoring offset {} for partition {} because it is already committed", offset, partition);
+ }
+ }
+
+
+
+ // the following two methods are static to allow access from the outside as well (Testcases)
+
+ /**
+ * This method's code is based on ZookeeperConsumerConnector.commitOffsetToZooKeeper()
+ */
+ public static void setOffset(ZkClient zkClient, String groupId, String topic, int partition, long offset) {
+ LOG.info("Setting offset for partition {} of topic {} in group {} to offset {}", partition, topic, groupId, offset);
+ TopicAndPartition tap = new TopicAndPartition(topic, partition);
+ ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, tap.topic());
+ ZkUtils.updatePersistentPath(zkClient, topicDirs.consumerOffsetDir() + "/" + tap.partition(), Long.toString(offset));
+ }
+
+ public static long getOffset(ZkClient zkClient, String groupId, String topic, int partition) {
+ TopicAndPartition tap = new TopicAndPartition(topic, partition);
+ ZKGroupTopicDirs topicDirs = new ZKGroupTopicDirs(groupId, tap.topic());
+ scala.Tuple2<String, Stat> data = ZkUtils.readData(zkClient, topicDirs.consumerOffsetDir() + "/" + tap.partition());
+ return Long.valueOf(data._1());
+ }
+
+
+ // ---------------------- (Java)Serialization methods for the consumerConfig -----------------
+
+ private void writeObject(ObjectOutputStream out)
+ throws IOException, ClassNotFoundException {
+ out.defaultWriteObject();
+ out.writeObject(consumerConfig.props().props());
+ }
+
+ private void readObject(ObjectInputStream in)
+ throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ Properties props = (Properties) in.readObject();
+ consumerConfig = new ConsumerConfig(props);
+ }
+
+
+ @Override
+ public TypeInformation<OUT> getProducedType() {
+ return deserializationSchema.getProducedType();
+ }
+
+
+ // ---------------------- Zookeeper Serializer copied from Kafka (because it has private access there) -----------------
+
+ public static class KafkaZKStringSerializer implements ZkSerializer {
+
+ @Override
+ public byte[] serialize(Object data) throws ZkMarshallingError {
+ try {
+ return ((String) data).getBytes("UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Object deserialize(byte[] bytes) throws ZkMarshallingError {
+ if (bytes == null) {
+ return null;
+ } else {
+ try {
+ return new String(bytes, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/KafkaTopicUtils.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/KafkaTopicUtils.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/KafkaTopicUtils.java
deleted file mode 100644
index 365961d..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/KafkaTopicUtils.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple;
-
-import java.io.UnsupportedEncodingException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Properties;
-import java.util.Set;
-
-import org.I0Itec.zkclient.ZkClient;
-import org.I0Itec.zkclient.exception.ZkMarshallingError;
-import org.I0Itec.zkclient.serialize.ZkSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.admin.AdminUtils;
-import kafka.api.PartitionMetadata;
-import kafka.api.TopicMetadata;
-import kafka.cluster.Broker;
-import kafka.common.LeaderNotAvailableException;
-import kafka.common.UnknownTopicOrPartitionException;
-import scala.collection.JavaConversions;
-import scala.collection.Seq;
-
-/**
- * For retrieving Kafka topic information (e.g. number of partitions),
- * or creating a topic.
- */
-public class KafkaTopicUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaTopicUtils.class);
-
- private ZkClient zkClient;
-
- public static final int DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MS = 10000;
- public static final int DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS = 10000;
-
- private final String zookeeperAddress;
- private final int sessionTimeoutMs;
- private final int connectionTimeoutMs;
-
- private volatile boolean isRunning = false;
-
- public KafkaTopicUtils(String zookeeperServer) {
- this(zookeeperServer, DEFAULT_ZOOKEEPER_SESSION_TIMEOUT_MS, DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS);
- }
-
- public KafkaTopicUtils(String zookeeperAddress, int sessionTimeoutMs, int connectionTimeoutMs) {
- this.zookeeperAddress = zookeeperAddress;
- this.sessionTimeoutMs = sessionTimeoutMs;
- this.connectionTimeoutMs = connectionTimeoutMs;
- }
-
- public void createTopic(String topicName, int numOfPartitions, int replicationFactor) {
-
- LOG.info("Creating Kafka topic '{}'", topicName);
- Properties topicConfig = new Properties();
- if (topicExists(topicName)) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Kafka topic \"{}\" already exists. Returning without action.", topicName);
- }
- } else {
- LOG.info("Connecting zookeeper");
-
- initZkClient();
- AdminUtils.createTopic(zkClient, topicName, numOfPartitions, replicationFactor, topicConfig);
- closeZkClient();
- }
- }
-
- public String getBrokerList(String topicName) {
- return getBrokerAddressList(getBrokerAddresses(topicName));
- }
-
- public String getBrokerList(String topicName, int partitionId) {
- return getBrokerAddressList(getBrokerAddresses(topicName, partitionId));
- }
-
- public Set<String> getBrokerAddresses(String topicName) {
- int numOfPartitions = getNumberOfPartitions(topicName);
-
- HashSet<String> brokers = new HashSet<String>();
- for (int i = 0; i < numOfPartitions; i++) {
- brokers.addAll(getBrokerAddresses(topicName, i));
- }
- return brokers;
- }
-
- public Set<String> getBrokerAddresses(String topicName, int partitionId) {
- PartitionMetadata partitionMetadata = waitAndGetPartitionMetadata(topicName, partitionId);
- Collection<Broker> inSyncReplicas = JavaConversions.asJavaCollection(partitionMetadata.isr());
-
- HashSet<String> addresses = new HashSet<String>();
- for (Broker broker : inSyncReplicas) {
- addresses.add(broker.connectionString());
- }
- return addresses;
- }
-
- private static String getBrokerAddressList(Set<String> brokerAddresses) {
- StringBuilder brokerAddressList = new StringBuilder("");
- for (String broker : brokerAddresses) {
- brokerAddressList.append(broker);
- brokerAddressList.append(',');
- }
- brokerAddressList.deleteCharAt(brokerAddressList.length() - 1);
-
- return brokerAddressList.toString();
- }
-
- public int getNumberOfPartitions(String topicName) {
- Seq<PartitionMetadata> partitionMetadataSeq = getTopicMetadata(topicName).partitionsMetadata();
- return JavaConversions.asJavaCollection(partitionMetadataSeq).size();
- }
-
- public PartitionMetadata waitAndGetPartitionMetadata(String topicName, int partitionId) {
- isRunning = true;
- PartitionMetadata partitionMetadata = null;
- while (isRunning) {
- try {
- partitionMetadata = getPartitionMetadata(topicName, partitionId);
- return partitionMetadata;
- } catch (LeaderNotAvailableException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Got {} trying to fetch metadata again", e.getMessage());
- }
- }
- }
- isRunning = false;
- return partitionMetadata;
- }
-
- public PartitionMetadata getPartitionMetadata(String topicName, int partitionId) {
- PartitionMetadata partitionMetadata = getPartitionMetadataWithErrorCode(topicName, partitionId);
- switch (partitionMetadata.errorCode()) {
- case 0:
- return partitionMetadata;
- case 3:
- throw new UnknownTopicOrPartitionException("While fetching metadata for " + topicName + " / " + partitionId);
- case 5:
- throw new LeaderNotAvailableException("While fetching metadata for " + topicName + " / " + partitionId);
- default:
- throw new RuntimeException("Unknown error occurred while fetching metadata for "
- + topicName + " / " + partitionId + ", with error code: " + partitionMetadata.errorCode());
- }
- }
-
- private PartitionMetadata getPartitionMetadataWithErrorCode(String topicName, int partitionId) {
- TopicMetadata topicInfo = getTopicMetadata(topicName);
-
- Collection<PartitionMetadata> partitions = JavaConversions.asJavaCollection(topicInfo.partitionsMetadata());
-
- Iterator<PartitionMetadata> iterator = partitions.iterator();
- for (PartitionMetadata partition : partitions) {
- if (partition.partitionId() == partitionId) {
- return partition;
- }
- }
-
- throw new RuntimeException("No such partition: " + topicName + " / " + partitionId);
- }
-
- public TopicMetadata getTopicMetadata(String topicName) {
- TopicMetadata topicMetadata = getTopicMetadataWithErrorCode(topicName);
- switch (topicMetadata.errorCode()) {
- case 0:
- return topicMetadata;
- case 3:
- throw new UnknownTopicOrPartitionException("While fetching metadata for topic " + topicName);
- case 5:
- throw new LeaderNotAvailableException("While fetching metadata for topic " + topicName);
- default:
- throw new RuntimeException("Unknown error occurred while fetching metadata for topic "
- + topicName + ", with error code: " + topicMetadata.errorCode());
- }
- }
-
- private TopicMetadata getTopicMetadataWithErrorCode(String topicName) {
- if (topicExists(topicName)) {
- initZkClient();
- TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topicName, zkClient);
- closeZkClient();
-
- return topicMetadata;
- } else {
- throw new RuntimeException("Topic does not exist: " + topicName);
- }
- }
-
- public boolean topicExists(String topicName) {
- initZkClient();
- boolean topicExists = AdminUtils.topicExists(zkClient, topicName);
- closeZkClient();
-
- return topicExists;
- }
-
- private void initZkClient() {
- zkClient = new ZkClient(zookeeperAddress, sessionTimeoutMs, connectionTimeoutMs,
- new KafkaZKStringSerializer());
- zkClient.waitUntilConnected();
- }
-
- private void closeZkClient() {
- zkClient.close();
- zkClient = null;
- }
-
- private static class KafkaZKStringSerializer implements ZkSerializer {
-
- @Override
- public byte[] serialize(Object data) throws ZkMarshallingError {
- try {
- return ((String) data).getBytes("UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Object deserialize(byte[] bytes) throws ZkMarshallingError {
- if (bytes == null) {
- return null;
- } else {
- try {
- return new String(bytes, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/MessageWithMetadata.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/MessageWithMetadata.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/MessageWithMetadata.java
deleted file mode 100644
index 3985350..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/MessageWithMetadata.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple;
-
-/**
- * POJO encapsulating records received from Kafka with their offset and partition.
- */
-public class MessageWithMetadata {
-
- private int partition;
- private long offset;
- private byte[] message;
-
- public MessageWithMetadata(long offset, byte[] message, int partition) {
- this.partition = partition;
- this.offset = offset;
- this.message = message;
- }
-
- public long getOffset() {
- return offset;
- }
-
- public byte[] getMessage() {
- return message;
- }
-
- public int getPartition() {
- return partition;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java
deleted file mode 100644
index a842160..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/PersistentKafkaSource.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple;
-
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import com.google.common.base.Preconditions;
-import kafka.consumer.ConsumerConfig;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.state.OperatorState;
-import org.apache.flink.streaming.api.checkpoint.Checkpointed;
-import org.apache.flink.streaming.runtime.tasks.StreamingRuntimeContext;
-import org.apache.flink.streaming.connectors.ConnectorSource;
-import org.apache.flink.streaming.connectors.kafka.api.simple.iterator.KafkaConsumerIterator;
-import org.apache.flink.streaming.connectors.kafka.api.simple.iterator.KafkaIdleConsumerIterator;
-import org.apache.flink.streaming.connectors.kafka.api.simple.iterator.KafkaMultiplePartitionsIterator;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.BeginningOffset;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.CurrentOffset;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.GivenOffset;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.KafkaOffset;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.Offset;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Kafka source persisting its offset through the {@link OperatorState} interface.
- * This allows the offset to be restored to the latest one that has been acknowledged
- * by the whole execution graph.
- *
- * @param <OUT>
- * Type of the messages on the topic.
- */
-public class PersistentKafkaSource<OUT> extends ConnectorSource<OUT> implements Checkpointed<HashMap<Integer, KafkaOffset>> {
-
- private static final long serialVersionUID = 1L;
- private static final Logger LOG = LoggerFactory.getLogger(PersistentKafkaSource.class);
-
- public static final String WAIT_ON_EMPTY_FETCH_KEY = "flink.waitOnEmptyFetchMillis";
- public static final String WAIT_ON_FAILED_LEADER_MS_KEY = "flink.waitOnFailedLeaderDetection";
- public static final int WAIT_ON_FAILED_LEADER__MS_DEFAULT = 2000;
-
- public static final String MAX_FAILED_LEADER_RETRIES_KEY = "flink.maxLeaderDetectionRetries";
- public static final int MAX_FAILED_LEADER_RETRIES_DEFAULT = 3;
-
- private final String topicId;
- private final KafkaOffset startingOffset;
- private transient ConsumerConfig consumerConfig; // ConsumerConfig is not serializable.
-
- private transient KafkaConsumerIterator iterator;
- private transient OperatorState<Map<Integer, KafkaOffset>> kafkaOffSetOperatorState;
-
- private transient Map<Integer, KafkaOffset> partitionOffsets;
-
- /**
- * Creates a persistent Kafka source that consumes a topic.
- * If there is are no new messages on the topic, this consumer will wait
- * 100 milliseconds before trying to fetch messages again.
- * The consumer will start consuming from the latest messages in the topic.
- *
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
- * @param topicId
- * ID of the Kafka topic.
- * @param deserializationSchema
- * User defined deserialization schema.
- */
- public PersistentKafkaSource(String zookeeperAddress, String topicId,
- DeserializationSchema<OUT> deserializationSchema) {
- this(zookeeperAddress, topicId, deserializationSchema, KafkaTopicUtils.DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS, 100);
- }
-
- /**
- * Creates a persistent Kafka source that consumes a topic.
- * If there is are no new messages on the topic, this consumer will wait
- * waitOnEmptyFetchMillis milliseconds before trying to fetch messages again.
- * The consumer will start consuming from the latest messages in the topic.
- *
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
- * @param topicId
- * ID of the Kafka topic.
- * @param deserializationSchema
- * User defined deserialization schema.
- * @param zookeeperSyncTimeMillis
- * Synchronization time with zookeeper.
- * @param waitOnEmptyFetchMillis
- * Time to wait before fetching for new message.
- */
- public PersistentKafkaSource(String zookeeperAddress, String topicId,
- DeserializationSchema<OUT> deserializationSchema, int zookeeperSyncTimeMillis, int waitOnEmptyFetchMillis) {
- this(zookeeperAddress, topicId, deserializationSchema, zookeeperSyncTimeMillis, waitOnEmptyFetchMillis, Offset.FROM_CURRENT);
- }
-
-
- /**
- * Creates a persistent Kafka source that consumes a topic.
- * If there is are no new messages on the topic, this consumer will wait
- * waitOnEmptyFetchMillis milliseconds before trying to fetch messages again.
- *
- * THIS CONSTRUCTOR IS DEPRECATED: USE the constructor with the ConsumerConfig.
- *
- * @param zookeeperAddress
- * Address of the Zookeeper host (with port number).
- * @param topicId
- * ID of the Kafka topic.
- * @param deserializationSchema
- * User defined deserialization schema.
- * @param zookeeperSyncTimeMillis
- * Synchronization time with zookeeper.
- * @param waitOnEmptyFetchMillis
- * Time to wait before fetching for new message.
- * @param startOffsetType
- * The offset to start from (beginning or current).
- */
- @Deprecated
- public PersistentKafkaSource(String zookeeperAddress, String topicId,DeserializationSchema<OUT> deserializationSchema, int zookeeperSyncTimeMillis, int waitOnEmptyFetchMillis, Offset startOffsetType) {
- this(topicId, deserializationSchema, startOffsetType, legacyParametersToConsumerConfig(zookeeperAddress, zookeeperSyncTimeMillis, waitOnEmptyFetchMillis));
- }
-
- private static ConsumerConfig legacyParametersToConsumerConfig(String zookeeperAddress, int zookeeperSyncTimeMillis, int waitOnEmptyFetchMillis) {
- Properties props = new Properties();
- props.setProperty("zookeeper.sync.time.ms", Integer.toString(zookeeperSyncTimeMillis));
- props.setProperty(WAIT_ON_EMPTY_FETCH_KEY, Integer.toString(waitOnEmptyFetchMillis));
- props.setProperty("zookeeper.connect", zookeeperAddress);
- props.setProperty("group.id", "flink-persistent-kafka-source");
- return new ConsumerConfig(props);
- }
-
- /**
- * Creates a persistent Kafka source that consumes a topic.
- * If there is are no new messages on the topic, this consumer will wait
- * waitOnEmptyFetchMillis milliseconds before trying to fetch messages again.
- *
- * @param topicId
- * ID of the Kafka topic.
- * @param deserializationSchema
- * User defined deserialization schema.
- * @param startOffsetType
- * The offset to start from (beginning or current).
- * @param consumerConfig
- * Additional configuration for the PersistentKafkaSource.
- * NOTE: This source will only respect certain configuration values from the config!
- */
- public PersistentKafkaSource(String topicId, DeserializationSchema<OUT> deserializationSchema, Offset startOffsetType, ConsumerConfig consumerConfig) {
- super(deserializationSchema);
- Preconditions.checkNotNull(topicId, "The topic id can not be null");
- Preconditions.checkNotNull(deserializationSchema, "The deserialization schema can not be null");
- Preconditions.checkNotNull(consumerConfig, "ConsumerConfig can not be null");
-
- this.consumerConfig = consumerConfig;
-
- this.topicId = topicId;
-
- switch (startOffsetType) {
- case FROM_BEGINNING:
- this.startingOffset = new BeginningOffset();
- break;
- case FROM_CURRENT:
- this.startingOffset = new CurrentOffset();
- break;
- default:
- this.startingOffset = new CurrentOffset();
- break;
- }
- }
-
- // ---------------------- Source lifecycle methods (open / run / cancel ) -----------------
-
- @SuppressWarnings("unchecked")
- @Override
- public void open(Configuration parameters) throws InterruptedException {
- LOG.info("Starting PersistentKafkaSource");
- StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
- int indexOfSubtask = context.getIndexOfThisSubtask();
- int numberOfSubtasks = context.getNumberOfParallelSubtasks();
-
- KafkaTopicUtils kafkaTopicUtils = new KafkaTopicUtils(consumerConfig.zkConnect(), consumerConfig.zkSyncTimeMs(), consumerConfig.zkConnectionTimeoutMs());
-
- int numberOfPartitions = kafkaTopicUtils.getNumberOfPartitions(topicId);
-
- if (indexOfSubtask >= numberOfPartitions) {
- LOG.info("Creating idle consumer because this subtask ({}) is higher than the number partitions ({})", indexOfSubtask + 1, numberOfPartitions);
- iterator = new KafkaIdleConsumerIterator();
- }
- else {
- if (partitionOffsets != null) {
- // we have restored state
- LOG.info("Initializing PersistentKafkaSource from existing state.");
- }
- else {
- LOG.info("No existing state found. Creating new");
- partitionOffsets = new HashMap<Integer, KafkaOffset>();
-
- for (int partitionIndex = indexOfSubtask; partitionIndex < numberOfPartitions; partitionIndex += numberOfSubtasks) {
- partitionOffsets.put(partitionIndex, startingOffset);
- }
-
- kafkaOffSetOperatorState = new OperatorState<Map<Integer, KafkaOffset>>(partitionOffsets);
- }
-
- iterator = new KafkaMultiplePartitionsIterator(topicId, partitionOffsets, kafkaTopicUtils, this.consumerConfig);
-
- if (LOG.isInfoEnabled()) {
- LOG.info("PersistentKafkaSource ({}/{}) listening to partitionOffsets {} of topic {}.",
- indexOfSubtask + 1, numberOfSubtasks, partitionOffsets.keySet(), topicId);
- }
- }
-
- iterator.initialize();
- }
-
- @Override
- public void run(Collector<OUT> collector) throws Exception {
- MessageWithMetadata msg;
- while (iterator.hasNext()) {
- msg = iterator.nextWithOffset();
- OUT out = schema.deserialize(msg.getMessage());
-
- if (schema.isEndOfStream(out)) {
- break;
- }
-
- collector.collect(out);
-
- // TODO avoid object creation
- partitionOffsets.put(msg.getPartition(), new GivenOffset(msg.getOffset()));
- kafkaOffSetOperatorState.update(partitionOffsets);
- }
- }
-
- @Override
- public void cancel() {
- LOG.info("PersistentKafkaSource has been cancelled");
- }
-
-
-
- // ---------------------- (Java)Serialization methods for the consumerConfig -----------------
-
- private void writeObject(ObjectOutputStream out)
- throws IOException, ClassNotFoundException {
- out.defaultWriteObject();
- out.writeObject(consumerConfig.props().props());
- }
-
- private void readObject(ObjectInputStream in)
- throws IOException, ClassNotFoundException {
- in.defaultReadObject();
- Properties props = (Properties) in.readObject();
- consumerConfig = new ConsumerConfig(props);
- }
-
- @Override
- public HashMap<Integer, KafkaOffset> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
- return new HashMap<Integer, KafkaOffset>(this.partitionOffsets);
- }
-
- @Override
- public void restoreState(HashMap<Integer, KafkaOffset> state) {
- this.partitionOffsets = state;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaConsumerIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaConsumerIterator.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaConsumerIterator.java
deleted file mode 100644
index 42fe003..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaConsumerIterator.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.iterator;
-
-import org.apache.flink.streaming.connectors.kafka.api.simple.MessageWithMetadata;
-
-/**
- * Iterator interface for different types of Kafka consumers.
- */
-public interface KafkaConsumerIterator {
-
- void initialize() throws InterruptedException;
-
- boolean hasNext();
-
- /**
- * Returns the next message received from Kafka as a
- * byte array.
- *
- * @return next message as a byte array.
- */
- byte[] next() throws InterruptedException;
-
- /**
- * Returns the next message and its offset received from
- * Kafka encapsulated in a POJO.
- *
- * @return next message and its offset.
- */
- MessageWithMetadata nextWithOffset() throws InterruptedException;
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaIdleConsumerIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaIdleConsumerIterator.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaIdleConsumerIterator.java
deleted file mode 100644
index 1935118..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaIdleConsumerIterator.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.iterator;
-
-import org.apache.flink.streaming.connectors.kafka.api.simple.MessageWithMetadata;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * No-op iterator. Used when more source tasks are available than Kafka partitions
- */
-public class KafkaIdleConsumerIterator implements KafkaConsumerIterator {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaIdleConsumerIterator.class);
-
- public KafkaIdleConsumerIterator() {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Idle Kafka consumer created. The subtask does nothing.");
- }
- }
-
-
- @Override
- public void initialize() throws InterruptedException {
-
- }
-
- @Override
- public boolean hasNext() {
- return false;
- }
-
- @Override
- public byte[] next() {
- throw new RuntimeException("Idle consumer has no input.");
- }
-
- @Override
- public MessageWithMetadata nextWithOffset() {
- throw new RuntimeException("Idle consumer has no input.");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaMultiplePartitionsIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaMultiplePartitionsIterator.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaMultiplePartitionsIterator.java
deleted file mode 100644
index daebaaf..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaMultiplePartitionsIterator.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.iterator;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import kafka.consumer.ConsumerConfig;
-import org.apache.flink.streaming.connectors.kafka.api.simple.KafkaTopicUtils;
-import org.apache.flink.streaming.connectors.kafka.api.simple.MessageWithMetadata;
-import org.apache.flink.streaming.connectors.kafka.api.simple.PersistentKafkaSource;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.KafkaOffset;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Iterator over multiple Kafka partitions.
- *
- * This is needed when num partitions > num kafka sources.
- */
-public class KafkaMultiplePartitionsIterator implements KafkaConsumerIterator {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaMultiplePartitionsIterator.class);
-
- protected List<KafkaSinglePartitionIterator> partitions;
- protected final ConsumerConfig consumerConfig;
-
- public KafkaMultiplePartitionsIterator(String topic,
- Map<Integer, KafkaOffset> partitionsWithOffset,
- KafkaTopicUtils kafkaTopicUtils, ConsumerConfig consumerConfig) {
- partitions = new ArrayList<KafkaSinglePartitionIterator>(partitionsWithOffset.size());
-
- this.consumerConfig = consumerConfig;
-
- for (Map.Entry<Integer, KafkaOffset> partitionWithOffset : partitionsWithOffset.entrySet()) {
- partitions.add(new KafkaSinglePartitionIterator(
- topic,
- partitionWithOffset.getKey(),
- partitionWithOffset.getValue(),
- kafkaTopicUtils,
- this.consumerConfig));
- }
- }
-
- @Override
- public void initialize() throws InterruptedException {
- LOG.info("Initializing iterator with {} partitions", partitions.size());
- String partInfo = "";
- for (KafkaSinglePartitionIterator partition : partitions) {
- partition.initialize();
- partInfo += partition.toString() + " ";
- }
- LOG.info("Initialized partitions {}", partInfo);
- }
-
- @Override
- public boolean hasNext() {
- return true;
- }
-
- @Override
- public byte[] next() throws InterruptedException {
- return nextWithOffset().getMessage();
- }
-
- protected int lastCheckedPartitionIndex = -1;
- private boolean gotNewMessage = false;
-
- @Override
- public MessageWithMetadata nextWithOffset() throws InterruptedException {
- KafkaSinglePartitionIterator partition;
-
- while (true) {
- for (int i = nextPartition(lastCheckedPartitionIndex); i < partitions.size(); i = nextPartition(i)) {
- partition = partitions.get(i);
-
- if (partition.fetchHasNext()) {
- gotNewMessage = true;
- lastCheckedPartitionIndex = i;
- return partition.nextWithOffset();
- }
- }
-
- // do not wait if a new message has been fetched
- if (!gotNewMessage) {
- try {
- Thread.sleep(consumerConfig.props().getInt(PersistentKafkaSource.WAIT_ON_EMPTY_FETCH_KEY), consumerConfig.fetchWaitMaxMs());
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting for new messages", e);
- }
- }
-
- gotNewMessage = false;
- }
- }
-
- protected int nextPartition(int currentPartition) {
- return (currentPartition + 1) % partitions.size();
- }
-}
[02/10] flink git commit: [FLINK-1953] [runtime] Integrate new
snapshot checkpoint coordinator with jobgraph and execution graph
Posted by se...@apache.org.
[FLINK-1953] [runtime] Integrate new snapshot checkpoint coordinator with jobgraph and execution graph
This closes #651
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9b7f8aa1
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9b7f8aa1
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9b7f8aa1
Branch: refs/heads/master
Commit: 9b7f8aa121e4a231632296d0809029aca9ebde6a
Parents: ff750e6
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Apr 30 19:59:36 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:57 2015 +0200
----------------------------------------------------------------------
.../checkpoint/CheckpointCoordinator.java | 169 ++++++-
.../CheckpointCoordinatorDeActivator.java | 56 +++
.../runtime/checkpoint/PendingCheckpoint.java | 11 +-
.../flink/runtime/checkpoint/StateForTask.java | 12 +-
.../deployment/TaskDeploymentDescriptor.java | 32 +-
.../flink/runtime/execution/Environment.java | 24 +-
.../flink/runtime/executiongraph/Execution.java | 20 +-
.../runtime/executiongraph/ExecutionGraph.java | 484 +++++++++++--------
.../executiongraph/ExecutionJobVertex.java | 3 +-
.../runtime/executiongraph/ExecutionVertex.java | 25 +-
.../apache/flink/runtime/jobgraph/JobGraph.java | 44 +-
.../jobgraph/tasks/AbstractInvokable.java | 6 +-
.../tasks/CheckpointCommittingOperator.java | 24 +
.../jobgraph/tasks/CheckpointedOperator.java | 24 +
.../jobgraph/tasks/JobSnapshottingSettings.java | 97 ++++
.../jobgraph/tasks/OperatorStateCarrier.java | 16 +-
.../messages/checkpoint/AbortCheckpoint.java | 49 --
.../checkpoint/AcknowledgeCheckpoint.java | 8 +-
.../messages/checkpoint/ConfirmCheckpoint.java | 28 +-
.../flink/runtime/state/LocalStateHandle.java | 27 +-
.../apache/flink/runtime/state/StateHandle.java | 11 +-
.../apache/flink/runtime/state/StateUtils.java | 54 +++
.../runtime/taskmanager/RuntimeEnvironment.java | 25 +-
.../apache/flink/runtime/taskmanager/Task.java | 150 +++++-
.../flink/runtime/util/SerializableObject.java | 28 ++
.../flink/runtime/jobmanager/JobManager.scala | 108 +++--
.../StreamCheckpointCoordinator.scala | 151 ------
.../messages/CheckpointingMessages.scala | 52 --
.../flink/runtime/taskmanager/TaskManager.scala | 44 +-
.../checkpoint/CheckpointCoordinatorTest.java | 14 +-
.../checkpoint/CheckpointStateRestoreTest.java | 235 +++++++++
.../checkpoint/CoordinatorShutdownTest.java | 144 ++++++
.../messages/CheckpointMessagesTest.java | 17 +-
.../operators/testutils/MockEnvironment.java | 11 +-
.../api/graph/StreamingJobGraphGenerator.java | 70 ++-
.../streaming/runtime/tasks/StreamTask.java | 41 +-
.../StreamCheckpointingITCase.java | 4 +-
.../TaskManagerFailureRecoveryITCase.java | 2 +-
38 files changed, 1630 insertions(+), 690 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
index 9647ca4..b3f6587 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinator.java
@@ -18,10 +18,17 @@
package org.apache.flink.runtime.checkpoint;
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.PoisonPill;
+import akka.actor.Props;
+
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.executiongraph.Execution;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.executiongraph.ExecutionJobVertex;
import org.apache.flink.runtime.executiongraph.ExecutionVertex;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint;
import org.apache.flink.runtime.messages.checkpoint.ConfirmCheckpoint;
import org.apache.flink.runtime.messages.checkpoint.TriggerCheckpoint;
@@ -41,7 +48,10 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
/**
- *
+ * The checkpoint coordinator coordinates the distributed snapshots of operators and state.
+ * It triggers the checkpoint by sending the messages to the relevant tasks and collects the
+ * checkpoint acknowledgements. It also collects and maintains the overview of the state handles
+ * reported by the tasks that acknowledge the checkpoint.
*/
public class CheckpointCoordinator {
@@ -76,13 +86,17 @@ public class CheckpointCoordinator {
private final AtomicInteger numUnsuccessfulCheckpointsTriggers = new AtomicInteger();
- /** The timer that processes the checkpoint timeouts */
- private final Timer timeoutTimer;
+ /** The timer that handles the checkpoint timeouts and triggers periodic checkpoints */
+ private final Timer timer;
private final long checkpointTimeout;
private final int numSuccessfulCheckpointsToRetain;
+ private TimerTask periodicScheduler;
+
+ private ActorRef jobStatusListener;
+
private boolean shutdown;
// --------------------------------------------------------------------------------------------
@@ -114,9 +128,13 @@ public class CheckpointCoordinator {
this.completedCheckpoints = new ArrayDeque<SuccessfulCheckpoint>(numSuccessfulCheckpointsToRetain + 1);
this.recentPendingCheckpoints = new ArrayDeque<Long>(NUM_GHOST_CHECKPOINT_IDS);
- timeoutTimer = new Timer("Checkpoint Timeout Handler", true);
+ timer = new Timer("Checkpoint Timer", true);
}
+ // --------------------------------------------------------------------------------------------
+ // Clean shutdown
+ // --------------------------------------------------------------------------------------------
+
/**
* Shuts down the checkpoint coordinator.
*
@@ -129,9 +147,22 @@ public class CheckpointCoordinator {
return;
}
shutdown = true;
+ LOG.info("Stopping checkpoint coordinator jor job " + job);
// shut down the thread that handles the timeouts
- timeoutTimer.cancel();
+ timer.cancel();
+
+ // make sure that the actor does not linger
+ if (jobStatusListener != null) {
+ jobStatusListener.tell(PoisonPill.getInstance(), ActorRef.noSender());
+ jobStatusListener = null;
+ }
+
+ // the scheduling thread needs also to go away
+ if (periodicScheduler != null) {
+ periodicScheduler.cancel();
+ periodicScheduler = null;
+ }
// clear and discard all pending checkpoints
for (PendingCheckpoint pending : pendingCheckpoints.values()) {
@@ -146,6 +177,10 @@ public class CheckpointCoordinator {
completedCheckpoints.clear();
}
}
+
+ public boolean isShutdown() {
+ return shutdown;
+ }
// --------------------------------------------------------------------------------------------
// Handling checkpoints and messages
@@ -235,7 +270,7 @@ public class CheckpointCoordinator {
throw new IllegalStateException("Checkpoint coordinator has been shutdown.");
}
pendingCheckpoints.put(checkpointID, checkpoint);
- timeoutTimer.schedule(canceller, checkpointTimeout);
+ timer.schedule(canceller, checkpointTimeout);
}
// send the messages to the tasks that trigger their checkpoint
@@ -270,7 +305,8 @@ public class CheckpointCoordinator {
}
final long checkpointId = message.getCheckpointId();
- boolean checkpointCompleted = false;
+
+ SuccessfulCheckpoint completed = null;
synchronized (lock) {
// we need to check inside the lock for being shutdown as well, otherwise we
@@ -286,7 +322,7 @@ public class CheckpointCoordinator {
if (checkpoint.isFullyAcknowledged()) {
LOG.info("Completed checkpoint " + checkpointId);
- SuccessfulCheckpoint completed = checkpoint.toCompletedCheckpoint();
+ completed = checkpoint.toCompletedCheckpoint();
completedCheckpoints.addLast(completed);
if (completedCheckpoints.size() > numSuccessfulCheckpointsToRetain) {
completedCheckpoints.removeFirst();
@@ -295,8 +331,6 @@ public class CheckpointCoordinator {
rememberRecentCheckpointId(checkpointId);
dropSubsumedCheckpoints(completed.getTimestamp());
-
- checkpointCompleted = true;
}
}
else {
@@ -323,12 +357,13 @@ public class CheckpointCoordinator {
// send the confirmation messages to the necessary targets. we do this here
// to be outside the lock scope
- if (checkpointCompleted) {
+ if (completed != null) {
+ final long timestamp = completed.getTimestamp();
for (ExecutionVertex ev : tasksToCommitTo) {
Execution ee = ev.getCurrentExecutionAttempt();
if (ee != null) {
ExecutionAttemptID attemptId = ee.getAttemptId();
- ConfirmCheckpoint confirmMessage = new ConfirmCheckpoint(job, attemptId, checkpointId);
+ ConfirmCheckpoint confirmMessage = new ConfirmCheckpoint(job, attemptId, checkpointId, timestamp);
ev.sendMessageToCurrentExecution(confirmMessage, ee.getAttemptId());
}
}
@@ -355,6 +390,64 @@ public class CheckpointCoordinator {
}
// --------------------------------------------------------------------------------------------
+ // Checkpoint State Restoring
+ // --------------------------------------------------------------------------------------------
+
+ public void restoreLatestCheckpointedState(Map<JobVertexID, ExecutionJobVertex> tasks,
+ boolean errorIfNoCheckpoint,
+ boolean allOrNothingState) throws Exception {
+ synchronized (lock) {
+ if (shutdown) {
+ throw new IllegalStateException("CheckpointCoordinator is hut down");
+ }
+
+ if (completedCheckpoints.isEmpty()) {
+ if (errorIfNoCheckpoint) {
+ throw new IllegalStateException("No completed checkpoint available");
+ } else {
+ return;
+ }
+ }
+
+ // restore from the latest checkpoint
+ SuccessfulCheckpoint latest = completedCheckpoints.getLast();
+
+ if (allOrNothingState) {
+ Map<ExecutionJobVertex, Integer> stateCounts = new HashMap<ExecutionJobVertex, Integer>();
+
+ for (StateForTask state : latest.getStates()) {
+ ExecutionJobVertex vertex = tasks.get(state.getOperatorId());
+ Execution exec = vertex.getTaskVertices()[state.getSubtask()].getCurrentExecutionAttempt();
+ exec.setInitialState(state.getState());
+
+ Integer count = stateCounts.get(vertex);
+ if (count != null) {
+ stateCounts.put(vertex, count+1);
+ } else {
+ stateCounts.put(vertex, 1);
+ }
+ }
+
+ // validate that either all task vertices have state, or none
+ for (Map.Entry<ExecutionJobVertex, Integer> entry : stateCounts.entrySet()) {
+ ExecutionJobVertex vertex = entry.getKey();
+ if (entry.getValue() != vertex.getParallelism()) {
+ throw new IllegalStateException(
+ "The checkpoint contained state only for a subset of tasks for vertex " + vertex);
+ }
+ }
+ }
+ else {
+ for (StateForTask state : latest.getStates()) {
+ ExecutionJobVertex vertex = tasks.get(state.getOperatorId());
+ Execution exec = vertex.getTaskVertices()[state.getSubtask()].getCurrentExecutionAttempt();
+ exec.setInitialState(state.getState());
+ }
+ }
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------
// Accessors
// --------------------------------------------------------------------------------------------
@@ -377,4 +470,56 @@ public class CheckpointCoordinator {
return new ArrayList<SuccessfulCheckpoint>(this.completedCheckpoints);
}
}
+
+ // --------------------------------------------------------------------------------------------
+ // Periodic scheduling of checkpoints
+ // --------------------------------------------------------------------------------------------
+
+ public void startPeriodicCheckpointScheduler(long interval) {
+ synchronized (lock) {
+ if (shutdown) {
+ throw new IllegalArgumentException("Checkpoint coordinator is shut down");
+ }
+
+ // cancel any previous scheduler
+ stopPeriodicCheckpointScheduler();
+
+ // start a new scheduler
+ periodicScheduler = new TimerTask() {
+ @Override
+ public void run() {
+ try {
+ triggerCheckpoint();
+ }
+ catch (Exception e) {
+ LOG.error("Exception while triggering checkpoint", e);
+ }
+ }
+ };
+ timer.scheduleAtFixedRate(periodicScheduler, interval, interval);
+ }
+ }
+
+ public void stopPeriodicCheckpointScheduler() {
+ synchronized (lock) {
+ if (periodicScheduler != null) {
+ periodicScheduler.cancel();
+ periodicScheduler = null;
+ }
+ }
+ }
+
+ public ActorRef createJobStatusListener(ActorSystem actorSystem, long checkpointInterval) {
+ synchronized (lock) {
+ if (shutdown) {
+ throw new IllegalArgumentException("Checkpoint coordinator is shut down");
+ }
+
+ if (jobStatusListener == null) {
+ Props props = Props.create(CheckpointCoordinatorDeActivator.class, this, checkpointInterval);
+ jobStatusListener = actorSystem.actorOf(props);
+ }
+ return jobStatusListener;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorDeActivator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorDeActivator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorDeActivator.java
new file mode 100644
index 0000000..a6c4d76
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointCoordinatorDeActivator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint;
+
+import akka.actor.UntypedActor;
+import org.apache.flink.runtime.jobgraph.JobStatus;
+import org.apache.flink.runtime.messages.ExecutionGraphMessages;
+
+/**
+ * This actor listens to changes in the JobStatus and activates or deactivates the periodic
+ * checkpoint scheduler.
+ */
+public class CheckpointCoordinatorDeActivator extends UntypedActor {
+
+ private final CheckpointCoordinator coordinator;
+ private final long interval;
+
+ public CheckpointCoordinatorDeActivator(CheckpointCoordinator coordinator, long interval) {
+ this.coordinator = coordinator;
+ this.interval = interval;
+ }
+
+ @Override
+ public void onReceive(Object message) {
+ if (message instanceof ExecutionGraphMessages.JobStatusChanged) {
+ JobStatus status = ((ExecutionGraphMessages.JobStatusChanged) message).newJobStatus();
+
+ if (status == JobStatus.RUNNING) {
+ // start the checkpoint scheduler
+ coordinator.startPeriodicCheckpointScheduler(interval);
+ }
+ else {
+ // anything else should stop the trigger for now
+ coordinator.stopPeriodicCheckpointScheduler();
+ }
+ }
+
+ // we ignore all other messages
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
index e221238..f25bff9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/PendingCheckpoint.java
@@ -22,6 +22,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.ExecutionVertex;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
import java.util.ArrayList;
import java.util.List;
@@ -31,6 +32,9 @@ import java.util.Map;
* A pending checkpoint is a checkpoint that has been started, but has not been
* acknowledged by all tasks that need to acknowledge it. Once all tasks have
* acknowledged it, it becomes a {@link SuccessfulCheckpoint}.
+ *
+ * <p>Note that the pending checkpoint, as well as the successful checkpoint keep the
+ * state handles always as serialized values, never as actual values.</p>
*/
public class PendingCheckpoint {
@@ -117,12 +121,12 @@ public class PendingCheckpoint {
return completed;
}
else {
- throw new IllegalStateException("Cannot complete checkpoint while nit all tasks are acknowledged");
+ throw new IllegalStateException("Cannot complete checkpoint while not all tasks are acknowledged");
}
}
}
- public boolean acknowledgeTask(ExecutionAttemptID attemptID, StateHandle state) {
+ public boolean acknowledgeTask(ExecutionAttemptID attemptID, SerializedValue<StateHandle<?>> state) {
synchronized (lock) {
if (discarded) {
return false;
@@ -158,6 +162,7 @@ public class PendingCheckpoint {
@Override
public String toString() {
- return "";
+ return String.format("PendingCheckpoint %d @ %d - confirmed=%d, pending=%d",
+ checkpointId, checkpointTimestamp, getNumberOfAcknowledgedTasks(), getNumberOfNonAcknowledgedTasks());
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateForTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateForTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateForTask.java
index 83a6dc8..26b3eb7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateForTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/StateForTask.java
@@ -20,16 +20,22 @@ package org.apache.flink.runtime.checkpoint;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
/**
* Simple bean to describe the state belonging to a parallel operator.
* Since we hold the state across execution attempts, we identify a task by its
* JobVertexId and subtask index.
+ *
+ * The state itself is kept in serialized from, since the checkpoint coordinator itself
+ * is never looking at it anyways and only sends it back out in case of a recovery.
+ * Furthermore, the state may involve user-defined classes that are not accessible without
+ * the respective classloader.
*/
public class StateForTask {
/** The state of the parallel operator */
- private final StateHandle state;
+ private final SerializedValue<StateHandle<?>> state;
/** The vertex id of the parallel operator */
private final JobVertexID operatorId;
@@ -37,7 +43,7 @@ public class StateForTask {
/** The index of the parallel subtask */
private final int subtask;
- public StateForTask(StateHandle state, JobVertexID operatorId, int subtask) {
+ public StateForTask(SerializedValue<StateHandle<?>> state, JobVertexID operatorId, int subtask) {
if (state == null || operatorId == null || subtask < 0) {
throw new IllegalArgumentException();
}
@@ -49,7 +55,7 @@ public class StateForTask {
// --------------------------------------------------------------------------------------------
- public StateHandle getState() {
+ public SerializedValue<StateHandle<?>> getState() {
return state;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
index 5d96903..0a1268d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/TaskDeploymentDescriptor.java
@@ -24,6 +24,7 @@ import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
import java.io.Serializable;
import java.util.Collection;
@@ -77,9 +78,8 @@ public final class TaskDeploymentDescriptor implements Serializable {
/** The list of JAR files required to run this task. */
private final List<BlobKey> requiredJarFiles;
- private StateHandle operatorStates;
-
-
+ private final SerializedValue<StateHandle<?>> operatorState;
+
/**
* Constructs a task deployment descriptor.
*/
@@ -89,15 +89,18 @@ public final class TaskDeploymentDescriptor implements Serializable {
Configuration taskConfiguration, String invokableClassName,
List<ResultPartitionDeploymentDescriptor> producedPartitions,
List<InputGateDeploymentDescriptor> inputGates,
- List<BlobKey> requiredJarFiles, int targetSlotNumber) {
+ List<BlobKey> requiredJarFiles, int targetSlotNumber,
+ SerializedValue<StateHandle<?>> operatorState) {
+ checkArgument(indexInSubtaskGroup >= 0);
+ checkArgument(numberOfSubtasks > indexInSubtaskGroup);
+ checkArgument(targetSlotNumber >= 0);
+
this.jobID = checkNotNull(jobID);
this.vertexID = checkNotNull(vertexID);
this.executionId = checkNotNull(executionId);
this.taskName = checkNotNull(taskName);
- checkArgument(indexInSubtaskGroup >= 0);
this.indexInSubtaskGroup = indexInSubtaskGroup;
- checkArgument(numberOfSubtasks > indexInSubtaskGroup);
this.numberOfSubtasks = numberOfSubtasks;
this.jobConfiguration = checkNotNull(jobConfiguration);
this.taskConfiguration = checkNotNull(taskConfiguration);
@@ -105,8 +108,8 @@ public final class TaskDeploymentDescriptor implements Serializable {
this.producedPartitions = checkNotNull(producedPartitions);
this.inputGates = checkNotNull(inputGates);
this.requiredJarFiles = checkNotNull(requiredJarFiles);
- checkArgument(targetSlotNumber >= 0);
this.targetSlotNumber = targetSlotNumber;
+ this.operatorState = operatorState;
}
public TaskDeploymentDescriptor(
@@ -115,14 +118,11 @@ public final class TaskDeploymentDescriptor implements Serializable {
Configuration taskConfiguration, String invokableClassName,
List<ResultPartitionDeploymentDescriptor> producedPartitions,
List<InputGateDeploymentDescriptor> inputGates,
- List<BlobKey> requiredJarFiles, int targetSlotNumber,
- StateHandle operatorStates) {
+ List<BlobKey> requiredJarFiles, int targetSlotNumber) {
this(jobID, vertexID, executionId, taskName, indexInSubtaskGroup, numberOfSubtasks,
jobConfiguration, taskConfiguration, invokableClassName, producedPartitions,
- inputGates, requiredJarFiles, targetSlotNumber);
-
- setOperatorState(operatorStates);
+ inputGates, requiredJarFiles, targetSlotNumber, null);
}
/**
@@ -232,11 +232,7 @@ public final class TaskDeploymentDescriptor implements Serializable {
return strBuilder.toString();
}
- public void setOperatorState(StateHandle operatorStates) {
- this.operatorStates = operatorStates;
- }
-
- public StateHandle getOperatorStates() {
- return operatorStates;
+ public SerializedValue<StateHandle<?>> getOperatorState() {
+ return operatorState;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
index 081e3ca..755f1ad 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.execution;
-import akka.actor.ActorRef;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
@@ -31,6 +30,7 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.memorymanager.MemoryManager;
+import org.apache.flink.runtime.state.StateHandle;
import java.util.Map;
import java.util.concurrent.Future;
@@ -148,6 +148,25 @@ public interface Environment {
*/
void reportAccumulators(Map<String, Accumulator<?, ?>> accumulators);
+ /**
+ * Confirms that the invokable has successfully completed all steps it needed to
+ * to for the checkpoint with the give checkpoint-ID. This method does not include
+ * any state in the checkpoint.
+ *
+ * @param checkpointId The ID of the checkpoint.
+ */
+ void acknowledgeCheckpoint(long checkpointId);
+
+ /**
+ * Confirms that the invokable has successfully completed all steps it needed to
+ * to for the checkpoint with the give checkpoint-ID. This method does include
+ * the given state in the checkpoint.
+ *
+ * @param checkpointId The ID of the checkpoint.
+ * @param state A handle to the state to be included in the checkpoint.
+ */
+ void acknowledgeCheckpoint(long checkpointId, StateHandle<?> state);
+
// --------------------------------------------------------------------------------------------
// Fields relevant to the I/O system. Should go into Task
// --------------------------------------------------------------------------------------------
@@ -159,7 +178,4 @@ public interface Environment {
InputGate getInputGate(int index);
InputGate[] getAllInputGates();
-
- // this should go away
- ActorRef getJobManager();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index 4e046dd..731d70f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -46,6 +46,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
import org.apache.flink.runtime.messages.Messages;
import org.apache.flink.runtime.messages.TaskMessages.TaskOperationResult;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
import org.apache.flink.runtime.taskmanager.Task;
import org.apache.flink.util.ExceptionUtils;
import org.slf4j.Logger;
@@ -129,7 +130,7 @@ public class Execution implements Serializable {
private volatile InstanceConnectionInfo assignedResourceLocation; // for the archived execution
- private StateHandle operatorState;
+ private SerializedValue<StateHandle<?>> operatorState;
// --------------------------------------------------------------------------------------------
@@ -204,6 +205,13 @@ public class Execution implements Serializable {
partialInputChannelDeploymentDescriptors = null;
}
+ public void setInitialState(SerializedValue<StateHandle<?>> initialState) {
+ if (state != ExecutionState.CREATED) {
+ throw new IllegalArgumentException("Can only assign operator state when execution attempt is in CREATED");
+ }
+ this.operatorState = initialState;
+ }
+
// --------------------------------------------------------------------------------------------
// Actions
// --------------------------------------------------------------------------------------------
@@ -325,7 +333,7 @@ public class Execution implements Serializable {
attemptNumber, slot.getInstance().getInstanceConnectionInfo().getHostname()));
}
- final TaskDeploymentDescriptor deployment = vertex.createDeploymentDescriptor(attemptId, slot);
+ final TaskDeploymentDescriptor deployment = vertex.createDeploymentDescriptor(attemptId, slot, operatorState);
// register this execution at the execution graph, to receive call backs
vertex.getExecutionGraph().registerExecution(this);
@@ -903,12 +911,4 @@ public class Execution implements Serializable {
return String.format("Attempt #%d (%s) @ %s - [%s]", attemptNumber, vertex.getSimpleName(),
(assignedResource == null ? "(unassigned)" : assignedResource.toString()), state);
}
-
- public void setOperatorState(StateHandle operatorStates) {
- this.operatorState = operatorStates;
- }
-
- public StateHandle getOperatorState() {
- return operatorState;
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
index d38913e..90cf42e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionGraph.java
@@ -18,12 +18,14 @@
package org.apache.flink.runtime.executiongraph;
-import akka.actor.ActorContext;
import akka.actor.ActorRef;
+
+import akka.actor.ActorSystem;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.JobException;
import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.checkpoint.CheckpointCoordinator;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.jobgraph.AbstractJobVertex;
@@ -32,20 +34,20 @@ import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobgraph.JobStatus;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.ScheduleMode;
-import org.apache.flink.runtime.jobmanager.StreamCheckpointCoordinator;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.messages.ExecutionGraphMessages;
-import org.apache.flink.runtime.state.StateHandle;
import org.apache.flink.runtime.taskmanager.TaskExecutionState;
+import org.apache.flink.runtime.util.SerializableObject;
import org.apache.flink.util.ExceptionUtils;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import scala.Tuple3;
-import scala.concurrent.duration.Duration;
+
import scala.concurrent.duration.FiniteDuration;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -54,7 +56,6 @@ import java.util.NoSuchElementException;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import static akka.dispatch.Futures.future;
@@ -80,8 +81,11 @@ import static akka.dispatch.Futures.future;
* about deployment of tasks and updates in the task status always use the ExecutionAttemptID to
* address the message receiver.</li>
* </ul>
- *
- *
+ *
+ * <p>The ExecutionGraph implements {@link java.io.Serializable}, because it can be archived by
+ * sending it to an archive actor via an actor message. The execution graph does contain some
+ * non-serializable fields. These fields are not required in the archived form and are cleared
+ * in the {@link #prepareForArchiving()} method.</p>
*/
public class ExecutionGraph implements Serializable {
@@ -92,9 +96,15 @@ public class ExecutionGraph implements Serializable {
/** The log object used for debugging. */
static final Logger LOG = LoggerFactory.getLogger(ExecutionGraph.class);
+
+ private static final int NUMBER_OF_SUCCESSFUL_CHECKPOINTS_TO_RETAIN = 1;
// --------------------------------------------------------------------------------------------
+ /** The lock used to secure all access to mutable fields, especially the tracking of progress
+ * within the job. */
+ private final SerializableObject progressLock = new SerializableObject();
+
/** The ID of the job this graph has been built for. */
private final JobID jobID;
@@ -104,9 +114,6 @@ public class ExecutionGraph implements Serializable {
/** The job configuration that was originally attached to the JobGraph. */
private final Configuration jobConfiguration;
- /** The classloader for the user code. Needed for calls into user code classes */
- private ClassLoader userClassLoader;
-
/** All job vertices that are part of this graph */
private final ConcurrentHashMap<JobVertexID, ExecutionJobVertex> tasks;
@@ -123,8 +130,11 @@ public class ExecutionGraph implements Serializable {
* inside the BlobService and are referenced via the BLOB keys. */
private final List<BlobKey> requiredJarFiles;
+ /** Listeners that receive messages when the entire job switches it status (such as from
+ * RUNNING to FINISHED) */
private final List<ActorRef> jobStatusListenerActors;
+ /** Listeners that receive messages whenever a single task execution changes its status */
private final List<ActorRef> executionListenerActors;
/** Timestamps (in milliseconds as returned by {@code System.currentTimeMillis()} when
@@ -133,10 +143,6 @@ public class ExecutionGraph implements Serializable {
* at {@code stateTimestamps[RUNNING.ordinal()]}. */
private final long[] stateTimestamps;
- /** The lock used to secure all access to mutable fields, especially the tracking of progress
- * within the job. */
- private final Object progressLock = new Object();
-
/** The timeout for all messages that require a response/acknowledgement */
private final FiniteDuration timeout;
@@ -158,8 +164,11 @@ public class ExecutionGraph implements Serializable {
* from results than need to be materialized. */
private ScheduleMode scheduleMode = ScheduleMode.FROM_SOURCES;
+ /** Flag that indicate whether the executed dataflow should be periodically snapshotted */
+ private boolean snapshotCheckpointsEnabled;
+
- // ------ Execution status and progress -------
+ // ------ Execution status and progress. These values are volatile, and accessed under the lock -------
/** Current status of the job execution */
private volatile JobStatus state = JobStatus.CREATED;
@@ -168,31 +177,36 @@ public class ExecutionGraph implements Serializable {
* that was not recoverable and triggered job failure */
private volatile Throwable failureCause;
- /** The scheduler to use for scheduling new tasks as they are needed */
- private Scheduler scheduler;
-
/** The position of the vertex that is next expected to finish.
* This is an index into the "verticesInCreationOrder" collection.
* Once this value has reached the number of vertices, the job is done. */
- private int nextVertexToFinish;
-
-
-
- private ActorContext parentContext;
-
- private ActorRef stateCheckpointerActor;
-
- private boolean checkpointingEnabled;
+ private volatile int nextVertexToFinish;
+
+
+ // ------ Fields that are relevant to the execution and need to be cleared before archiving -------
- private long checkpointingInterval = 5000;
+ /** The scheduler to use for scheduling new tasks as they are needed */
+ @SuppressWarnings("NonSerializableFieldInSerializableClass")
+ private Scheduler scheduler;
- public ExecutionGraph(JobID jobId, String jobName, Configuration jobConfig, FiniteDuration timeout) {
- this(jobId, jobName, jobConfig, timeout, new ArrayList<BlobKey>());
- }
+ /** The classloader for the user code. Needed for calls into user code classes */
+ @SuppressWarnings("NonSerializableFieldInSerializableClass")
+ private ClassLoader userClassLoader;
+
+ /** The coordinator for checkpoints, if snapshot checkpoints are enabled */
+ @SuppressWarnings("NonSerializableFieldInSerializableClass")
+ private CheckpointCoordinator checkpointCoordinator;
+
+
+ // --------------------------------------------------------------------------------------------
+ // Constructors
+ // --------------------------------------------------------------------------------------------
- public ExecutionGraph(JobID jobId, String jobName, Configuration jobConfig,
- FiniteDuration timeout, List<BlobKey> requiredJarFiles) {
- this(jobId, jobName, jobConfig, timeout, requiredJarFiles, Thread.currentThread().getContextClassLoader());
+ /**
+ * This constructor is for tests only, because it does not include class loading information.
+ */
+ ExecutionGraph(JobID jobId, String jobName, Configuration jobConfig, FiniteDuration timeout) {
+ this(jobId, jobName, jobConfig, timeout, new ArrayList<BlobKey>(), ExecutionGraph.class.getClassLoader());
}
public ExecutionGraph(JobID jobId, String jobName, Configuration jobConfig, FiniteDuration timeout,
@@ -224,18 +238,8 @@ public class ExecutionGraph implements Serializable {
}
// --------------------------------------------------------------------------------------------
-
- public void setStateCheckpointerActor(ActorRef stateCheckpointerActor) {
- this.stateCheckpointerActor = stateCheckpointerActor;
- }
-
- public ActorRef getStateCheckpointerActor() {
- return stateCheckpointerActor;
- }
-
- public void setParentContext(ActorContext parentContext) {
- this.parentContext = parentContext;
- }
+ // Configuration of Data-flow wide execution settings
+ // --------------------------------------------------------------------------------------------
public void setNumberOfRetriesLeft(int numberOfRetriesLeft) {
if (numberOfRetriesLeft < -1) {
@@ -259,46 +263,97 @@ public class ExecutionGraph implements Serializable {
return delayBeforeRetrying;
}
- public void attachJobGraph(List<AbstractJobVertex> topologiallySorted) throws JobException {
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format("Attaching %d topologically sorted vertices to existing job graph with %d "
- + "vertices and %d intermediate results.", topologiallySorted.size(), tasks.size(), intermediateResults.size()));
+ public boolean isQueuedSchedulingAllowed() {
+ return this.allowQueuedScheduling;
+ }
+
+ public void setQueuedSchedulingAllowed(boolean allowed) {
+ this.allowQueuedScheduling = allowed;
+ }
+
+ public void setScheduleMode(ScheduleMode scheduleMode) {
+ this.scheduleMode = scheduleMode;
+ }
+
+ public ScheduleMode getScheduleMode() {
+ return scheduleMode;
+ }
+
+ public void enableSnaphotCheckpointing(long interval, long checkpointTimeout,
+ List<ExecutionJobVertex> verticesToTrigger,
+ List<ExecutionJobVertex> verticesToWaitFor,
+ List<ExecutionJobVertex> verticesToCommitTo,
+ ActorSystem actorSystem)
+ {
+ // simple sanity checks
+ if (interval < 10 || checkpointTimeout < 10) {
+ throw new IllegalArgumentException();
+ }
+ if (state != JobStatus.CREATED) {
+ throw new IllegalStateException("Job must be in CREATED state");
}
+
+ ExecutionVertex[] tasksToTrigger = collectExecutionVertices(verticesToTrigger);
+ ExecutionVertex[] tasksToWaitFor = collectExecutionVertices(verticesToWaitFor);
+ ExecutionVertex[] tasksToCommitTo = collectExecutionVertices(verticesToCommitTo);
- final long createTimestamp = System.currentTimeMillis();
+ // disable to make sure existing checkpoint coordinators are cleared
+ disableSnaphotCheckpointing();
- for (AbstractJobVertex jobVertex : topologiallySorted) {
-
- // create the execution job vertex and attach it to the graph
- ExecutionJobVertex ejv = new ExecutionJobVertex(this, jobVertex, 1, timeout, createTimestamp);
- ejv.connectToPredecessors(this.intermediateResults);
-
- ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv);
- if (previousTask != null) {
- throw new JobException(String.format("Encountered two job vertices with ID %s : previous=[%s] / new=[%s]",
- jobVertex.getID(), ejv, previousTask));
- }
-
- for (IntermediateResult res : ejv.getProducedDataSets()) {
- IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res);
- if (previousDataSet != null) {
- throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]",
- res.getId(), res, previousDataSet));
- }
- }
-
- this.verticesInCreationOrder.add(ejv);
+ // create the coordinator that triggers and commits checkpoints and holds the state
+ snapshotCheckpointsEnabled = true;
+ checkpointCoordinator = new CheckpointCoordinator(jobID, NUMBER_OF_SUCCESSFUL_CHECKPOINTS_TO_RETAIN,
+ checkpointTimeout, tasksToTrigger, tasksToWaitFor, tasksToCommitTo);
+
+ // the periodic checkpoint scheduler is activated and deactivated as a result of
+ // job status changes (running -> on, all other states -> off)
+ registerJobStatusListener(checkpointCoordinator.createJobStatusListener(actorSystem, interval));
+ }
+
+ public void disableSnaphotCheckpointing() {
+ if (state != JobStatus.CREATED) {
+ throw new IllegalStateException("Job must be in CREATED state");
+ }
+
+ snapshotCheckpointsEnabled = false;
+ if (checkpointCoordinator != null) {
+ checkpointCoordinator.shutdown();
+ checkpointCoordinator = null;
}
}
+
+ public boolean isSnapshotCheckpointsEnabled() {
+ return snapshotCheckpointsEnabled;
+ }
- public void setCheckpointingEnabled(boolean checkpointingEnabled) {
- this.checkpointingEnabled = checkpointingEnabled;
+ public CheckpointCoordinator getCheckpointCoordinator() {
+ return checkpointCoordinator;
}
- public void setCheckpointingInterval(long checkpointingInterval) {
- this.checkpointingInterval = checkpointingInterval;
+ private ExecutionVertex[] collectExecutionVertices(List<ExecutionJobVertex> jobVertices) {
+ if (jobVertices.size() == 1) {
+ ExecutionJobVertex jv = jobVertices.get(0);
+ if (jv.getGraph() != this) {
+ throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph");
+ }
+ return jv.getTaskVertices();
+ }
+ else {
+ ArrayList<ExecutionVertex> all = new ArrayList<ExecutionVertex>();
+ for (ExecutionJobVertex jv : jobVertices) {
+ if (jv.getGraph() != this) {
+ throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph");
+ }
+ all.addAll(Arrays.asList(jv.getTaskVertices()));
+ }
+ return all.toArray(new ExecutionVertex[all.size()]);
+ }
}
+ // --------------------------------------------------------------------------------------------
+ // Properties and Status of the Execution Graph
+ // --------------------------------------------------------------------------------------------
+
/**
* Returns a list of BLOB keys referring to the JAR files required to run this job
* @return list of BLOB keys referring to the JAR files required to run this job
@@ -307,8 +362,6 @@ public class ExecutionGraph implements Serializable {
return this.requiredJarFiles;
}
- // --------------------------------------------------------------------------------------------
-
public Scheduler getScheduler() {
return scheduler;
}
@@ -396,26 +449,42 @@ public class ExecutionGraph implements Serializable {
return this.stateTimestamps[status.ordinal()];
}
- public boolean isQueuedSchedulingAllowed() {
- return this.allowQueuedScheduling;
- }
+ // --------------------------------------------------------------------------------------------
+ // Actions
+ // --------------------------------------------------------------------------------------------
- public void setQueuedSchedulingAllowed(boolean allowed) {
- this.allowQueuedScheduling = allowed;
- }
+ public void attachJobGraph(List<AbstractJobVertex> topologiallySorted) throws JobException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format("Attaching %d topologically sorted vertices to existing job graph with %d "
+ + "vertices and %d intermediate results.", topologiallySorted.size(), tasks.size(), intermediateResults.size()));
+ }
- public void setScheduleMode(ScheduleMode scheduleMode) {
- this.scheduleMode = scheduleMode;
- }
+ final long createTimestamp = System.currentTimeMillis();
- public ScheduleMode getScheduleMode() {
- return scheduleMode;
- }
+ for (AbstractJobVertex jobVertex : topologiallySorted) {
- // --------------------------------------------------------------------------------------------
- // Actions
- // --------------------------------------------------------------------------------------------
+ // create the execution job vertex and attach it to the graph
+ ExecutionJobVertex ejv = new ExecutionJobVertex(this, jobVertex, 1, timeout, createTimestamp);
+ ejv.connectToPredecessors(this.intermediateResults);
+ ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv);
+ if (previousTask != null) {
+ throw new JobException(String.format("Encountered two job vertices with ID %s : previous=[%s] / new=[%s]",
+ jobVertex.getID(), ejv, previousTask));
+ }
+
+ for (IntermediateResult res : ejv.getProducedDataSets()) {
+ IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res);
+ if (previousDataSet != null) {
+ throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]",
+ res.getId(), res, previousDataSet));
+ }
+ }
+
+ this.verticesInCreationOrder.add(ejv);
+ }
+ }
+
public void scheduleForExecution(Scheduler scheduler) throws JobException {
if (scheduler == null) {
throw new IllegalArgumentException("Scheduler must not be null.");
@@ -431,32 +500,24 @@ public class ExecutionGraph implements Serializable {
switch (scheduleMode) {
case FROM_SOURCES:
- // initially, we simply take the ones without inputs.
- // next, we implement the logic to go back from vertices that need computation
- // to the ones we need to start running
+ // simply take the vertices without inputs.
for (ExecutionJobVertex ejv : this.tasks.values()) {
if (ejv.getJobVertex().isInputVertex()) {
ejv.scheduleAll(scheduler, allowQueuedScheduling);
}
}
-
break;
case ALL:
for (ExecutionJobVertex ejv : getVerticesTopologically()) {
ejv.scheduleAll(scheduler, allowQueuedScheduling);
}
-
break;
case BACKTRACKING:
+ // go back from vertices that need computation to the ones we need to run
throw new JobException("BACKTRACKING is currently not supported as schedule mode.");
}
-
- if (checkpointingEnabled) {
- stateCheckpointerActor = StreamCheckpointCoordinator.spawn(parentContext, this,
- Duration.create(checkpointingInterval, TimeUnit.MILLISECONDS));
- }
}
else {
throw new IllegalStateException("Job may only be scheduled from state " + JobStatus.CREATED);
@@ -508,6 +569,83 @@ public class ExecutionGraph implements Serializable {
}
}
+ public void restart() {
+ try {
+ if (state == JobStatus.FAILED) {
+ if (!transitionState(JobStatus.FAILED, JobStatus.RESTARTING)) {
+ throw new IllegalStateException("Execution Graph left the state FAILED while trying to restart.");
+ }
+ }
+
+ synchronized (progressLock) {
+ if (state != JobStatus.RESTARTING) {
+ throw new IllegalStateException("Can only restart job from state restarting.");
+ }
+ if (scheduler == null) {
+ throw new IllegalStateException("The execution graph has not been scheduled before - scheduler is null.");
+ }
+
+ this.currentExecutions.clear();
+
+ for (ExecutionJobVertex jv : this.verticesInCreationOrder) {
+ jv.resetForNewExecution();
+ }
+
+ for (int i = 0; i < stateTimestamps.length; i++) {
+ stateTimestamps[i] = 0;
+ }
+ nextVertexToFinish = 0;
+ transitionState(JobStatus.RESTARTING, JobStatus.CREATED);
+
+ // if we have checkpointed state, reload it into the executions
+ if (checkpointCoordinator != null) {
+ checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), false, false);
+ }
+ }
+
+ scheduleForExecution(scheduler);
+ }
+ catch (Throwable t) {
+ fail(t);
+ }
+ }
+
+ /**
+ * This method cleans fields that are irrelevant for the archived execution attempt.
+ */
+ public void prepareForArchiving() {
+ if (!state.isTerminalState()) {
+ throw new IllegalStateException("Can only archive the job from a terminal state");
+ }
+
+ // clear the non-serializable fields
+ userClassLoader = null;
+ scheduler = null;
+ checkpointCoordinator = null;
+
+ for (ExecutionJobVertex vertex : verticesInCreationOrder) {
+ vertex.prepareForArchiving();
+ }
+
+ intermediateResults.clear();
+ currentExecutions.clear();
+ requiredJarFiles.clear();
+ jobStatusListenerActors.clear();
+ executionListenerActors.clear();
+ }
+
+ /**
+ * For testing: This waits until the job execution has finished.
+ * @throws InterruptedException
+ */
+ public void waitUntilFinished() throws InterruptedException {
+ synchronized (progressLock) {
+ while (nextVertexToFinish < verticesInCreationOrder.size()) {
+ progressLock.wait();
+ }
+ }
+ }
+
private boolean transitionState(JobStatus current, JobStatus newState) {
return transitionState(current, newState, null);
}
@@ -551,24 +689,32 @@ public class ExecutionGraph implements Serializable {
if (nextPos == verticesInCreationOrder.size()) {
// we are done, transition to the final state
-
+ JobStatus current;
while (true) {
- JobStatus current = this.state;
- if (current == JobStatus.RUNNING && transitionState(current, JobStatus.FINISHED)) {
- break;
+ current = this.state;
+
+ if (current == JobStatus.RUNNING) {
+ if (transitionState(current, JobStatus.FINISHED)) {
+ postRunCleanup();
+ break;
+ }
}
- if (current == JobStatus.CANCELLING && transitionState(current, JobStatus.CANCELED)) {
- break;
+ else if (current == JobStatus.CANCELLING) {
+ if (transitionState(current, JobStatus.CANCELED)) {
+ postRunCleanup();
+ break;
+ }
}
- if (current == JobStatus.FAILING) {
+ else if (current == JobStatus.FAILING) {
if (numberOfRetriesLeft > 0 && transitionState(current, JobStatus.RESTARTING)) {
numberOfRetriesLeft--;
future(new Callable<Object>() {
@Override
public Object call() throws Exception {
- try{
+ try {
Thread.sleep(delayBeforeRetrying);
- }catch(InterruptedException e){
+ }
+ catch(InterruptedException e){
// should only happen on shutdown
}
restart();
@@ -578,13 +724,15 @@ public class ExecutionGraph implements Serializable {
break;
}
else if (numberOfRetriesLeft <= 0 && transitionState(current, JobStatus.FAILED, failureCause)) {
+ postRunCleanup();
break;
}
}
- if (current == JobStatus.CANCELED || current == JobStatus.CREATED || current == JobStatus.FINISHED) {
+ else {
fail(new Exception("ExecutionGraph went into final state from state " + current));
}
}
+ // done transitioning the state
// also, notify waiters
progressLock.notifyAll();
@@ -592,6 +740,19 @@ public class ExecutionGraph implements Serializable {
}
}
}
+
+ private void postRunCleanup() {
+ try {
+ CheckpointCoordinator coord = this.checkpointCoordinator;
+ this.checkpointCoordinator = null;
+ if (coord != null) {
+ coord.shutdown();
+ }
+ }
+ catch (Exception e) {
+ LOG.error("Error while cleaning up after execution", e);
+ }
+ }
// --------------------------------------------------------------------------------------------
// Callbacks and Callback Utilities
@@ -623,13 +784,6 @@ public class ExecutionGraph implements Serializable {
return false;
}
}
-
- public void loadOperatorStates(Map<Tuple3<JobVertexID, Integer, Long> , StateHandle> states) {
- synchronized (this.progressLock) {
- for (Map.Entry<Tuple3<JobVertexID, Integer, Long>, StateHandle> state : states.entrySet())
- tasks.get(state.getKey()._1()).getTaskVertices()[state.getKey()._2()].setOperatorState(state.getValue());
- }
- }
public void scheduleOrUpdateConsumers(ResultPartitionID partitionId) {
@@ -670,21 +824,19 @@ public class ExecutionGraph implements Serializable {
// Listeners & Observers
// --------------------------------------------------------------------------------------------
- public void registerJobStatusListener(ActorRef listener){
- this.jobStatusListenerActors.add(listener);
- }
-
- public void registerExecutionListener(ActorRef listener){
- this.executionListenerActors.add(listener);
+ public void registerJobStatusListener(ActorRef listener) {
+ if (listener != null) {
+ this.jobStatusListenerActors.add(listener);
+ }
}
- public boolean containsJobStatusListener(ActorRef listener) {
- return this.jobStatusListenerActors.contains(listener);
+ public void registerExecutionListener(ActorRef listener) {
+ if (listener != null) {
+ this.executionListenerActors.add(listener);
+ }
}
-
- /**
- * NOTE: This method never throws an error, only logs errors caused by the notified listeners.
- */
+
+
private void notifyJobStatusChange(JobStatus newState, Throwable error) {
if (jobStatusListenerActors.size() > 0) {
ExecutionGraphMessages.JobStatusChanged message =
@@ -695,10 +847,7 @@ public class ExecutionGraph implements Serializable {
}
}
}
-
- /**
- * NOTE: This method never throws an error, only logs errors caused by the notified listeners.
- */
+
void notifyExecutionChange(JobVertexID vertexId, int subtask, ExecutionAttemptID executionID, ExecutionState
newExecutionState, Throwable error)
{
@@ -722,65 +871,4 @@ public class ExecutionGraph implements Serializable {
fail(error);
}
}
-
- public void restart() {
- try {
- if (state == JobStatus.FAILED) {
- if (!transitionState(JobStatus.FAILED, JobStatus.RESTARTING)) {
- throw new IllegalStateException("Execution Graph left the state FAILED while trying to restart.");
- }
- }
-
- synchronized (progressLock) {
- if (state != JobStatus.RESTARTING) {
- throw new IllegalStateException("Can only restart job from state restarting.");
- }
- if (scheduler == null) {
- throw new IllegalStateException("The execution graph has not been scheduled before - scheduler is null.");
- }
-
- this.currentExecutions.clear();
-
- for (ExecutionJobVertex jv : this.verticesInCreationOrder) {
- jv.resetForNewExecution();
- }
-
- for (int i = 0; i < stateTimestamps.length; i++) {
- stateTimestamps[i] = 0;
- }
- nextVertexToFinish = 0;
- transitionState(JobStatus.RESTARTING, JobStatus.CREATED);
- }
-
- scheduleForExecution(scheduler);
- }
- catch (Throwable t) {
- fail(t);
- }
- }
-
- /**
- * This method cleans fields that are irrelevant for the archived execution attempt.
- */
- public void prepareForArchiving() {
- if (!state.isTerminalState()) {
- throw new IllegalStateException("Can only archive the job from a terminal state");
- }
-
- userClassLoader = null;
-
- for (ExecutionJobVertex vertex : verticesInCreationOrder) {
- vertex.prepareForArchiving();
- }
-
- intermediateResults.clear();
- currentExecutions.clear();
- requiredJarFiles.clear();
- jobStatusListenerActors.clear();
- executionListenerActors.clear();
-
- scheduler = null;
- parentContext = null;
- stateCheckpointerActor = null;
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
index acbc17a..59b3bb6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionJobVertex.java
@@ -35,6 +35,7 @@ import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup;
+import org.apache.flink.runtime.util.SerializableObject;
import org.slf4j.Logger;
import scala.concurrent.duration.FiniteDuration;
@@ -52,7 +53,7 @@ public class ExecutionJobVertex implements Serializable {
/** Use the same log for all ExecutionGraph classes */
private static final Logger LOG = ExecutionGraph.LOG;
- private final Object stateMonitor = new Object();
+ private final SerializableObject stateMonitor = new SerializableObject();
private final ExecutionGraph graph;
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
index a44fc6f..2ad3a55 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.executiongraph;
import akka.actor.ActorRef;
+
import org.apache.flink.runtime.JobException;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor;
@@ -41,8 +42,11 @@ import org.apache.flink.runtime.jobmanager.scheduler.CoLocationConstraint;
import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup;
import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException;
import org.apache.flink.runtime.jobmanager.scheduler.Scheduler;
+
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
import org.slf4j.Logger;
+
import scala.concurrent.duration.FiniteDuration;
import java.io.Serializable;
@@ -95,8 +99,6 @@ public class ExecutionVertex implements Serializable {
private volatile boolean scheduleLocalOnly;
- private StateHandle operatorState;
-
// --------------------------------------------------------------------------------------------
public ExecutionVertex(ExecutionJobVertex jobVertex, int subTaskIndex,
@@ -212,14 +214,6 @@ public class ExecutionVertex implements Serializable {
public InstanceConnectionInfo getCurrentAssignedResourceLocation() {
return currentExecution.getAssignedResourceLocation();
}
-
- public void setOperatorState(StateHandle operatorState) {
- this.operatorState = operatorState;
- }
-
- public StateHandle getOperatorState() {
- return operatorState;
- }
public ExecutionGraph getExecutionGraph() {
return this.jobVertex.getGraph();
@@ -421,11 +415,6 @@ public class ExecutionVertex implements Serializable {
if (grp != null) {
this.locationConstraint = grp.getLocationConstraint(subTaskIndex);
}
-
- if (operatorState != null) {
- execution.setOperatorState(operatorState);
- }
-
}
else {
throw new IllegalStateException("Cannot reset a vertex that is in state " + state);
@@ -524,6 +513,7 @@ public class ExecutionVertex implements Serializable {
// clear the unnecessary fields in this class
this.resultPartitions = null;
this.inputEdges = null;
+ this.locationConstraint = null;
this.locationConstraintInstances = null;
}
@@ -588,10 +578,13 @@ public class ExecutionVertex implements Serializable {
/**
* Creates a task deployment descriptor to deploy a subtask to the given target slot.
+ *
+ * TODO: This should actually be in the EXECUTION
*/
TaskDeploymentDescriptor createDeploymentDescriptor(
ExecutionAttemptID executionId,
- SimpleSlot targetSlot) {
+ SimpleSlot targetSlot,
+ SerializedValue<StateHandle<?>> operatorState) {
// Produced intermediate results
List<ResultPartitionDeploymentDescriptor> producedPartitions = new ArrayList<ResultPartitionDeploymentDescriptor>(resultPartitions.size());
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
index 6d895f9..28fa78e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java
@@ -38,9 +38,18 @@ import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.blob.BlobClient;
import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.jobgraph.tasks.JobSnapshottingSettings;
/**
- * A job graph represents an entire Flink runtime job.
+ * The JobGraph represents a Flink dataflow program, at the low level that the JobManager accepts.
+ * All programs from higher level APIs are transformed into JobGraphs.
+ *
+ * <p>The JobGraph is a graph of vertices and intermediate results that are connected together to
+ * form a DAG. Note that iterations (feedback edges) are currently not encoded inside the JobGraph
+ * but inside certain special vertices that establish the feedback channel amongst themselves.</p>
+ *
+ * <p>The JobGraph defines the job-wide configuration settings, while each vertex and intermediate result
+ * define the characteristics of the concrete operation and intermediate data.</p>
*/
public class JobGraph implements Serializable {
@@ -74,11 +83,12 @@ public class JobGraph implements Serializable {
/** flag to enable queued scheduling */
private boolean allowQueuedScheduling;
+ /** The mode in which the job is scheduled */
private ScheduleMode scheduleMode = ScheduleMode.FROM_SOURCES;
- private boolean checkpointingEnabled = false;
+ /** The settings for asynchronous snapshotting */
+ private JobSnapshottingSettings snapshotSettings;
- private long checkpointingInterval = 10000;
// --------------------------------------------------------------------------------------------
@@ -258,20 +268,24 @@ public class JobGraph implements Serializable {
return this.taskVertices.size();
}
- public void setCheckpointingEnabled(boolean checkpointingEnabled) {
- this.checkpointingEnabled = checkpointingEnabled;
- }
-
- public boolean isCheckpointingEnabled() {
- return checkpointingEnabled;
- }
-
- public void setCheckpointingInterval(long checkpointingInterval) {
- this.checkpointingInterval = checkpointingInterval;
+ /**
+ * Sets the settings for asynchronous snapshots. A value of {@code null} means that
+ * snapshotting is not enabled.
+ *
+ * @param settings The snapshot settings, or null, to disable snapshotting.
+ */
+ public void setSnapshotSettings(JobSnapshottingSettings settings) {
+ this.snapshotSettings = settings;
}
- public long getCheckpointingInterval() {
- return checkpointingInterval;
+ /**
+ * Gets the settings for asynchronous snapshots. This method returns null, when
+ * snapshotting is not enabled.
+ *
+ * @return The snapshot settings, or null, if snapshotting is not enabled.
+ */
+ public JobSnapshottingSettings getSnapshotSettings() {
+ return snapshotSettings;
}
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
index 445c842..1cf5db2 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/AbstractInvokable.java
@@ -40,7 +40,7 @@ public abstract class AbstractInvokable {
/** The environment assigned to this invokable. */
- private volatile Environment environment;
+ private Environment environment;
/** The execution config, cached from the deserialization from the JobConfiguration */
private ExecutionConfig executionConfig;
@@ -66,14 +66,14 @@ public abstract class AbstractInvokable {
* @param environment
* the environment of this task
*/
- public final void setEnvironment(final Environment environment) {
+ public final void setEnvironment(Environment environment) {
this.environment = environment;
}
/**
* Returns the environment of this task.
*
- * @return the environment of this task or <code>null</code> if the environment has not yet been set
+ * @return The environment of this task.
*/
public Environment getEnvironment() {
return this.environment;
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java
new file mode 100644
index 0000000..69cb1f8
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointCommittingOperator.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph.tasks;
+
+public interface CheckpointCommittingOperator {
+
+ void confirmCheckpoint(long checkpointId, long timestamp);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java
new file mode 100644
index 0000000..d07b07e
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/CheckpointedOperator.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph.tasks;
+
+public interface CheckpointedOperator {
+
+ void triggerCheckpoint(long checkpointId, long timestamp);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettings.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettings.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettings.java
new file mode 100644
index 0000000..86c9b60
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/JobSnapshottingSettings.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph.tasks;
+
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+
+import java.util.List;
+
+/**
+ * The JobSnapshottingSettings are attached to a JobGraph and describe the settings
+ * for the asynchronous snapshotting of the JobGraph, such as interval, and which vertices
+ * need to participate.
+ */
+public class JobSnapshottingSettings implements java.io.Serializable{
+
+ private static final long serialVersionUID = -2593319571078198180L;
+
+ /** The default time in which pending checkpoints need to be acknowledged before timing out */
+ public static final long DEFAULT_SNAPSHOT_TIMEOUT = 10 * 60 * 1000; // 10 minutes
+
+ private final List<JobVertexID> verticesToTrigger;
+
+ private final List<JobVertexID> verticesToAcknowledge;
+
+ private final List<JobVertexID> verticesToConfirm;
+
+ private final long checkpointInterval;
+
+ private final long checkpointTimeout;
+
+
+ public JobSnapshottingSettings(List<JobVertexID> verticesToTrigger,
+ List<JobVertexID> verticesToAcknowledge,
+ List<JobVertexID> verticesToConfirm,
+ long checkpointInterval)
+ {
+ this(verticesToTrigger, verticesToAcknowledge, verticesToConfirm, checkpointInterval, DEFAULT_SNAPSHOT_TIMEOUT);
+ }
+
+ public JobSnapshottingSettings(List<JobVertexID> verticesToTrigger,
+ List<JobVertexID> verticesToAcknowledge,
+ List<JobVertexID> verticesToConfirm,
+ long checkpointInterval, long checkpointTimeout)
+ {
+ this.verticesToTrigger = verticesToTrigger;
+ this.verticesToAcknowledge = verticesToAcknowledge;
+ this.verticesToConfirm = verticesToConfirm;
+ this.checkpointInterval = checkpointInterval;
+ this.checkpointTimeout = checkpointTimeout;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ public List<JobVertexID> getVerticesToTrigger() {
+ return verticesToTrigger;
+ }
+
+ public List<JobVertexID> getVerticesToAcknowledge() {
+ return verticesToAcknowledge;
+ }
+
+ public List<JobVertexID> getVerticesToConfirm() {
+ return verticesToConfirm;
+ }
+
+ public long getCheckpointInterval() {
+ return checkpointInterval;
+ }
+
+ public long getCheckpointTimeout() {
+ return checkpointTimeout;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public String toString() {
+ return String.format("SnapshotSettings: interval=%d, timeout=%d, trigger=%s, ack=%s, commit=%s",
+ checkpointInterval, checkpointTimeout, verticesToTrigger, verticesToAcknowledge, verticesToConfirm);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
index 670dc3f..576edb6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/tasks/OperatorStateCarrier.java
@@ -21,12 +21,18 @@ package org.apache.flink.runtime.jobgraph.tasks;
import org.apache.flink.runtime.state.StateHandle;
/**
- * This is an interface meant to be implemented by any invokable that has to support state recovery.
- * It is mainly used by the TaskManager to identify operators that support state recovery in order
- * to inject their initial state upon creation.
+ * This interface must be implemented by any invokable that has recoverable state.
+ * The method {@link #setInitialState(org.apache.flink.runtime.state.StateHandle)} is used
+ * to set the initial state of the operator, upon recovery.
*/
-public interface OperatorStateCarrier {
+public interface OperatorStateCarrier<T extends StateHandle<?>> {
- public void injectState(StateHandle stateHandle);
+ /**
+ * Sets the initial state of the operator, upon recovery. The initial state is typically
+ * a snapshot of the state from a previous execution.
+ *
+ * @param stateHandle The handle to the state.
+ */
+ public void setInitialState(T stateHandle);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AbortCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AbortCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AbortCheckpoint.java
deleted file mode 100644
index 0493ba6..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AbortCheckpoint.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.messages.checkpoint;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-
-/**
- * This message is sent from the {@link org.apache.flink.runtime.jobmanager.JobManager} to the
- * {@link org.apache.flink.runtime.taskmanager.TaskManager} to tell a task that the checkpoint
- * has been confirmed and that the task can commit the checkpoint to the outside world.
- */
-public class AbortCheckpoint extends AbstractCheckpointMessage implements java.io.Serializable {
-
- private static final long serialVersionUID = 2094094662279578953L;
-
- public AbortCheckpoint(JobID job, ExecutionAttemptID taskExecutionId, long checkpointId) {
- super(job, taskExecutionId, checkpointId);
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public boolean equals(Object o) {
- return this == o || ( (o instanceof AbortCheckpoint) && super.equals(o));
- }
-
- @Override
- public String toString() {
- return String.format("AbortCheckpoint %d for (%s/%s)",
- getCheckpointId(), getJob(), getTaskExecutionId());
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AcknowledgeCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AcknowledgeCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AcknowledgeCheckpoint.java
index dd94e37..db12e0a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AcknowledgeCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/AcknowledgeCheckpoint.java
@@ -21,6 +21,7 @@ package org.apache.flink.runtime.messages.checkpoint;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.state.StateHandle;
+import org.apache.flink.runtime.util.SerializedValue;
/**
* This message is sent from the {@link org.apache.flink.runtime.taskmanager.TaskManager} to the
@@ -33,18 +34,19 @@ public class AcknowledgeCheckpoint extends AbstractCheckpointMessage implements
private static final long serialVersionUID = -7606214777192401493L;
- private final StateHandle state;
+ private final SerializedValue<StateHandle<?>> state;
public AcknowledgeCheckpoint(JobID job, ExecutionAttemptID taskExecutionId, long checkpointId) {
this(job, taskExecutionId, checkpointId, null);
}
- public AcknowledgeCheckpoint(JobID job, ExecutionAttemptID taskExecutionId, long checkpointId, StateHandle state) {
+ public AcknowledgeCheckpoint(JobID job, ExecutionAttemptID taskExecutionId, long checkpointId,
+ SerializedValue<StateHandle<?>> state) {
super(job, taskExecutionId, checkpointId);
this.state = state;
}
- public StateHandle getState() {
+ public SerializedValue<StateHandle<?>> getState() {
return state;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/ConfirmCheckpoint.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/ConfirmCheckpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/ConfirmCheckpoint.java
index cdfd202..d3a4374 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/ConfirmCheckpoint.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/messages/checkpoint/ConfirmCheckpoint.java
@@ -30,15 +30,39 @@ public class ConfirmCheckpoint extends AbstractCheckpointMessage implements java
private static final long serialVersionUID = 2094094662279578953L;
- public ConfirmCheckpoint(JobID job, ExecutionAttemptID taskExecutionId, long checkpointId) {
+ /** The timestamp associated with the checkpoint */
+ private final long timestamp;
+
+ public ConfirmCheckpoint(JobID job, ExecutionAttemptID taskExecutionId, long checkpointId, long timestamp) {
super(job, taskExecutionId, checkpointId);
+ this.timestamp = timestamp;
}
// --------------------------------------------------------------------------------------------
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ // --------------------------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() + (int) (timestamp ^ (timestamp >>> 32));
+ }
+
@Override
public boolean equals(Object o) {
- return this == o || ( (o instanceof ConfirmCheckpoint) && super.equals(o));
+ if (this == o) {
+ return true;
+ }
+ else if (o instanceof ConfirmCheckpoint) {
+ ConfirmCheckpoint that = (ConfirmCheckpoint) o;
+ return this.timestamp == that.timestamp && super.equals(o);
+ }
+ else {
+ return false;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
index 98712b5..f47b054 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/LocalStateHandle.java
@@ -18,36 +18,23 @@
package org.apache.flink.runtime.state;
-import org.apache.flink.util.InstantiationUtil;
-
-import java.io.IOException;
import java.util.Map;
/**
- * A StateHandle that includes a copy of the state itself. This state handle is recommended for
- * cases where the operatorState is lightweight enough to pass throughout the network.
- *
- * State is kept in a byte[] because it may contain userclasses, which akka is not able to handle.
+ * A StateHandle that includes a map of operator states directly.
*/
-public class LocalStateHandle implements StateHandle{
+public class LocalStateHandle implements StateHandle<Map<String, OperatorState<?>>> {
+
+ private static final long serialVersionUID = 2093619217898039610L;
- transient private Map<String, OperatorState<?>> stateMap;
- private final byte[] state;
+ private final Map<String, OperatorState<?>> stateMap;
- public LocalStateHandle(Map<String,OperatorState<?>> state) throws IOException {
+ public LocalStateHandle(Map<String,OperatorState<?>> state) {
this.stateMap = state;
- this.state = InstantiationUtil.serializeObject(state);
}
@Override
- public Map<String,OperatorState<?>> getState(ClassLoader usercodeClassloader) {
- if(stateMap == null) {
- try {
- stateMap = (Map<String, OperatorState<?>>) InstantiationUtil.deserializeObject(this.state, usercodeClassloader);
- } catch (Exception e) {
- throw new RuntimeException("Error while deserializing the state", e);
- }
- }
+ public Map<String,OperatorState<?>> getState() {
return stateMap;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java
index 1852ce8..409383c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateHandle.java
@@ -19,20 +19,19 @@
package org.apache.flink.runtime.state;
import java.io.Serializable;
-import java.util.Map;
/**
* StateHandle is a general handle interface meant to abstract operator state fetching.
* A StateHandle implementation can for example include the state itself in cases where the state
* is lightweight or fetching it lazily from some external storage when the state is too large.
*/
-public interface StateHandle extends Serializable {
+public interface StateHandle<T> extends Serializable {
/**
- * getState should retrieve and return the state managed the handle.
+ * This retrieves and return the state represented by the handle.
*
- * @return
+ * @return The state represented by the handle.
+ * @throws java.lang.Exception Thrown, if the state cannot be fetched.
*/
- public Map<String,OperatorState<?>> getState(ClassLoader userClassloader);
-
+ T getState() throws Exception;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/9b7f8aa1/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java
new file mode 100644
index 0000000..2cdfef3
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/state/StateUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.state;
+
+import org.apache.flink.runtime.jobgraph.tasks.OperatorStateCarrier;
+
+/**
+ * A collection of utility methods for dealing with operator state.
+ */
+public class StateUtils {
+
+ /**
+ * Utility method to define a common generic bound to be used for setting a generic state
+ * handle on a generic state carrier.
+ *
+ * This has no impact on runtime, since internally, it performs
+ * unchecked casts. The purpose is merely to allow the use of generic interfaces without resorting
+ * to raw types, by giving the compiler a common type bound.
+ *
+ * @param op The state carrier operator.
+ * @param state The state handle.
+ * @param <T> Type bound for the
+ */
+ public static <T extends StateHandle<?>> void setOperatorState(OperatorStateCarrier<?> op, StateHandle<?> state) {
+ @SuppressWarnings("unchecked")
+ OperatorStateCarrier<T> typedOp = (OperatorStateCarrier<T>) op;
+ @SuppressWarnings("unchecked")
+ T typedHandle = (T) state;
+
+ typedOp.setInitialState(typedHandle);
+ }
+
+
+ // ------------------------------------------------------------------------
+
+ /** Do not instantiate */
+ private StateUtils() {}
+}
[10/10] flink git commit: [tests] Add test that checkes canceling
against a failed TaskManager process
Posted by se...@apache.org.
[tests] Add test that checkes canceling against a failed TaskManager process
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4bd35259
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4bd35259
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4bd35259
Branch: refs/heads/master
Commit: 4bd352599d0b4ff97ae8cf7eebc528dbcbd33dff
Parents: 959d227
Author: Stephan Ewen <se...@apache.org>
Authored: Tue May 12 16:45:05 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:58 2015 +0200
----------------------------------------------------------------------
.../recovery/ProcessFailureCancelingITCase.java | 289 +++++++++++++++++++
1 file changed, 289 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4bd35259/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
new file mode 100644
index 0000000..5482fc5
--- /dev/null
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.test.recovery;
+
+import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.pattern.Patterns;
+import akka.util.Timeout;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.client.program.ProgramInvocationException;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
+import org.apache.flink.runtime.client.JobStatusMessage;
+import org.apache.flink.runtime.jobmanager.JobManager;
+import org.apache.flink.runtime.messages.JobManagerMessages;
+import org.apache.flink.runtime.net.NetUtils;
+import org.apache.flink.runtime.testutils.CommonTestUtils;
+
+import org.junit.Test;
+
+import scala.Some;
+import scala.Tuple2;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.flink.runtime.testutils.CommonTestUtils.getCurrentClasspath;
+import static org.apache.flink.runtime.testutils.CommonTestUtils.getJavaCommandPath;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * This test makes sure that jobs are canceled properly in cases where
+ * the task manager went down and did not respond to cancel messages.
+ */
+@SuppressWarnings("serial")
+public class ProcessFailureCancelingITCase {
+
+ @Test
+ public void testCancelingOnProcessFailure() {
+ final StringWriter processOutput = new StringWriter();
+
+ ActorSystem jmActorSystem = null;
+ Process taskManagerProcess = null;
+
+ try {
+ // check that we run this test only if the java command
+ // is available on this machine
+ String javaCommand = getJavaCommandPath();
+ if (javaCommand == null) {
+ System.out.println("---- Skipping Process Failure test : Could not find java executable ----");
+ return;
+ }
+
+ // create a logging file for the process
+ File tempLogFile = File.createTempFile(getClass().getSimpleName() + "-", "-log4j.properties");
+ tempLogFile.deleteOnExit();
+ CommonTestUtils.printLog4jDebugConfig(tempLogFile);
+
+ // find a free port to start the JobManager
+ final int jobManagerPort = NetUtils.getAvailablePort();
+
+ // start a JobManager
+ Tuple2<String, Object> localAddress = new Tuple2<String, Object>("localhost", jobManagerPort);
+
+ Configuration jmConfig = new Configuration();
+ jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "5 s");
+ jmConfig.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2000 s");
+ jmConfig.setInteger(ConfigConstants.AKKA_WATCH_THRESHOLD, 10);
+ jmConfig.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "10 s");
+
+ jmActorSystem = AkkaUtils.createActorSystem(jmConfig, new Some<Tuple2<String, Object>>(localAddress));
+ ActorRef jmActor = JobManager.startJobManagerActors(jmConfig, jmActorSystem)._1();
+
+ // the TaskManager java command
+ String[] command = new String[] {
+ javaCommand,
+ "-Dlog.level=DEBUG",
+ "-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath(),
+ "-Xms80m", "-Xmx80m",
+ "-classpath", getCurrentClasspath(),
+ AbstractProcessFailureRecoveryTest.TaskManagerProcessEntryPoint.class.getName(),
+ String.valueOf(jobManagerPort)
+ };
+
+ // start the first two TaskManager processes
+ taskManagerProcess = new ProcessBuilder(command).start();
+ new AbstractProcessFailureRecoveryTest.PipeForwarder(taskManagerProcess.getErrorStream(), processOutput);
+
+ // we wait for the JobManager to have the two TaskManagers available
+ // wait for at most 30 seconds
+ waitUntilNumTaskManagersAreRegistered(jmActor, 1, 30000);
+
+ final Throwable[] errorRef = new Throwable[1];
+
+ // start the test program, which infinitely blocks
+ Runnable programRunner = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment("localhost", jobManagerPort);
+ env.setParallelism(2);
+ env.setNumberOfExecutionRetries(0);
+ env.getConfig().disableSysoutLogging();
+
+ env.generateSequence(0, Long.MAX_VALUE)
+
+ .map(new MapFunction<Long, Long>() {
+
+ @Override
+ public Long map(Long value) throws Exception {
+ synchronized (this) {
+ wait();
+ }
+ return 0L;
+ }
+ })
+
+ .output(new DiscardingOutputFormat<Long>());
+
+ env.execute();
+ }
+ catch (Throwable t) {
+ errorRef[0] = t;
+ }
+ }
+ };
+
+ Thread programThread = new Thread(programRunner);
+
+ // kill the TaskManager
+ taskManagerProcess.destroy();
+ taskManagerProcess = null;
+
+ // immediately submit the job. this should hit the case
+ // where the JobManager still thinks it has the TaskManager and tries to send it tasks
+ programThread.start();
+
+ // try to cancel the job
+ cancelRunningJob(jmActor);
+
+ // we should see a failure within reasonable time (10s is the ask timeout).
+ // since the CI environment is often slow, we conservatively give it up to 2 minutes,
+ // to fail, which is much lower than the failure time given by the heartbeats ( > 2000s)
+
+ programThread.join(120000);
+
+ assertFalse("The program did not cancel in time (2 minutes)", programThread.isAlive());
+
+ Throwable error = errorRef[0];
+ assertNotNull("The program did not fail properly", error);
+
+ assertTrue(error instanceof ProgramInvocationException);
+ // all seems well :-)
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ printProcessLog("TaskManager", processOutput.toString());
+ fail(e.getMessage());
+ }
+ catch (Error e) {
+ e.printStackTrace();
+ printProcessLog("TaskManager 1", processOutput.toString());
+ throw e;
+ }
+ finally {
+ if (taskManagerProcess != null) {
+ taskManagerProcess.destroy();
+ }
+ if (jmActorSystem != null) {
+ jmActorSystem.shutdown();
+ }
+ }
+ }
+
+ private void cancelRunningJob(ActorRef jobManager) throws Exception {
+ final FiniteDuration askTimeout = new FiniteDuration(10, TimeUnit.SECONDS);
+
+ // try at most for 30 seconds
+ final long deadline = System.currentTimeMillis() + 30000;
+
+ JobID jobId = null;
+
+ do {
+ Future<Object> response = Patterns.ask(jobManager,
+ JobManagerMessages.getRequestRunningJobsStatus(), new Timeout(askTimeout));
+
+ Object result;
+ try {
+ result = Await.result(response, askTimeout);
+ }
+ catch (Exception e) {
+ throw new Exception("Could not retrieve running jobs from the JobManager.", e);
+ }
+
+ if (result instanceof JobManagerMessages.RunningJobsStatus) {
+
+ List<JobStatusMessage> jobs = ((JobManagerMessages.RunningJobsStatus) result).getStatusMessages();
+
+ if (jobs.size() == 1) {
+ jobId = jobs.get(0).getJobId();
+ break;
+ }
+ }
+ }
+ while (System.currentTimeMillis() < deadline);
+
+ if (jobId == null) {
+ // we never found it running, must have failed already
+ return;
+ }
+
+ // tell the JobManager to cancel the job
+ jobManager.tell(new JobManagerMessages.CancelJob(jobId), ActorRef.noSender());
+ }
+
+ private void waitUntilNumTaskManagersAreRegistered(ActorRef jobManager, int numExpected, long maxDelay)
+ throws Exception
+ {
+ final long deadline = System.currentTimeMillis() + maxDelay;
+ while (true) {
+ long remaining = deadline - System.currentTimeMillis();
+ if (remaining <= 0) {
+ fail("The TaskManagers did not register within the expected time (" + maxDelay + "msecs)");
+ }
+
+ FiniteDuration timeout = new FiniteDuration(remaining, TimeUnit.MILLISECONDS);
+
+ try {
+ Future<?> result = Patterns.ask(jobManager,
+ JobManagerMessages.getRequestNumberRegisteredTaskManager(),
+ new Timeout(timeout));
+ Integer numTMs = (Integer) Await.result(result, timeout);
+ if (numTMs == numExpected) {
+ break;
+ }
+ }
+ catch (TimeoutException e) {
+ // ignore and retry
+ }
+ catch (ClassCastException e) {
+ fail("Wrong response: " + e.getMessage());
+ }
+ }
+ }
+
+ private void printProcessLog(String processName, String log) {
+ if (log == null || log.length() == 0) {
+ return;
+ }
+
+ System.out.println("-----------------------------------------");
+ System.out.println(" BEGIN SPAWNED PROCESS LOG FOR " + processName);
+ System.out.println("-----------------------------------------");
+ System.out.println(log);
+ System.out.println("-----------------------------------------");
+ System.out.println(" END SPAWNED PROCESS LOG");
+ System.out.println("-----------------------------------------");
+ }
+}
[07/10] flink git commit: [FLINK-1935] Reimplement
PersistentKafkaSource using high level Kafka API
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaSinglePartitionIterator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaSinglePartitionIterator.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaSinglePartitionIterator.java
deleted file mode 100644
index bf3b3f4..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/iterator/KafkaSinglePartitionIterator.java
+++ /dev/null
@@ -1,392 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.iterator;
-
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedChannelException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import kafka.consumer.ConsumerConfig;
-import org.apache.flink.streaming.connectors.kafka.api.simple.KafkaTopicUtils;
-import org.apache.flink.streaming.connectors.kafka.api.simple.MessageWithMetadata;
-import org.apache.flink.streaming.connectors.kafka.api.simple.PersistentKafkaSource;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.BeginningOffset;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.CurrentOffset;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.KafkaOffset;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.api.FetchRequest;
-import kafka.api.FetchRequestBuilder;
-import kafka.cluster.Broker;
-import kafka.common.ErrorMapping;
-import kafka.common.NotLeaderForPartitionException;
-import kafka.javaapi.FetchResponse;
-import kafka.javaapi.PartitionMetadata;
-import kafka.javaapi.TopicMetadata;
-import kafka.javaapi.TopicMetadataRequest;
-import kafka.javaapi.consumer.SimpleConsumer;
-import kafka.message.MessageAndOffset;
-
-/**
- * Iterates the records received from a partition of a Kafka topic as byte arrays.
- *
- * This code is in parts based on https://gist.github.com/ashrithr/5811266.
- */
-public class KafkaSinglePartitionIterator implements KafkaConsumerIterator, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaSinglePartitionIterator.class);
-
- private List<String> hosts;
- private String topic;
- private int partition;
- private long readOffset;
- private transient SimpleConsumer consumer;
- private List<String> replicaBrokers;
- private String clientName;
- private Broker leadBroker;
- private final ConsumerConfig consumerConfig;
-
- private KafkaOffset initialOffset;
- private transient Iterator<MessageAndOffset> iter;
- private transient FetchResponse fetchResponse;
-
- /**
- * Constructor with configurable wait time on empty fetch. For connecting to the Kafka service
- * we use the so called simple or low level Kafka API thus directly connecting to one of the brokers.
- *
- * @param topic
- * Name of the topic to listen to
- * @param partition
- * Partition in the chosen topic
- * @param initialOffset
- * Offset to start consuming at
- * @param kafkaTopicUtils
- * Util for receiving topic metadata
- */
- public KafkaSinglePartitionIterator(String topic, int partition, KafkaOffset initialOffset,
- KafkaTopicUtils kafkaTopicUtils, ConsumerConfig consumerConfig) {
-
- Set<String> brokerAddresses = kafkaTopicUtils.getBrokerAddresses(topic, partition);
- this.hosts = new ArrayList<String>(brokerAddresses);
-
- this.consumerConfig = consumerConfig;
- this.topic = topic;
- this.partition = partition;
-
- this.initialOffset = initialOffset;
-
- this.replicaBrokers = new ArrayList<String>();
- }
-
- // --------------------------------------------------------------------------------------------
- // Initializing a connection
- // --------------------------------------------------------------------------------------------
-
- /**
- * Initializes the connection by detecting the leading broker of
- * the topic and establishing a connection to it.
- */
- public void initialize() {
- if (LOG.isInfoEnabled()) {
- LOG.info("Initializing consumer {} / {} with hosts {}", topic, partition, hosts);
- }
-
- PartitionMetadata metadata = getPartitionMetadata();
-
- leadBroker = metadata.leader();
- clientName = "Client_" + topic + "_" + partition;
-
- consumer = new SimpleConsumer(leadBroker.host(), leadBroker.port(), consumerConfig.socketTimeoutMs(), consumerConfig.socketReceiveBufferBytes(), clientName);
-
- try {
- readOffset = initialOffset.getOffset(consumer, topic, partition, clientName);
- } catch (NotLeaderForPartitionException e) {
- throw new RuntimeException("Unable to get offset",e);
- }
-
- try {
- resetFetchResponse(readOffset);
- } catch (ClosedChannelException e) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Got ClosedChannelException, trying to find new leader.");
- }
- findNewLeader();
- }
- }
-
- private PartitionMetadata getPartitionMetadata() {
- PartitionMetadata metadata;
- int retry = 0;
- int waitTime = consumerConfig.props().getInt(PersistentKafkaSource.WAIT_ON_FAILED_LEADER_MS_KEY, PersistentKafkaSource.WAIT_ON_FAILED_LEADER__MS_DEFAULT);
- do {
- metadata = findLeader(hosts, topic, partition);
- /*try {
- Thread.sleep(10000);
- } catch (InterruptedException e) {
- throw new RuntimeException("Establishing connection to Kafka failed", e);
- } */
- if(metadata == null) {
- retry++;
- if(retry == consumerConfig.props().getInt(PersistentKafkaSource.MAX_FAILED_LEADER_RETRIES_KEY, PersistentKafkaSource.MAX_FAILED_LEADER_RETRIES_DEFAULT)) {
- throw new RuntimeException("Tried finding a leader "+retry+" times without success");
- }
- LOG.warn("Unable to get leader and partition metadata. Waiting {} ms until retrying. Retries so far {}",waitTime, retry-1);
- try {
- Thread.sleep(waitTime);
- } catch (InterruptedException e) {
- throw new RuntimeException("Establishing connection to Kafka failed", e);
- }
- }
- } while (metadata == null);
-
- if (metadata.leader() == null) {
- throw new RuntimeException("Can't find Leader for Topic and Partition. (at " + hosts + ")");
- }
-
- return metadata;
- }
-
- /**
- * Sets the partition to read from.
- *
- * @param partition
- * partition number
- */
- public void setPartition(int partition) {
- this.partition = partition;
- }
-
- // --------------------------------------------------------------------------------------------
- // Iterator methods
- // --------------------------------------------------------------------------------------------
-
- /**
- * Convenience method to emulate iterator behaviour.
- *
- * @return whether the iterator has a next element
- */
- public boolean hasNext() {
- return true;
- }
-
- /**
- * Returns the next message received from Kafka as a
- * byte array.
- *
- * @return next message as a byte array.
- */
- public byte[] next() {
- return nextWithOffset().getMessage();
- }
-
- public boolean fetchHasNext() {
- synchronized (fetchResponse) {
- if (!iter.hasNext()) {
- try {
- resetFetchResponse(readOffset);
- } catch (ClosedChannelException e) {
- if (LOG.isWarnEnabled()) {
- LOG.warn("Got ClosedChannelException, trying to find new leader.", e);
- }
- findNewLeader();
- }
- return iter.hasNext();
- } else {
- return true;
- }
- }
- }
-
- /**
- * Returns the next message and its offset received from
- * Kafka encapsulated in a POJO.
- *
- * @return next message and its offset.
- */
- public MessageWithMetadata nextWithOffset() {
-
- synchronized (fetchResponse) {
- if (!iter.hasNext()) {
- throw new RuntimeException(
- "Trying to read when response is not fetched. Call fetchHasNext() first.");
- }
-
- MessageAndOffset messageAndOffset = iter.next();
- long currentOffset = messageAndOffset.offset();
-
- while (currentOffset < readOffset) {
- messageAndOffset = iter.next();
- currentOffset = messageAndOffset.offset();
- }
-
- readOffset = messageAndOffset.nextOffset();
- ByteBuffer payload = messageAndOffset.message().payload();
-
- byte[] bytes = new byte[payload.limit()];
- payload.get(bytes);
-
- return new MessageWithMetadata(messageAndOffset.offset(), bytes, partition);
- }
- }
-
- // --------------------------------------------------------------------------------------------
- // Internal utilities
- // --------------------------------------------------------------------------------------------
-
- private void resetFetchResponse(long offset) throws ClosedChannelException {
- FetchRequest req = new FetchRequestBuilder().clientId(clientName)
- .addFetch(topic, partition, offset, consumerConfig.fetchMessageMaxBytes()).build();
-
- fetchResponse = consumer.fetch(req);
-
- if (fetchResponse.hasError()) {
- short code = fetchResponse.errorCode(topic, partition);
-
- if (LOG.isErrorEnabled()) {
- LOG.error("Error fetching data from the Broker:" + leadBroker + " Reason: " + code);
- }
-
- if (code == ErrorMapping.OffsetOutOfRangeCode()) {
- if (LOG.isErrorEnabled()) {
- LOG.error("Asked for invalid offset {}", offset);
- }
- String reset = consumerConfig.autoOffsetReset();
- if(reset.equals("smallest")) {
- LOG.info("Setting read offset to beginning (smallest)");
- readOffset = new BeginningOffset().getOffset(consumer, topic, partition, clientName);
- } else if(reset.equals("largest")) {
- LOG.info("Setting read offset to current offset (largest)");
- readOffset = new CurrentOffset().getOffset(consumer, topic, partition, clientName);
- } else {
- throw new RuntimeException("Unknown 'autooffset.reset' mode '"+reset+"' Supported values are 'smallest' and 'largest'.");
- }
- }
-
- findNewLeader();
- }
-
- iter = fetchResponse.messageSet(topic, partition).iterator();
- }
-
- private void findNewLeader() {
- consumer.close();
- consumer = null;
- leadBroker = findNewLeader(leadBroker, topic, partition);
- consumer = new SimpleConsumer(leadBroker.host(), leadBroker.port(), consumerConfig.socketTimeoutMs(), consumerConfig.socketReceiveBufferBytes(), clientName);
- }
-
- private PartitionMetadata findLeader(List<String> addresses, String topic, int partition) {
-
- PartitionMetadata returnMetaData = null;
- loop:
- for (String address : addresses) {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Trying to find leader via broker: {}", address);
- }
-
- String[] split = address.split(":");
- String host = split[0];
- int port = Integer.parseInt(split[1]);
-
- SimpleConsumer consumer = null;
- try {
- consumer = new SimpleConsumer(host, port, consumerConfig.socketTimeoutMs(), consumerConfig.socketReceiveBufferBytes(), "leaderLookup");
- List<String> topics = Collections.singletonList(topic);
-
- TopicMetadataRequest req = new TopicMetadataRequest(topics);
-
- kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);
-
- List<TopicMetadata> metaData = resp.topicsMetadata();
- for (TopicMetadata item : metaData) {
- for (PartitionMetadata part : item.partitionsMetadata()) {
- if (part.partitionId() == partition) {
- returnMetaData = part;
- break loop;
- }
- }
- }
- } catch (Exception e) {
- if (e instanceof ClosedChannelException) {
- LOG.warn("Got ClosedChannelException while trying to communicate with Broker" +
- "[{}] to find Leader for [{}, {}]. Trying other replicas.", address, topic, partition);
- } else {
- throw new RuntimeException("Error communicating with Broker [" + address + "] to find Leader for [" + topic + ", " + partition + "]", e);
- }
- } finally {
- if (consumer != null) {
- consumer.close();
- }
- }
- }
- if (returnMetaData != null) {
- replicaBrokers.clear();
- for (kafka.cluster.Broker replica : returnMetaData.replicas()) {
- replicaBrokers.add(replica.host() + ":" + replica.port());
- }
- }
- return returnMetaData;
- }
-
- private Broker findNewLeader(Broker oldLeader, String topic, int a_partition) {
- for (int i = 0; i < 3; i++) {
- if (LOG.isInfoEnabled()) {
- LOG.info("Trying to find a new leader after Broker failure.");
- }
- boolean goToSleep = false;
- PartitionMetadata metadata = findLeader(replicaBrokers, topic, a_partition);
- if (metadata == null) {
- goToSleep = true;
- } else if (metadata.leader() == null) {
- goToSleep = true;
- } else if (oldLeader.host().equalsIgnoreCase(metadata.leader().host()) && i == 0) {
- // first time through if the leader hasn't changed give ZooKeeper a second to recover
- // second time, assume the broker did recover before failover, or it was a non-Broker issue
- //
- goToSleep = true;
- } else {
- return metadata.leader();
- }
- if (goToSleep) {
- try {
- Thread.sleep(10000);
- } catch (InterruptedException ie) {
- }
- }
- }
- throw new RuntimeException("Unable to find new leader after Broker failure.");
- }
-
- public int getId() {
- return this.partition;
- }
-
- @Override
- public String toString() {
- return "SinglePartitionIterator{partition="+partition+" readOffset="+readOffset+"}";
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/BeginningOffset.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/BeginningOffset.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/BeginningOffset.java
deleted file mode 100644
index 15e7b36..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/BeginningOffset.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.offset;
-
-import kafka.api.OffsetRequest;
-import kafka.javaapi.consumer.SimpleConsumer;
-
-public class BeginningOffset extends KafkaOffset {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
- return getLastOffset(consumer, topic, partition, OffsetRequest.EarliestTime(), clientName);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/CurrentOffset.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/CurrentOffset.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/CurrentOffset.java
deleted file mode 100644
index 6119f32..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/CurrentOffset.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.offset;
-
-import kafka.api.OffsetRequest;
-import kafka.javaapi.consumer.SimpleConsumer;
-
-public class CurrentOffset extends KafkaOffset {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public long getOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
- return getLastOffset(consumer, topic, partition, OffsetRequest.LatestTime(), clientName);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/GivenOffset.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/GivenOffset.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/GivenOffset.java
deleted file mode 100644
index 3aec7ff..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/GivenOffset.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.offset;
-
-import kafka.javaapi.consumer.SimpleConsumer;
-
-/**
- * Offset given by a message read from Kafka.
- */
-public class GivenOffset extends KafkaOffset {
-
- private static final long serialVersionUID = 1L;
- private final long offset;
-
- public GivenOffset(long offset) {
- this.offset = offset;
- }
-
- @Override
- public long getOffset(SimpleConsumer consumer, String topic, int partition, String clientName) {
- return offset;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/KafkaOffset.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/KafkaOffset.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/KafkaOffset.java
deleted file mode 100644
index 2eaa2b8..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/KafkaOffset.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.offset;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.api.PartitionOffsetRequestInfo;
-import kafka.common.TopicAndPartition;
-import kafka.javaapi.OffsetResponse;
-import kafka.javaapi.consumer.SimpleConsumer;
-
-/**
- * Superclass for various kinds of KafkaOffsets.
- */
-public abstract class KafkaOffset implements Serializable {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaOffset.class);
-
- private static final long serialVersionUID = 1L;
-
- public abstract long getOffset(SimpleConsumer consumer, String topic, int partition,
- String clientName);
-
- /**
- *
- * @param consumer
- * @param topic
- * @param partition
- * @param whichTime Type of offset request (latest time / earliest time)
- * @param clientName
- * @return
- */
- protected long getLastOffset(SimpleConsumer consumer, String topic, int partition,
- long whichTime, String clientName) {
- TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
- Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
- requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
-
- kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
- kafka.api.OffsetRequest.CurrentVersion(), clientName);
- OffsetResponse response = consumer.getOffsetsBefore(request);
-
- while (response.hasError()) {
- int errorCode = response.errorCode(topic, partition);
- LOG.warn("Response has error. Error code "+errorCode);
- switch (errorCode) {
- case 6:
- case 3:
- LOG.warn("Kafka broker trying to fetch from a non-leader broker.");
- break;
- default:
- throw new RuntimeException("Error fetching data from Kafka broker. Error code " + errorCode);
- }
-
- request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
- response = consumer.getOffsetsBefore(request);
- }
-
- long[] offsets = response.offsets(topic, partition);
- if(offsets.length > 1) {
- LOG.warn("The offset response unexpectedly contained more than one offset: "+ Arrays.toString(offsets) + " Using only first one");
- }
- return offsets[0];
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/Offset.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/Offset.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/Offset.java
deleted file mode 100644
index 02c49df..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/main/java/org/apache/flink/streaming/connectors/kafka/api/simple/offset/Offset.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka.api.simple.offset;
-
-/**
- * Enum controlling the offset behavior of the PersistentKafkaSource.
- */
-public enum Offset {
- /**
- * Read the Kafka topology from the beginning
- */
- FROM_BEGINNING,
- /**
- * Read the topology from the current offset. (Default).
- */
- FROM_CURRENT
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
index 87c6a34..246756c 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaITCase.java
@@ -25,14 +25,20 @@ import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
import java.util.Properties;
import java.util.Random;
+import kafka.admin.AdminUtils;
+import kafka.api.PartitionMetadata;
import kafka.consumer.ConsumerConfig;
-import org.apache.commons.lang.SerializationUtils;
+import kafka.network.SocketServer;
+import org.I0Itec.zkclient.ZkClient;
import org.apache.curator.test.TestingServer;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.functions.RichFlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.client.JobExecutionException;
@@ -40,25 +46,25 @@ import org.apache.flink.runtime.net.NetUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.WindowMapFunction;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import org.apache.flink.streaming.api.functions.source.SourceFunction;
+import org.apache.flink.streaming.api.windowing.helper.Count;
import org.apache.flink.streaming.connectors.kafka.api.KafkaSink;
import org.apache.flink.streaming.connectors.kafka.api.KafkaSource;
-import org.apache.flink.streaming.connectors.kafka.api.simple.KafkaTopicUtils;
-import org.apache.flink.streaming.connectors.kafka.api.simple.PersistentKafkaSource;
-import org.apache.flink.streaming.connectors.kafka.api.simple.offset.Offset;
+import org.apache.flink.streaming.connectors.kafka.api.persistent.PersistentKafkaSource;
import org.apache.flink.streaming.connectors.kafka.partitioner.SerializableKafkaPartitioner;
import org.apache.flink.streaming.connectors.kafka.util.KafkaLocalSystemTime;
-import org.apache.flink.streaming.util.serialization.DeserializationSchema;
import org.apache.flink.streaming.util.serialization.JavaDefaultStringSchema;
-import org.apache.flink.streaming.util.serialization.SerializationSchema;
import org.apache.flink.util.Collector;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
@@ -66,6 +72,7 @@ import org.slf4j.LoggerFactory;
import kafka.server.KafkaConfig;
import kafka.server.KafkaServer;
+import scala.collection.Seq;
/**
* Code in this test is based on the following GitHub repository:
@@ -91,9 +98,15 @@ public class KafkaITCase {
private static TestingServer zookeeper;
private static List<KafkaServer> brokers;
+ private static String brokerConnectionStrings = "";
private static boolean shutdownKafkaBroker;
+ private static ConsumerConfig standardCC;
+
+ private static ZkClient zkClient;
+
+
@BeforeClass
public static void prepare() throws IOException {
LOG.info("Starting KafkaITCase.prepare()");
@@ -118,6 +131,12 @@ public class KafkaITCase {
brokers = new ArrayList<KafkaServer>(NUMBER_OF_KAFKA_SERVERS);
for (int i = 0; i < NUMBER_OF_KAFKA_SERVERS; i++) {
brokers.add(getKafkaServer(i, tmpKafkaDirs.get(i)));
+ SocketServer socketServer = brokers.get(i).socketServer();
+ String host = "localhost";
+ if(socketServer.host() != null) {
+ host = socketServer.host();
+ }
+ brokerConnectionStrings += host+":"+socketServer.port()+",";
}
LOG.info("ZK and KafkaServer started.");
@@ -125,6 +144,17 @@ public class KafkaITCase {
LOG.warn("Test failed with exception", t);
Assert.fail("Test failed with: " + t.getMessage());
}
+
+ Properties cProps = new Properties();
+ cProps.setProperty("zookeeper.connect", zookeeperConnectionString);
+ cProps.setProperty("group.id", "flink-tests");
+ cProps.setProperty("auto.commit.enable", "false");
+
+ cProps.setProperty("auto.offset.reset", "smallest"); // read from the beginning.
+
+ standardCC = new ConsumerConfig(cProps);
+
+ zkClient = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
}
@AfterClass
@@ -142,8 +172,196 @@ public class KafkaITCase {
LOG.warn("ZK.stop() failed", e);
}
}
+ zkClient.close();
+ }
+
+
+ @Test
+ public void testOffsetManipulation() {
+ ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
+
+ final String topicName = "testOffsetManipulation";
+
+ // create topic
+ Properties topicConfig = new Properties();
+ LOG.info("Creating topic {}", topicName);
+ AdminUtils.createTopic(zk, topicName, 3, 2, topicConfig);
+
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 0, 1337);
+
+ Assert.assertEquals(1337L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0));
+
+ zk.close();
+ }
+ /**
+ * We want to use the High level java consumer API but manage the offset in Zookeeper manually.
+ *
+ */
+ @Test
+ public void testPersistentSourceWithOffsetUpdates() throws Exception {
+ LOG.info("Starting testPersistentSourceWithOffsetUpdates()");
+
+ ZkClient zk = new ZkClient(standardCC.zkConnect(), standardCC.zkSessionTimeoutMs(), standardCC.zkConnectionTimeoutMs(), new PersistentKafkaSource.KafkaZKStringSerializer());
+
+ final String topicName = "testOffsetHacking";
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(3);
+ env.getConfig().disableSysoutLogging();
+ env.enableCheckpointing(50);
+ env.setNumberOfExecutionRetries(0);
+
+ // create topic
+ Properties topicConfig = new Properties();
+ LOG.info("Creating topic {}", topicName);
+ AdminUtils.createTopic(zk, topicName, 3, 2, topicConfig);
+
+ // write a sequence from 0 to 99 to each of the three partitions.
+ writeSequence(env, topicName, 0, 99);
+
+ readSequence(env, standardCC, topicName, 0, 100, 300);
+
+ // check offsets
+ Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 0));
+ Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 1));
+ Assert.assertEquals("The offset seems incorrect", 99L, PersistentKafkaSource.getOffset(zk, standardCC.groupId(), topicName, 2));
+
+
+ LOG.info("Manipulating offsets");
+ // set the offset to 25, 50, and 75 for the three partitions
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 0, 50);
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 1, 50);
+ PersistentKafkaSource.setOffset(zk, standardCC.groupId(), topicName, 2, 50);
+
+ // create new env
+ env = StreamExecutionEnvironment.createLocalEnvironment(3);
+ env.getConfig().disableSysoutLogging();
+ readSequence(env, standardCC, topicName, 50, 50, 150);
+
+ zk.close();
+
+ LOG.info("Finished testPersistentSourceWithOffsetUpdates()");
+ }
+
+ private void readSequence(StreamExecutionEnvironment env, ConsumerConfig cc, String topicName, final int valuesStartFrom, final int valuesCount, final int finalCount) throws Exception {
+ LOG.info("Reading sequence for verification until final count {}", finalCount);
+ DataStream<Tuple2<Integer, Integer>> source = env.addSource(
+ new PersistentKafkaSource<Tuple2<Integer, Integer>>(topicName, new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1,1), env.getConfig()), cc)
+ )
+ //add a sleeper mapper. Since there is no good way of "shutting down" a running topology, we have
+ // to play this trick. The problem is that we have to wait until all checkpoints are confirmed
+ .map(new MapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>() {
+ @Override
+ public Tuple2<Integer, Integer> map(Tuple2<Integer, Integer> value) throws Exception {
+ Thread.sleep(75);
+ return value;
+ }
+ }).setParallelism(3);
+
+ // verify data
+ DataStream<Integer> validIndexes = source.flatMap(new RichFlatMapFunction<Tuple2<Integer, Integer>, Integer>() {
+ int[] values = new int[valuesCount];
+ int count = 0;
+
+ @Override
+ public void flatMap(Tuple2<Integer, Integer> value, Collector<Integer> out) throws Exception {
+ values[value.f1 - valuesStartFrom]++;
+ count++;
+
+ LOG.info("Reader " + getRuntimeContext().getIndexOfThisSubtask() + " got " + value + " count=" + count + "/" + finalCount);
+ // verify if we've seen everything
+
+ if (count == finalCount) {
+ LOG.info("Received all values");
+ for (int i = 0; i < values.length; i++) {
+ int v = values[i];
+ if (v != 3) {
+ throw new RuntimeException("Expected v to be 3, but was " + v + " on element " + i + " array=" + Arrays.toString(values));
+ }
+ }
+ // test has passed
+ throw new SuccessException();
+ }
+ }
+
+ }).setParallelism(1);
+
+ tryExecute(env, "Read data from Kafka");
+
+ LOG.info("Successfully read sequence for verification");
+ }
+
+
+
+ private void writeSequence(StreamExecutionEnvironment env, String topicName, final int from, final int to) throws Exception {
+ LOG.info("Writing sequence from {} to {} to topic {}", from, to, topicName);
+ DataStream<Tuple2<Integer, Integer>> stream = env.addSource(new RichParallelSourceFunction<Tuple2<Integer, Integer>>() {
+ private static final long serialVersionUID = 1L;
+ boolean running = true;
+
+ @Override
+ public void run(Collector<Tuple2<Integer, Integer>> collector) throws Exception {
+ LOG.info("Starting source.");
+ int cnt = from;
+ int partition = getRuntimeContext().getIndexOfThisSubtask();
+ while (running) {
+ LOG.info("Writing " + cnt + " to partition " + partition);
+ collector.collect(new Tuple2<Integer, Integer>(getRuntimeContext().getIndexOfThisSubtask(), cnt));
+ if (cnt == to) {
+ LOG.info("Writer reached end.");
+ return;
+ }
+ cnt++;
+ }
+ }
+
+ @Override
+ public void cancel() {
+ LOG.info("Source got cancel()");
+ running = false;
+ }
+ }).setParallelism(3);
+ stream.addSink(new KafkaSink<Tuple2<Integer, Integer>>(brokerConnectionStrings,
+ topicName,
+ new Utils.TypeInformationSerializationSchema<Tuple2<Integer, Integer>>(new Tuple2<Integer, Integer>(1, 1), env.getConfig()),
+ new T2Partitioner()
+ )).setParallelism(3);
+ env.execute("Write sequence from " + from + " to " + to + " to topic " + topicName);
+ LOG.info("Finished writing sequence");
+ }
+
+ private static class T2Partitioner implements SerializableKafkaPartitioner {
+ @Override
+ public int partition(Object key, int numPartitions) {
+ if(numPartitions != 3) {
+ throw new IllegalArgumentException("Expected three partitions");
+ }
+ Tuple2<Integer, Integer> element = (Tuple2<Integer, Integer>) key;
+ return element.f0;
+ }
+ }
+
+ public static void tryExecute(StreamExecutionEnvironment see, String name) throws Exception {
+ try {
+ see.execute(name);
+ } catch (JobExecutionException good) {
+ Throwable t = good.getCause();
+ int limit = 0;
+ while (!(t instanceof SuccessException)) {
+ if(t == null) {
+ LOG.warn("Test failed with exception", good);
+ Assert.fail("Test failed with: " + good.getMessage());
+ }
+
+ t = t.getCause();
+ if (limit++ == 20) {
+ LOG.warn("Test failed with exception", good);
+ Assert.fail("Test failed with: " + good.getMessage());
+ }
+ }
+ }
}
+
@Test
public void regularKafkaSourceTest() throws Exception {
LOG.info("Starting KafkaITCase.regularKafkaSourceTest()");
@@ -152,10 +370,9 @@ public class KafkaITCase {
createTestTopic(topic, 1, 1);
final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
-
// add consuming topology:
DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
- new KafkaSource<Tuple2<Long, String>>(zookeeperConnectionString, topic, "myFlinkGroup", new TupleSerializationSchema(), 5000));
+ new KafkaSource<Tuple2<Long, String>>(zookeeperConnectionString, topic, "myFlinkGroup", new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()), 5000));
consuming.addSink(new SinkFunction<Tuple2<Long, String>>() {
int elCnt = 0;
int start = -1;
@@ -210,22 +427,9 @@ public class KafkaITCase {
running = false;
}
});
- stream.addSink(new KafkaSink<Tuple2<Long, String>>(zookeeperConnectionString, topic, new TupleSerializationSchema()));
+ stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig())));
- try {
- env.setParallelism(1);
- env.execute();
- } catch (JobExecutionException good) {
- Throwable t = good.getCause();
- int limit = 0;
- while (!(t instanceof SuccessException)) {
- t = t.getCause();
- if (limit++ == 20) {
- LOG.warn("Test failed with exception", good);
- Assert.fail("Test failed with: " + good.getMessage());
- }
- }
- }
+ tryExecute(env, "regular kafka source test");
LOG.info("Finished KafkaITCase.regularKafkaSourceTest()");
}
@@ -241,7 +445,10 @@ public class KafkaITCase {
// add consuming topology:
DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
- new PersistentKafkaSource<Tuple2<Long, String>>(zookeeperConnectionString, topic, new TupleSerializationSchema(), 5000, 100, Offset.FROM_BEGINNING));
+ new PersistentKafkaSource<Tuple2<Long, String>>(topic,
+ new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()),
+ standardCC
+ ));
consuming.addSink(new RichSinkFunction<Tuple2<Long, String>>() {
int elCnt = 0;
int start = -1;
@@ -304,22 +511,9 @@ public class KafkaITCase {
running = false;
}
});
- stream.addSink(new KafkaSink<Tuple2<Long, String>>(zookeeperConnectionString, topic, new TupleSerializationSchema()));
+ stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig())));
- try {
- env.setParallelism(1);
- env.execute();
- } catch (JobExecutionException good) {
- Throwable t = good.getCause();
- int limit = 0;
- while (!(t instanceof SuccessException)) {
- t = t.getCause();
- if (limit++ == 20) {
- LOG.warn("Test failed with exception", good);
- Assert.fail("Test failed with: " + good.getMessage());
- }
- }
- }
+ tryExecute(env, "tupletesttopology");
LOG.info("Finished KafkaITCase.tupleTestTopology()");
}
@@ -347,16 +541,19 @@ public class KafkaITCase {
consumerProps.setProperty("fetch.message.max.bytes", Integer.toString(1024 * 1024 * 30));
consumerProps.setProperty("zookeeper.connect", zookeeperConnectionString);
consumerProps.setProperty("group.id", "test");
+ consumerProps.setProperty("auto.commit.enable", "false");
+ consumerProps.setProperty("auto.offset.reset", "smallest");
ConsumerConfig cc = new ConsumerConfig(consumerProps);
DataStreamSource<Tuple2<Long, byte[]>> consuming = env.addSource(
- new PersistentKafkaSource<Tuple2<Long, byte[]>>(topic, serSchema, Offset.FROM_BEGINNING, cc));
+ new PersistentKafkaSource<Tuple2<Long, byte[]>>(topic, serSchema, cc));
consuming.addSink(new SinkFunction<Tuple2<Long, byte[]>>() {
int elCnt = 0;
@Override
public void invoke(Tuple2<Long, byte[]> value) throws Exception {
+ LOG.info("Received {}", value.f0);
elCnt++;
if(value.f0 == -1) {
// we should have seen 11 elements now.
@@ -370,7 +567,7 @@ public class KafkaITCase {
throw new RuntimeException("More than 10 elements seen: "+elCnt);
}
}
- });
+ }).setParallelism(1);
// add producing topology
DataStream<Tuple2<Long, byte[]>> stream = env.addSource(new RichSourceFunction<Tuple2<Long, byte[]>>() {
@@ -398,6 +595,7 @@ public class KafkaITCase {
} catch (InterruptedException ignored) {
}
if(cnt == 10) {
+ LOG.info("Send end signal");
// signal end
collector.collect(new Tuple2<Long, byte[]>(-1L, new byte[]{1}));
running = false;
@@ -412,31 +610,16 @@ public class KafkaITCase {
}
});
- stream.addSink(new KafkaSink<Tuple2<Long, byte[]>>(zookeeperConnectionString, topic,
+ stream.addSink(new KafkaSink<Tuple2<Long, byte[]>>(brokerConnectionStrings, topic,
new Utils.TypeInformationSerializationSchema<Tuple2<Long, byte[]>>(new Tuple2<Long, byte[]>(0L, new byte[]{0}), env.getConfig()))
);
- try {
- env.setParallelism(1);
- env.execute();
- } catch (JobExecutionException good) {
- Throwable t = good.getCause();
- int limit = 0;
- while (!(t instanceof SuccessException)) {
- t = t.getCause();
- if (limit++ == 20) {
- LOG.warn("Test failed with exception", good);
- Assert.fail("Test failed with: " + good.getMessage());
- }
- }
- }
+ tryExecute(env, "big topology test");
LOG.info("Finished KafkaITCase.bigRecordTestTopology()");
}
- private static boolean partitionerHasBeenCalled = false;
-
@Test
public void customPartitioningTestTopology() throws Exception {
LOG.info("Starting KafkaITCase.customPartitioningTestTopology()");
@@ -449,7 +632,9 @@ public class KafkaITCase {
// add consuming topology:
DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(
- new PersistentKafkaSource<Tuple2<Long, String>>(zookeeperConnectionString, topic, new TupleSerializationSchema(), 5000, 100, Offset.FROM_BEGINNING));
+ new PersistentKafkaSource<Tuple2<Long, String>>(topic,
+ new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()),
+ standardCC));
consuming.addSink(new SinkFunction<Tuple2<Long, String>>() {
int start = -1;
BitSet validator = new BitSet(101);
@@ -519,23 +704,9 @@ public class KafkaITCase {
running = false;
}
});
- stream.addSink(new KafkaSink<Tuple2<Long, String>>(zookeeperConnectionString, topic, new TupleSerializationSchema(), new CustomPartitioner()));
-
- try {
- env.setParallelism(1);
- env.execute();
- } catch (JobExecutionException good) {
- Throwable t = good.getCause();
- int limit = 0;
- while (!(t instanceof SuccessException)) {
- t = t.getCause();
- if (limit++ == 20) {
- throw good;
- }
- }
+ stream.addSink(new KafkaSink<Tuple2<Long, String>>(brokerConnectionStrings, topic, new Utils.TypeInformationSerializationSchema<Tuple2<Long, String>>(new Tuple2<Long, String>(1L, ""), env.getConfig()), new CustomPartitioner()));
- assertTrue(partitionerHasBeenCalled);
- }
+ tryExecute(env, "custom partitioning test");
LOG.info("Finished KafkaITCase.customPartitioningTestTopology()");
}
@@ -547,7 +718,6 @@ public class KafkaITCase {
@Override
public int partition(Object key, int numPartitions) {
- partitionerHasBeenCalled = true;
@SuppressWarnings("unchecked")
Tuple2<Long, String> tuple = (Tuple2<Long, String>) key;
@@ -561,25 +731,6 @@ public class KafkaITCase {
}
}
- private static class TupleSerializationSchema implements DeserializationSchema<Tuple2<Long, String>>, SerializationSchema<Tuple2<Long, String>, byte[]> {
-
- @SuppressWarnings("unchecked")
- @Override
- public Tuple2<Long, String> deserialize(byte[] message) {
- Object deserializedObject = SerializationUtils.deserialize(message);
- return (Tuple2<Long, String>) deserializedObject;
- }
-
- @Override
- public byte[] serialize(Tuple2<Long, String> element) {
- return SerializationUtils.serialize(element);
- }
-
- @Override
- public boolean isEndOfStream(Tuple2<Long, String> nextElement) {
- return false;
- }
- }
@Test
public void simpleTestTopology() throws Exception {
@@ -591,7 +742,7 @@ public class KafkaITCase {
// add consuming topology:
DataStreamSource<String> consuming = env.addSource(
- new PersistentKafkaSource<String>(zookeeperConnectionString, topic, new JavaDefaultStringSchema(), 5000, 100, Offset.FROM_BEGINNING));
+ new PersistentKafkaSource<String>(topic, new JavaDefaultStringSchema(), standardCC));
consuming.addSink(new SinkFunction<String>() {
int elCnt = 0;
int start = -1;
@@ -643,34 +794,34 @@ public class KafkaITCase {
running = false;
}
});
- stream.addSink(new KafkaSink<String>(zookeeperConnectionString, topic, new JavaDefaultStringSchema()));
+ stream.addSink(new KafkaSink<String>(brokerConnectionStrings, topic, new JavaDefaultStringSchema()));
- try {
- env.setParallelism(1);
- env.execute();
- } catch (JobExecutionException good) {
- Throwable t = good.getCause();
- int limit = 0;
- while (!(t instanceof SuccessException)) {
- t = t.getCause();
- if (limit++ == 20) {
- LOG.warn("Test failed with exception", good);
- Assert.fail("Test failed with: " + good.getMessage());
- }
- }
- }
+ tryExecute(env, "simpletest");
}
private static boolean leaderHasShutDown = false;
- @Test
+ @Test(timeout=60000)
public void brokerFailureTest() throws Exception {
String topic = "brokerFailureTestTopic";
createTestTopic(topic, 2, 2);
- KafkaTopicUtils kafkaTopicUtils = new KafkaTopicUtils(zookeeperConnectionString);
- final String leaderToShutDown = kafkaTopicUtils.waitAndGetPartitionMetadata(topic, 0).leader().get().connectionString();
+ // KafkaTopicUtils kafkaTopicUtils = new KafkaTopicUtils(zookeeperConnectionString);
+ // final String leaderToShutDown = kafkaTopicUtils.waitAndGetPartitionMetadata(topic, 0).leader().get().connectionString();
+
+ PartitionMetadata firstPart = null;
+ do {
+ if(firstPart != null) {
+ LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
+ // not the first try. Sleep a bit
+ Thread.sleep(150);
+ }
+ Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
+ firstPart = partitionMetadata.head();
+ } while(firstPart.errorCode() != 0);
+
+ final String leaderToShutDown = firstPart.leader().get().connectionString();
final Thread brokerShutdown = new Thread(new Runnable() {
@Override
@@ -704,7 +855,7 @@ public class KafkaITCase {
// add consuming topology:
DataStreamSource<String> consuming = env.addSource(
- new PersistentKafkaSource<String>(zookeeperConnectionString, topic, new JavaDefaultStringSchema(), 5000, 10, Offset.FROM_BEGINNING));
+ new PersistentKafkaSource<String>(topic, new JavaDefaultStringSchema(), standardCC));
consuming.setParallelism(1);
consuming.addSink(new SinkFunction<String>() {
@@ -717,7 +868,7 @@ public class KafkaITCase {
@Override
public void invoke(String value) throws Exception {
- LOG.info("Got message = " + value + " leader has shut down "+leaderHasShutDown+" el cnt = "+elCnt+" to rec"+ numOfMessagesToBeCorrect);
+ LOG.info("Got message = " + value + " leader has shut down " + leaderHasShutDown + " el cnt = " + elCnt + " to rec" + numOfMessagesToBeCorrect);
String[] sp = value.split("-");
int v = Integer.parseInt(sp[1]);
@@ -736,8 +887,8 @@ public class KafkaITCase {
shutdownKafkaBroker = true;
}
- if(leaderHasShutDown) { // it only makes sence to check once the shutdown is completed
- if (elCnt >= stopAfterMessages ) {
+ if (leaderHasShutDown) { // it only makes sence to check once the shutdown is completed
+ if (elCnt >= stopAfterMessages) {
// check if everything in the bitset is set to true
int nc;
if ((nc = validator.nextClearBit(0)) < numOfMessagesToBeCorrect) {
@@ -779,29 +930,18 @@ public class KafkaITCase {
running = false;
}
});
- stream.addSink(new KafkaSink<String>(zookeeperConnectionString, topic, new JavaDefaultStringSchema()))
+ stream.addSink(new KafkaSink<String>(brokerConnectionStrings, topic, new JavaDefaultStringSchema()))
.setParallelism(1);
- try {
- env.setParallelism(1);
- env.execute();
- } catch (JobExecutionException good) {
- Throwable t = good.getCause();
- int limit = 0;
- while (!(t instanceof SuccessException)) {
- t = t.getCause();
- if (limit++ == 20) {
- LOG.warn("Test failed with exception", good);
- Assert.fail("Test failed with: " + good.getMessage());
- }
- }
- }
+ tryExecute(env, "broker failure test");
}
private void createTestTopic(String topic, int numberOfPartitions, int replicationFactor) {
- KafkaTopicUtils kafkaTopicUtils = new KafkaTopicUtils(zookeeperConnectionString);
- kafkaTopicUtils.createTopic(topic, numberOfPartitions, replicationFactor);
+ // create topic
+ Properties topicConfig = new Properties();
+ LOG.info("Creating topic {}", topic);
+ AdminUtils.createTopic(zkClient, topic, numberOfPartitions, replicationFactor, topicConfig);
}
private static TestingServer getZookeeper() throws Exception {
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTopicUtilsTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTopicUtilsTest.java b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTopicUtilsTest.java
deleted file mode 100644
index 5f0e198..0000000
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTopicUtilsTest.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.connectors.kafka;
-
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.curator.test.TestingServer;
-import org.apache.flink.runtime.net.NetUtils;
-import org.apache.flink.streaming.connectors.kafka.api.simple.KafkaTopicUtils;
-import org.apache.flink.streaming.connectors.kafka.util.KafkaLocalSystemTime;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import kafka.api.PartitionMetadata;
-import kafka.server.KafkaConfig;
-import kafka.server.KafkaServer;
-
-public class KafkaTopicUtilsTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(KafkaTopicUtilsTest.class);
- private static final int NUMBER_OF_BROKERS = 2;
- private static final String TOPIC = "myTopic";
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
-
- @Test
- public void test() {
- int zkPort;
- String kafkaHost;
- String zookeeperConnectionString;
-
- File tmpZkDir;
- List<File> tmpKafkaDirs;
- Map<String, KafkaServer> kafkaServers = null;
- TestingServer zookeeper = null;
-
- try {
- tmpZkDir = tempFolder.newFolder();
-
- tmpKafkaDirs = new ArrayList<File>(NUMBER_OF_BROKERS);
- for (int i = 0; i < NUMBER_OF_BROKERS; i++) {
- tmpKafkaDirs.add(tempFolder.newFolder());
- }
-
- zkPort = NetUtils.getAvailablePort();
- kafkaHost = InetAddress.getLocalHost().getHostName();
- zookeeperConnectionString = "localhost:" + zkPort;
-
- // init zookeeper
- zookeeper = new TestingServer(zkPort, tmpZkDir);
-
- // init kafka kafkaServers
- kafkaServers = new HashMap<String, KafkaServer>();
-
- for (int i = 0; i < NUMBER_OF_BROKERS; i++) {
- KafkaServer kafkaServer = getKafkaServer(kafkaHost, zookeeperConnectionString, i, tmpKafkaDirs.get(i));
- kafkaServers.put(kafkaServer.config().advertisedHostName() + ":" + kafkaServer.config().advertisedPort(), kafkaServer);
- }
-
- // create Kafka topic
- final KafkaTopicUtils kafkaTopicUtils = new KafkaTopicUtils(zookeeperConnectionString);
- kafkaTopicUtils.createTopic(TOPIC, 1, 2);
-
- // check whether topic exists
- assertTrue(kafkaTopicUtils.topicExists(TOPIC));
-
- // check number of partitions
- assertEquals(1, kafkaTopicUtils.getNumberOfPartitions(TOPIC));
-
- // get partition metadata without error
- PartitionMetadata partitionMetadata = kafkaTopicUtils.waitAndGetPartitionMetadata(TOPIC, 0);
- assertEquals(0, partitionMetadata.errorCode());
-
- // get broker list
- assertEquals(new HashSet<String>(kafkaServers.keySet()), kafkaTopicUtils.getBrokerAddresses(TOPIC));
- } catch (IOException e) {
- fail(e.toString());
- } catch (Exception e) {
- fail(e.toString());
- } finally {
- LOG.info("Shutting down all services");
- for (KafkaServer broker : kafkaServers.values()) {
- if (broker != null) {
- broker.shutdown();
- }
- }
-
- if (zookeeper != null) {
- try {
- zookeeper.stop();
- } catch (IOException e) {
- LOG.warn("ZK.stop() failed", e);
- }
- }
- }
- }
-
- /**
- * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed)
- */
- private static KafkaServer getKafkaServer(String kafkaHost, String zookeeperConnectionString, int brokerId, File tmpFolder) throws UnknownHostException {
- Properties kafkaProperties = new Properties();
-
- int kafkaPort = NetUtils.getAvailablePort();
-
- // properties have to be Strings
- kafkaProperties.put("advertised.host.name", kafkaHost);
- kafkaProperties.put("port", Integer.toString(kafkaPort));
- kafkaProperties.put("broker.id", Integer.toString(brokerId));
- kafkaProperties.put("log.dir", tmpFolder.toString());
- kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
- KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);
-
- KafkaServer server = new KafkaServer(kafkaConfig, new KafkaLocalSystemTime());
- server.startup();
- return server;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
index dc20726..9ede613 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/src/test/resources/log4j-test.properties
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-log4j.rootLogger=OFF, testlogger
+log4j.rootLogger=INFO, testlogger
log4j.appender.testlogger=org.apache.log4j.ConsoleAppender
log4j.appender.testlogger.target = System.err
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
index 196f7ec..4bd89c4 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
@@ -18,6 +18,8 @@
package org.apache.flink.streaming.api.checkpoint;
+import java.io.Serializable;
+
/**
* This interface marks a function/operator as <i>asynchronously checkpointed</i>.
* Similar to the {@link Checkpointed} interface, the function must produce a
@@ -32,4 +34,4 @@ package org.apache.flink.streaming.api.checkpoint;
* {@link #snapshotState(long, long)} method is typically a copy or shadow copy
* of the actual state.</p>
*/
-public interface CheckpointedAsynchronously extends Checkpointed {}
+public interface CheckpointedAsynchronously<T extends Serializable> extends Checkpointed<T> {}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
index a59407c..3efad93 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java
@@ -30,8 +30,7 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment {
*/
@Override
public JobExecutionResult execute() throws Exception {
- return ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(), getParallelism(),
- getConfig().isSysoutLoggingEnabled());
+ return execute(DEFAULT_JOB_NAME);
}
/**
@@ -44,7 +43,9 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment {
*/
@Override
public JobExecutionResult execute(String jobName) throws Exception {
- return ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(jobName), getParallelism(),
- getConfig().isSysoutLoggingEnabled());
+ JobExecutionResult result = ClusterUtil.runOnMiniCluster(this.streamGraph.getJobGraph(jobName), getParallelism(),
+ getConfig().isSysoutLoggingEnabled());
+ streamGraph.clear(); // clear graph to allow submitting another job via the same environment.
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
index b0471f9..2e33b82 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/environment/StreamExecutionEnvironment.java
@@ -35,6 +35,7 @@ import org.apache.flink.api.java.io.TextInputFormat;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.typeutils.MissingTypeInfo;
import org.apache.flink.api.java.typeutils.PojoTypeInfo;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.api.java.typeutils.TypeExtractor;
import org.apache.flink.client.program.Client;
import org.apache.flink.client.program.Client.OptimizerPlanEnvironment;
@@ -49,7 +50,6 @@ import org.apache.flink.streaming.api.functions.source.FileReadFunction;
import org.apache.flink.streaming.api.functions.source.FileSourceFunction;
import org.apache.flink.streaming.api.functions.source.FromElementsFunction;
import org.apache.flink.streaming.api.functions.source.GenSequenceFunction;
-import org.apache.flink.streaming.api.functions.source.GenericSourceFunction;
import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.functions.source.SocketTextStreamFunction;
@@ -67,6 +67,8 @@ import com.esotericsoftware.kryo.Serializer;
*/
public abstract class StreamExecutionEnvironment {
+ public final static String DEFAULT_JOB_NAME = "Flink Streaming Job";
+
private static int defaultLocalParallelism = Runtime.getRuntime().availableProcessors();
private long bufferTimeout = 100;
@@ -624,8 +626,8 @@ public abstract class StreamExecutionEnvironment {
TypeInformation<OUT> outTypeInfo;
- if (function instanceof GenericSourceFunction) {
- outTypeInfo = ((GenericSourceFunction<OUT>) function).getType();
+ if (function instanceof ResultTypeQueryable) {
+ outTypeInfo = ((ResultTypeQueryable<OUT>) function).getProducedType();
} else {
try {
outTypeInfo = TypeExtractor.createTypeInfo(SourceFunction.class,
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/GenericSourceFunction.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/GenericSourceFunction.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/GenericSourceFunction.java
deleted file mode 100644
index 0113cfe..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/GenericSourceFunction.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.functions.source;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-
-public interface GenericSourceFunction<T> {
-
- TypeInformation<T> getType();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
index 93bf8eb..271c05c 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java
@@ -60,8 +60,7 @@ import org.slf4j.LoggerFactory;
public class StreamGraph extends StreamingPlan {
private static final Logger LOG = LoggerFactory.getLogger(StreamGraph.class);
- private final static String DEAFULT_JOB_NAME = "Flink Streaming Job";
- private String jobName = DEAFULT_JOB_NAME;
+ private String jobName = StreamExecutionEnvironment.DEFAULT_JOB_NAME;
private final StreamExecutionEnvironment environemnt;
private final ExecutionConfig executionConfig;
@@ -70,17 +69,25 @@ public class StreamGraph extends StreamingPlan {
private long checkpointingInterval = 5000;
private boolean chaining = true;
- private final Map<Integer, StreamNode> streamNodes;
- private final Set<Integer> sources;
+ private Map<Integer, StreamNode> streamNodes;
+ private Set<Integer> sources;
- private final Map<Integer, StreamLoop> streamLoops;
- protected final Map<Integer, StreamLoop> vertexIDtoLoop;
+ private Map<Integer, StreamLoop> streamLoops;
+ protected Map<Integer, StreamLoop> vertexIDtoLoop;
public StreamGraph(StreamExecutionEnvironment environment) {
this.environemnt = environment;
executionConfig = environment.getConfig();
+ // create an empty new stream graph.
+ clear();
+ }
+
+ /**
+ * Remove all registered nodes etc.
+ */
+ public void clear() {
streamNodes = new HashMap<Integer, StreamNode>();
streamLoops = new HashMap<Integer, StreamLoop>();
vertexIDtoLoop = new HashMap<Integer, StreamGraph.StreamLoop>();
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
index f66e394..24a08eb 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java
@@ -33,7 +33,6 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecordSerializer;
import org.apache.flink.streaming.runtime.tasks.StreamTaskContext;
import org.apache.flink.util.Collector;
-import org.apache.flink.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -138,8 +137,7 @@ public abstract class StreamOperator<IN, OUT> implements Serializable {
callUserFunction();
} catch (Exception e) {
if (LOG.isErrorEnabled()) {
- LOG.error("Calling user function failed due to: {}",
- StringUtils.stringifyException(e));
+ LOG.error("Calling user function failed", e);
}
throw new RuntimeException(e);
}
@@ -168,7 +166,7 @@ public abstract class StreamOperator<IN, OUT> implements Serializable {
try {
FunctionUtils.closeFunction(userFunction);
} catch (Exception e) {
- throw new RuntimeException("Error when closing the function: " + e.getMessage());
+ throw new RuntimeException("Error when closing the function", e);
}
}
@@ -187,8 +185,7 @@ public abstract class StreamOperator<IN, OUT> implements Serializable {
public void setChainingStrategy(ChainingStrategy strategy) {
if (strategy == ChainingStrategy.ALWAYS) {
if (!(this instanceof ChainableStreamOperator)) {
- throw new RuntimeException(
- "Operator needs to extend ChainableOperator to be chained");
+ throw new RuntimeException("Operator needs to extend ChainableOperator to be chained");
}
}
this.chainingStrategy = strategy;
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionableState.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionableState.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionableState.java
deleted file mode 100644
index 4dd4b45..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/state/PartitionableState.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.state;
-
-import org.apache.flink.runtime.state.OperatorState;
-
-/**
- * Base class for representing operator states that can be repartitioned for
- * state state and load balancing.
- *
- * @param <T>
- * The type of the operator state.
- */
-public abstract class PartitionableState<T> extends OperatorState<T> {
-
- private static final long serialVersionUID = 1L;
-
- PartitionableState(T initialState) {
- super(initialState);
- }
-
- /**
- * Repartitions(divides) the current state into the given number of new
- * partitions. The created partitions will be used to redistribute then
- * rebuild the state among the parallel instances of the operator. The
- * implementation should reflect the partitioning of the input values to
- * maintain correct operator behavior.
- *
- * </br> </br> It is also assumed that if we would {@link #reBuild} the
- * repartitioned state we would basically get the same as before.
- *
- *
- * @param numberOfPartitions
- * The desired number of partitions. The method must return an
- * array of that size.
- * @return The array containing the state part for each partition.
- */
- public abstract OperatorState<T>[] repartition(int numberOfPartitions);
-
- /**
- * Rebuilds the current state partition from the given parts. Used for
- * building the state after a re-balance phase.
- *
- * @param parts
- * The state parts that will be used to rebuild the current
- * partition.
- * @return The rebuilt operator state.
- */
- public abstract OperatorState<T> reBuild(OperatorState<T>... parts);
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 930c9b4..0259568 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -273,7 +273,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
synchronized (checkpointLock) {
if (isRunning) {
try {
- LOG.info("Starting checkpoint " + checkpointId);
+ LOG.info("Starting checkpoint {} on task {}", checkpointId, getName());
// first draw the state that should go into checkpoint
LocalStateHandle state;
@@ -282,7 +282,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
state = userState == null ? null : new LocalStateHandle(userState);
}
catch (Exception e) {
- throw new Exception("Error while drawing snapshot of the user state.");
+ throw new Exception("Error while drawing snapshot of the user state.", e);
}
// now emit the checkpoint barriers
@@ -333,8 +333,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
triggerCheckpoint(sStep.getId(), sStep.getTimestamp());
}
catch (Exception e) {
- throw new RuntimeException(
- "Error triggering a checkpoint as the result of receiving checkpoint barrier", e);
+ throw new RuntimeException("Error triggering a checkpoint as the result of receiving checkpoint barrier", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java
index faaa79b..87c9757 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/DeserializationSchema.java
@@ -17,9 +17,11 @@
package org.apache.flink.streaming.util.serialization;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+
import java.io.Serializable;
-public interface DeserializationSchema<T> extends Serializable {
+public interface DeserializationSchema<T> extends Serializable, ResultTypeQueryable<T> {
/**
* Deserializes the incoming data.
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/JavaDefaultStringSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/JavaDefaultStringSchema.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/JavaDefaultStringSchema.java
index 93d13ab..a4b1419 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/JavaDefaultStringSchema.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/JavaDefaultStringSchema.java
@@ -18,6 +18,8 @@
package org.apache.flink.streaming.util.serialization;
import org.apache.commons.lang3.SerializationUtils;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
public class JavaDefaultStringSchema implements DeserializationSchema<String>, SerializationSchema<String, byte[]> {
@@ -38,4 +40,9 @@ public class JavaDefaultStringSchema implements DeserializationSchema<String>, S
return SerializationUtils.deserialize(message);
}
+ @Override
+ public TypeInformation<String> getProducedType() {
+ return TypeExtractor.getForClass(String.class);
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/RawSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/RawSchema.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/RawSchema.java
index e457bef..9c5885f 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/RawSchema.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/RawSchema.java
@@ -17,6 +17,9 @@
package org.apache.flink.streaming.util.serialization;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+
public class RawSchema implements DeserializationSchema<byte[]>,
SerializationSchema<byte[], byte[]> {
@@ -36,4 +39,9 @@ public class RawSchema implements DeserializationSchema<byte[]>,
public byte[] serialize(byte[] element) {
return element;
}
+
+ @Override
+ public TypeInformation<byte[]> getProducedType() {
+ return TypeExtractor.getForClass(byte[].class);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
index 3d0a0d5..7c5946d 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/util/serialization/SimpleStringSchema.java
@@ -17,6 +17,9 @@
package org.apache.flink.streaming.util.serialization;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+
public class SimpleStringSchema implements DeserializationSchema<String>,
SerializationSchema<String, String> {
@@ -37,4 +40,8 @@ public class SimpleStringSchema implements DeserializationSchema<String>,
return element;
}
+ @Override
+ public TypeInformation<String> getProducedType() {
+ return TypeExtractor.getForClass(String.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/54e95761/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/OperatorStateTest.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/OperatorStateTest.java b/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/OperatorStateTest.java
deleted file mode 100644
index 136a091..0000000
--- a/flink-staging/flink-streaming/flink-streaming-core/src/test/java/org/apache/flink/streaming/api/state/OperatorStateTest.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.api.state;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.flink.runtime.state.OperatorState;
-import org.apache.flink.runtime.state.StateCheckpoint;
-import org.junit.Test;
-
-public class OperatorStateTest {
-
- @Test
- public void testOperatorState() {
- OperatorState<Integer> os = new OperatorState<Integer>(5);
-
- StateCheckpoint<Integer> scp = os.checkpoint();
-
- assertTrue(os.stateEquals(scp.restore()));
-
- assertEquals((Integer) 5, os.getState());
-
- os.update(10);
-
- assertEquals((Integer) 10, os.getState());
- }
-
-}
[06/10] flink git commit: [streaming] StreamTask updated to handle
chained states
Posted by se...@apache.org.
[streaming] StreamTask updated to handle chained states
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c8de6568
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c8de6568
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c8de6568
Branch: refs/heads/master
Commit: c8de6568d1522dc2aa4c10353634e2978edaaafa
Parents: 54e9576
Author: Gyula Fora <gy...@apache.org>
Authored: Wed May 6 12:24:41 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue May 12 21:35:58 2015 +0200
----------------------------------------------------------------------
.../streaming/runtime/tasks/OutputHandler.java | 6 +-
.../streaming/runtime/tasks/StreamTask.java | 57 +++++++++++++++++--
.../ProcessFailureStreamingRecoveryITCase.java | 60 ++++++++++++++------
3 files changed, 101 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c8de6568/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
index 55c0551..4b631a7 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/OutputHandler.java
@@ -49,7 +49,7 @@ public class OutputHandler<OUT> {
private ClassLoader cl;
private Collector<OUT> outerCollector;
- public List<ChainableStreamOperator<?, ?>> chainedOperators;
+ private List<ChainableStreamOperator<?, ?>> chainedOperators;
private Map<StreamEdge, StreamOutput<?>> outputMap;
@@ -98,6 +98,10 @@ public class OutputHandler<OUT> {
public Collection<StreamOutput<?>> getOutputs() {
return outputMap.values();
}
+
+ public List<ChainableStreamOperator<?, ?>> getChainedOperators(){
+ return chainedOperators;
+ }
/**
* This method builds up a nested collector which encapsulates all the
http://git-wip-us.apache.org/repos/asf/flink/blob/c8de6568/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
index 0259568..7e09f22 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java
@@ -19,7 +19,11 @@ package org.apache.flink.streaming.runtime.tasks;
import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.functors.NotNullPredicate;
import org.apache.flink.runtime.event.task.TaskEvent;
import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
@@ -37,7 +41,6 @@ import org.apache.flink.streaming.runtime.streamrecord.StreamRecordSerializer;
import org.apache.flink.util.Collector;
import org.apache.flink.util.MutableObjectIterator;
import org.apache.flink.util.StringUtils;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,6 +61,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
private InputHandler<IN> inputHandler;
protected OutputHandler<OUT> outputHandler;
private StreamOperator<IN, OUT> streamOperator;
+ private boolean chained;
protected volatile boolean isRunning = false;
private StreamingRuntimeContext context;
@@ -94,6 +98,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
public void setInputsOutputs() {
inputHandler = new InputHandler<IN>(this);
outputHandler = new OutputHandler<OUT>(this);
+ chained = !outputHandler.getChainedOperators().isEmpty();
}
protected void setOperator() {
@@ -165,7 +170,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
protected void openOperator() throws Exception {
streamOperator.open(getTaskConfiguration());
- for (ChainableStreamOperator<?, ?> operator : outputHandler.chainedOperators) {
+ for (ChainableStreamOperator<?, ?> operator : outputHandler.getChainedOperators()) {
operator.setRuntimeContext(context);
operator.open(getTaskConfiguration());
}
@@ -174,7 +179,7 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
protected void closeOperator() throws Exception {
streamOperator.close();
- for (ChainableStreamOperator<?, ?> operator : outputHandler.chainedOperators) {
+ for (ChainableStreamOperator<?, ?> operator : outputHandler.getChainedOperators()) {
operator.close();
}
}
@@ -256,7 +261,26 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
// here, we later resolve the state handle into the actual state by
// loading the state described by the handle from the backup store
Serializable state = stateHandle.getState();
- streamOperator.restoreInitialState(state);
+
+ if (chained) {
+ @SuppressWarnings("unchecked")
+ List<Serializable> chainedStates = (List<Serializable>) state;
+
+ Serializable headState = chainedStates.get(0);
+ if (headState != null) {
+ streamOperator.restoreInitialState(headState);
+ }
+
+ for (int i = 1; i < chainedStates.size(); i++) {
+ Serializable chainedState = chainedStates.get(i);
+ if (chainedState != null) {
+ outputHandler.getChainedOperators().get(i - 1).restoreInitialState(chainedState);
+ }
+ }
+
+ } else {
+ streamOperator.restoreInitialState(state);
+ }
}
/**
@@ -278,7 +302,25 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
// first draw the state that should go into checkpoint
LocalStateHandle state;
try {
- Serializable userState = streamOperator.getStateSnapshotFromFunction(checkpointId, timestamp);
+ Serializable userState = streamOperator.getStateSnapshotFromFunction(
+ checkpointId, timestamp);
+
+ if (chained) {
+ // We construct a list of states for chained tasks
+ List<Serializable> chainedStates = new ArrayList<Serializable>();
+
+ chainedStates.add(userState);
+
+ for (StreamOperator<?, ?> chainedOperator : outputHandler.getChainedOperators()) {
+ chainedStates.add(chainedOperator.getStateSnapshotFromFunction(
+ checkpointId, timestamp));
+ }
+
+ userState = CollectionUtils.exists(chainedStates,
+ NotNullPredicate.INSTANCE) ? (Serializable) chainedStates
+ : null;
+ }
+
state = userState == null ? null : new LocalStateHandle(userState);
}
catch (Exception e) {
@@ -309,6 +351,11 @@ public class StreamTask<IN, OUT> extends AbstractInvokable implements StreamTask
// we do nothing here so far. this should call commit on the source function, for example
synchronized (checkpointLock) {
streamOperator.confirmCheckpointCompleted(checkpointId, timestamp);
+ if (chained) {
+ for (StreamOperator<?, ?> op : outputHandler.getChainedOperators()) {
+ op.confirmCheckpointCompleted(checkpointId, timestamp);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/c8de6568/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
index 65e39a2..c3a6314 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureStreamingRecoveryITCase.java
@@ -19,6 +19,7 @@
package org.apache.flink.test.recovery;
import org.apache.commons.io.FileUtils;
+import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
@@ -77,25 +78,15 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
env.enableCheckpointing(200);
DataStream<Long> result = env.addSource(new SleepyDurableGenerateSequence(coordinateDir, DATA_COUNT))
-
- // make sure every mapper is involved
-// .shuffle()
-
- // populate the coordinate directory so we can proceed to TaskManager failure
- .map(new RichMapFunction<Long, Long>() {
-
- private boolean markerCreated = false;
-
+ // add a non-chained no-op map to test the chain state restore logic
+ .distribute().map(new MapFunction<Long, Long>() {
@Override
public Long map(Long value) throws Exception {
- if (!markerCreated) {
- int taskIndex = getRuntimeContext().getIndexOfThisSubtask();
- touchFile(new File(coordinateDir, READY_MARKER_FILE_PREFIX + taskIndex));
- markerCreated = true;
- }
return value;
}
- });
+ })
+ // populate the coordinate directory so we can proceed to TaskManager failure
+ .map(new StatefulMapper(coordinateDir));
//write result to temporary file
result.addSink(new RichSinkFunction<Long>() {
@@ -161,7 +152,6 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
}
@Override
- @SuppressWarnings("unchecked")
public void run(Collector<Long> collector) throws Exception {
StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
@@ -203,6 +193,44 @@ public class ProcessFailureStreamingRecoveryITCase extends AbstractProcessFailur
collected = state;
}
}
+
+ public static class StatefulMapper extends RichMapFunction<Long, Long> implements
+ Checkpointed<Integer> {
+ private boolean markerCreated = false;
+ private File coordinateDir;
+ private boolean restored = false;
+
+ public StatefulMapper(File coordinateDir) {
+ this.coordinateDir = coordinateDir;
+ }
+
+ @Override
+ public Long map(Long value) throws Exception {
+ if (!markerCreated) {
+ int taskIndex = getRuntimeContext().getIndexOfThisSubtask();
+ touchFile(new File(coordinateDir, READY_MARKER_FILE_PREFIX + taskIndex));
+ markerCreated = true;
+ }
+ return value;
+ }
+
+ @Override
+ public void close() {
+ if (!restored) {
+ fail();
+ }
+ }
+
+ @Override
+ public Integer snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {
+ return 1;
+ }
+
+ @Override
+ public void restoreState(Integer state) {
+ restored = true;
+ }
+ }
private static void fileBatchHasEveryNumberLower(int numFiles, int numbers, File path) throws IOException {