You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/05/11 22:19:35 UTC
[1/8] flink git commit: [runtime] Fix TaskExecutionState against
non-serializable exceptions.
Repository: flink
Updated Branches:
refs/heads/master d259e6962 -> fbea2da26
[runtime] Fix TaskExecutionState against non-serializable exceptions.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1d368a4b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1d368a4b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1d368a4b
Branch: refs/heads/master
Commit: 1d368a4b77f3ea53809e622564c87b657547c405
Parents: 1c8d866
Author: Stephan Ewen <se...@apache.org>
Authored: Sun May 3 01:57:37 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon May 11 21:07:57 2015 +0200
----------------------------------------------------------------------
.../runtime/taskmanager/TaskExecutionState.java | 37 ++++++++++++++++----
1 file changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1d368a4b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
index 96d56eb..6c85ab5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/TaskExecutionState.java
@@ -18,12 +18,12 @@
package org.apache.flink.runtime.taskmanager;
-import java.io.IOException;
import java.util.Arrays;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.util.ExceptionUtils;
import org.apache.flink.util.InstantiationUtil;
/**
@@ -68,6 +68,7 @@ public class TaskExecutionState implements java.io.Serializable {
/**
* Creates a new task execution state update, with an attached exception.
+ * This constructor may never throw an exception.
*
* @param jobID
* the ID of the job the task belongs to
@@ -91,13 +92,35 @@ public class TaskExecutionState implements java.io.Serializable {
this.cachedError = error;
if (error != null) {
+ byte[] serializedError;
try {
- this.serializedError = InstantiationUtil.serializeObject(error);
+ serializedError = InstantiationUtil.serializeObject(error);
}
- catch (IOException e) {
- throw new RuntimeException("Error while serializing task exception", e);
+ catch (Throwable t) {
+ // could not serialize exception. send the stringified version instead
+ try {
+ this.cachedError = new Exception(ExceptionUtils.stringifyException(error));
+ serializedError = InstantiationUtil.serializeObject(this.cachedError);
+ }
+ catch (Throwable tt) {
+ // seems like we cannot do much to report the actual exception
+ // report a placeholder instead
+ try {
+ this.cachedError = new Exception("Cause is a '" + error.getClass().getName()
+ + "' (failed to serialize or stringify)");
+ serializedError = InstantiationUtil.serializeObject(this.cachedError);
+ }
+ catch (Throwable ttt) {
+ // this should never happen unless the JVM is fubar.
+ // we just report the state without the error
+ this.cachedError = null;
+ serializedError = null;
+ }
+ }
}
- } else {
+ this.serializedError = serializedError;
+ }
+ else {
this.serializedError = null;
}
}
@@ -181,7 +204,7 @@ public class TaskExecutionState implements java.io.Serializable {
public String toString() {
return String.format("TaskState jobId=%s, executionId=%s, state=%s, error=%s",
jobID, executionId, executionState,
- cachedError == null ? "(null)"
- : cachedError.getClass().getName() + ": " + cachedError.getMessage());
+ cachedError == null ? (serializedError == null ? "(null)" : "(serialized)")
+ : (cachedError.getClass().getName() + ": " + cachedError.getMessage()));
}
}
[6/8] flink git commit: Update build target path in README.md
Posted by se...@apache.org.
Update build target path in README.md
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/59faf464
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/59faf464
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/59faf464
Branch: refs/heads/master
Commit: 59faf464a9aaf253603a400c5d680b25be29bd15
Parents: 8e61301
Author: Stephan Ewen <se...@apache.org>
Authored: Sun May 3 14:10:58 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon May 11 21:13:44 2015 +0200
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/59faf464/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 1c1f78c..a50f980 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ cd flink
mvn clean package -DskipTests # this will take up to 5 minutes
```
-Flink is now installed in `flink-dist/target`
+Flink is now installed in `build-target`
## Developing Flink
[8/8] flink git commit: [FLINK-1969] [runtime] Remove deprecated
profiler code
Posted by se...@apache.org.
[FLINK-1969] [runtime] Remove deprecated profiler code
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fbea2da2
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fbea2da2
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fbea2da2
Branch: refs/heads/master
Commit: fbea2da26d01c470687a5ad217a5fd6ad1de89e4
Parents: 59faf46
Author: Stephan Ewen <se...@apache.org>
Authored: Sun May 3 14:10:35 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon May 11 21:13:44 2015 +0200
----------------------------------------------------------------------
.../runtime/profiling/JobManagerProfiler.java | 75 ----
.../runtime/profiling/ProfilingException.java | 42 --
.../runtime/profiling/ProfilingListener.java | 27 --
.../flink/runtime/profiling/ProfilingUtils.java | 180 ---------
.../runtime/profiling/TaskManagerProfiler.java | 56 ---
.../profiling/impl/EnvironmentThreadSet.java | 217 ----------
.../profiling/impl/InstanceProfiler.java | 342 ----------------
.../profiling/impl/JobProfilingData.java | 158 --------
.../InternalExecutionVertexProfilingData.java | 90 -----
...ernalExecutionVertexThreadProfilingData.java | 96 -----
.../types/InternalInstanceProfilingData.java | 244 ------------
.../impl/types/InternalProfilingData.java | 23 --
.../impl/types/ProfilingDataContainer.java | 101 -----
.../profiling/types/InstanceProfilingEvent.java | 399 -------------------
.../types/InstanceSummaryProfilingEvent.java | 89 -----
.../runtime/profiling/types/ProfilingEvent.java | 135 -------
.../types/SingleInstanceProfilingEvent.java | 147 -------
.../profiling/types/ThreadProfilingEvent.java | 150 -------
.../profiling/types/VertexProfilingEvent.java | 142 -------
.../flink/runtime/jobmanager/JobManager.scala | 26 +-
.../runtime/jobmanager/JobManagerProfiler.scala | 55 ---
.../messages/JobManagerProfilerMessages.scala | 34 --
.../messages/TaskManagerProfilerMessages.scala | 55 ---
.../taskmanager/TaskManagerProfiler.scala | 182 ---------
.../profiling/impl/InstanceProfilerTest.java | 182 ---------
.../profiling/types/ProfilingTypesTest.java | 177 --------
.../runtime/testingUtils/TestingCluster.scala | 4 +-
.../test/util/ForkableFlinkMiniCluster.scala | 4 +-
.../apache/flink/yarn/ApplicationMaster.scala | 10 +-
29 files changed, 12 insertions(+), 3430 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/JobManagerProfiler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/JobManagerProfiler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/JobManagerProfiler.java
deleted file mode 100644
index 4c4a20e..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/JobManagerProfiler.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.profiling;
-
-import org.apache.flink.runtime.executiongraph.ExecutionGraph;
-import org.apache.flink.api.common.JobID;
-
-/**
- * This interface must be implemented by profiling components
- * for the job manager.
- *
- */
-public interface JobManagerProfiler {
-
- /**
- * Registers the given {@link ExecutionGraph} for profiling.
- *
- * @param executionGraph
- * the {@link ExecutionGraph} to register for profiling
- */
- void registerProfilingJob(ExecutionGraph executionGraph);
-
- /**
- * Unregisters the given {@link ExecutionGraph} from profiling. Calling this
- * method will also unregister all of the job's registered listeners.
- *
- * @param executionGraph the {@link ExecutionGraph} to unregister.
- */
- void unregisterProfilingJob(ExecutionGraph executionGraph);
-
- /**
- * Registers the given {@link ProfilingListener} object to receive
- * profiling data for the job with the given job ID.
- *
- * @param jobID
- * the ID of the job to receive profiling data for
- * @param profilingListener
- * the {@link ProfilingListener} object to register
- */
- void registerForProfilingData(JobID jobID, ProfilingListener profilingListener);
-
- /**
- * Unregisters the given {@link ProfilingListener} object from receiving
- * profiling data issued by the job manager's profiling component.
- *
- * @param jobID
- * the ID of the job the {@link ProfilingListener} object has originally been registered for
- * @param profilingListener
- * the {@link ProfilingListener} object to unregister
- */
- void unregisterFromProfilingData(JobID jobID, ProfilingListener profilingListener);
-
- /**
- * Shuts done the job manager's profiling component
- * and stops all its internal processes.
- */
- void shutdown();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingException.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingException.java
deleted file mode 100644
index f3148d9..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.profiling;
-
-/**
- * A profiling exception is thrown if an error occur during profiling execution.
- *
- */
-public class ProfilingException extends Exception {
-
- /**
- * Generated serialVersionUID.
- */
- private static final long serialVersionUID = -3282996556813630561L;
-
- /**
- * Constructs a new profiling exception with the given error message.
- *
- * @param errorMsg
- * The error message to be included in the exception.
- */
- public ProfilingException(String errorMsg) {
- super(errorMsg);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingListener.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingListener.java
deleted file mode 100644
index b3116ed..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingListener.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.profiling;
-
-import org.apache.flink.runtime.profiling.types.ProfilingEvent;
-
-public interface ProfilingListener {
-
- void processProfilingEvents(ProfilingEvent profilingEvent);
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingUtils.java
deleted file mode 100644
index 95211be..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/ProfilingUtils.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.profiling;
-
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.InetAddress;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.flink.runtime.instance.InstanceConnectionInfo;
-import org.apache.flink.util.StringUtils;
-
-/**
- * This class contains utility functions to load and configure Nephele's
- * profiling component.
- *
- */
-public class ProfilingUtils {
-
- /**
- * The logging instance used to report problems.
- */
- private static final Logger LOG = LoggerFactory.getLogger(ProfilingUtils.class);
-
- /**
- * The key to check the job manager's profiling component should be enabled.
- */
- public static final String ENABLE_PROFILING_KEY = "jobmanager.profiling.enable";
-
- /**
- * The class name of the job manager's profiling component to load if progiling is enabled.
- */
- public static final String JOBMANAGER_CLASSNAME_KEY = "jobmanager.profiling.classname";
-
- /**
- * The class name of the task manager's profiling component to load if profiling is enabled.
- */
- public static final String TASKMANAGER_CLASSNAME_KEY = "taskmanager.profiling.classname";
-
- /**
- * The key to check whether job profiling should be enabled for a specific job.
- */
- public static final String PROFILE_JOB_KEY = "job.profiling.enable";
-
- /**
- * The key to check the port of the job manager's profiling RPC server.
- */
- public static final String JOBMANAGER_RPC_PORT_KEY = "jobmanager.profiling.rpc.port";
-
- /**
- * The default network port the job manager's profiling component starts its RPC server on.
- */
- public static final int JOBMANAGER_DEFAULT_RPC_PORT = 6124;
-
- /**
- * Key to interval in which a task manager is supposed to send profiling data to the job manager.
- */
- public static final String TASKMANAGER_REPORTINTERVAL_KEY = "taskmanager.profiling.reportinterval";
-
- /**
- * Default interval which the task manager uses to report profiling data to the job manager.
- */
- public static final int DEFAULT_TASKMANAGER_REPORTINTERVAL = 2;
-
- /**
- * Creates an instance of the job manager's profiling component.
- *
- * @param profilerClassName
- * the class name of the profiling component to load
- * @param jobManagerBindAddress
- * the address the job manager's RPC server is bound to
- * @return an instance of the job manager profiling component or <code>null</code> if an error occurs
- */
- @SuppressWarnings("unchecked")
- public static JobManagerProfiler loadJobManagerProfiler(String profilerClassName, InetAddress jobManagerBindAddress) {
-
- final Class<? extends JobManagerProfiler> profilerClass;
- try {
- profilerClass = (Class<? extends JobManagerProfiler>) Class.forName(profilerClassName);
- } catch (ClassNotFoundException e) {
- LOG.error("Cannot find class " + profilerClassName + ": " + StringUtils.stringifyException(e));
- return null;
- }
-
- JobManagerProfiler profiler = null;
-
- try {
-
- final Constructor<JobManagerProfiler> constr = (Constructor<JobManagerProfiler>) profilerClass.getConstructor(InetAddress.class);
- profiler = constr.newInstance(jobManagerBindAddress);
-
- } catch(InvocationTargetException e) {
- LOG.error("Cannot create profiler: " + StringUtils.stringifyException(e));
- return null;
- } catch (NoSuchMethodException e) {
- LOG.error("Cannot create profiler: " + StringUtils.stringifyException(e));
- return null;
- } catch (InstantiationException e) {
- LOG.error("Cannot create profiler: " + StringUtils.stringifyException(e));
- return null;
- } catch (IllegalAccessException e) {
- LOG.error("Cannot create profiler: " + StringUtils.stringifyException(e));
- return null;
- } catch (IllegalArgumentException e) {
- LOG.error("Cannot create profiler: " + StringUtils.stringifyException(e));
- return null;
- }
-
- return profiler;
- }
-
- /**
- * Creates an instance of the task manager's profiling component.
- *
- * @param profilerClassName
- * the class name of the profiling component to load
- * @return an instance of the task manager profiling component or <code>null</code> if an error occurs
- */
- @SuppressWarnings("unchecked")
- public static TaskManagerProfiler loadTaskManagerProfiler(String profilerClassName, InetAddress jobManagerAddress,
- InstanceConnectionInfo instanceConnectionInfo) {
-
- final Class<? extends TaskManagerProfiler> profilerClass;
- try {
- profilerClass = (Class<? extends TaskManagerProfiler>) Class.forName(profilerClassName);
- } catch (ClassNotFoundException e) {
- LOG.error("Cannot find class " + profilerClassName + ": " + StringUtils.stringifyException(e));
- return null;
- }
-
- Constructor<? extends TaskManagerProfiler> constructor = null;
- try {
- constructor = profilerClass.getConstructor(InetAddress.class, InstanceConnectionInfo.class);
- } catch (SecurityException e1) {
- LOG.error("Security exception while retrieving constructor for class " + profilerClass.getCanonicalName()
- + ".", e1);
- return null;
- } catch (NoSuchMethodException e1) {
- LOG.error("Class " + profilerClass.getCanonicalName() + " does not have a constructor taking a " +
- "InetAddress and InstanceConnectionInfo parameter.", e1);
- return null;
- }
-
- TaskManagerProfiler profiler = null;
- try {
- profiler = constructor.newInstance(jobManagerAddress, instanceConnectionInfo);
- } catch (IllegalArgumentException e) {
- LOG.error("IllegalArgumentException while creating object of class " + profilerClass.getCanonicalName() +
- ".", e);
- } catch (InstantiationException e) {
- LOG.error("Could not instantiate object of class " + profilerClass.getCanonicalName() + ".",e);
- } catch (IllegalAccessException e) {
- LOG.error("IllegalAccessException while creating object of class " + profilerClass.getCanonicalName() + ".",
- e);
- } catch (InvocationTargetException e) {
- LOG.error("InvocationTargetException while creating object of class " + profilerClass.getCanonicalName() +
- ".", e);
- }
-
- return profiler;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/TaskManagerProfiler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/TaskManagerProfiler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/TaskManagerProfiler.java
deleted file mode 100644
index 709a05c..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/TaskManagerProfiler.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.runtime.taskmanager.Task;
-
-/**
- * This interface must be implemented by profiling components
- * for the task manager manager.
- */
-public interface TaskManagerProfiler {
-
- /**
- * Registers a {@link org.apache.flink.runtime.taskmanager.Task} object for profiling.
- *
- * @param task
- * task to be register a profiling listener for
- * @param jobConfiguration
- * the job configuration sent with the task
- */
- void registerTask(Task task, Configuration jobConfiguration);
-
- /**
- * Unregisters all previously registered {@link org.apache.flink.runtime.taskmanager.Task}
- * objects for the vertex identified by the given ID.
- *
- * @param id
- * the ID of the vertex to unregister the
- * {@link org.apache.flink.runtime.taskmanager.Task} objects for
- */
- void unregisterTask(ExecutionAttemptID id);
-
- /**
- * Shuts done the task manager's profiling component
- * and stops all its internal processes.
- */
- void shutdown();
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/EnvironmentThreadSet.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/EnvironmentThreadSet.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/EnvironmentThreadSet.java
deleted file mode 100644
index d5ce137..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/EnvironmentThreadSet.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.impl;
-
-import java.lang.management.ThreadInfo;
-import java.lang.management.ThreadMXBean;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.profiling.impl.types.InternalExecutionVertexThreadProfilingData;
-
-public class EnvironmentThreadSet {
-
- private static final long NANO_TO_MILLISECONDS = 1000 * 1000;
-
- private static final long PERCENT = 100;
-
-
-
- private final Thread mainThread;
-
- private final JobVertexID vertexId;
-
- private final int subtask;
-
- private final ExecutionAttemptID executionId;
-
- private final Map<Thread, CPUUtilizationSnapshot> userThreads = new HashMap<Thread, CPUUtilizationSnapshot>();
-
- private CPUUtilizationSnapshot mainThreadSnapshot;
-
-
- public EnvironmentThreadSet(ThreadMXBean tmx, Thread mainThread, JobVertexID vertexId, int subtask, ExecutionAttemptID executionId) {
- this.mainThread = mainThread;
- this.vertexId = vertexId;
- this.subtask = subtask;
- this.executionId = executionId;
-
- this.mainThreadSnapshot = createCPUUtilizationSnapshot(tmx, mainThread, System.currentTimeMillis());
- }
-
- public Thread getMainThread() {
- return this.mainThread;
- }
-
- public void addUserThread(ThreadMXBean tmx, Thread thread) {
- synchronized (this.userThreads) {
- this.userThreads.put(thread, createCPUUtilizationSnapshot(tmx, thread, System.currentTimeMillis()));
- }
- }
-
- public void removeUserThread(Thread thread) {
- synchronized (this.userThreads) {
- this.userThreads.remove(thread);
- }
- }
-
- public int getNumberOfUserThreads() {
- synchronized (this.userThreads) {
- return this.userThreads.size();
- }
- }
-
- private CPUUtilizationSnapshot createCPUUtilizationSnapshot(ThreadMXBean tmx, Thread thread, long timestamp) {
- final long threadId = thread.getId();
-
- final ThreadInfo threadInfo = tmx.getThreadInfo(threadId);
- if(threadInfo == null) {
- return null;
- }
-
- return new CPUUtilizationSnapshot(timestamp,
- tmx.getThreadCpuTime(threadId) / NANO_TO_MILLISECONDS,
- tmx.getThreadUserTime(threadId) / NANO_TO_MILLISECONDS,
- threadInfo.getWaitedTime(),
- threadInfo.getBlockedTime());
- }
-
- public InternalExecutionVertexThreadProfilingData captureCPUUtilization(JobID jobID, ThreadMXBean tmx, long timestamp) {
-
- synchronized (this.userThreads) {
-
- // Calculate utilization for main thread first
- final CPUUtilizationSnapshot newMainThreadSnapshot = createCPUUtilizationSnapshot(tmx, this.mainThread, timestamp);
- if(newMainThreadSnapshot == null) {
- return null;
- }
-
- final long mainInterval = newMainThreadSnapshot.getTimestamp() - this.mainThreadSnapshot.getTimestamp();
-
- if (mainInterval == 0) {
- return null;
- }
-
- long cputime = newMainThreadSnapshot.getTotalCPUTime() - this.mainThreadSnapshot.getTotalCPUTime();
- long usrtime = newMainThreadSnapshot.getTotalCPUUserTime() - this.mainThreadSnapshot.getTotalCPUUserTime();
- long systime = cputime - usrtime;
- long waitime = newMainThreadSnapshot.getTotalCPUWaitTime() - this.mainThreadSnapshot.getTotalCPUWaitTime();
- long blktime = newMainThreadSnapshot.getTotalCPUBlockTime()
- - this.mainThreadSnapshot.getTotalCPUBlockTime();
-
- int sumUsrTime = (int) ((usrtime * PERCENT) / mainInterval);
- int sumSysTime = (int) ((systime * PERCENT) / mainInterval);
- int sumBlkTime = (int) ((blktime * PERCENT) / mainInterval);
- int sumWaiTime = (int) ((waitime * PERCENT) / mainInterval);
-
- // Update snapshot
- this.mainThreadSnapshot = newMainThreadSnapshot;
-
- if (!this.userThreads.isEmpty()) {
-
- final Iterator<Thread> it = this.userThreads.keySet().iterator();
- int divisor = this.userThreads.size();
- while (it.hasNext()) {
-
- final Thread userThread = it.next();
- final CPUUtilizationSnapshot newUtilizationSnaphot = createCPUUtilizationSnapshot(tmx, userThread,
- timestamp);
- final CPUUtilizationSnapshot oldUtilizationSnapshot = this.userThreads.get(userThread);
-
- long interval = newUtilizationSnaphot.getTimestamp() - oldUtilizationSnapshot.getTimestamp();
-
- if (interval == 0) {
- --divisor;
- continue;
- }
-
- cputime = newUtilizationSnaphot.getTotalCPUTime() - oldUtilizationSnapshot.getTotalCPUTime();
- usrtime = newUtilizationSnaphot.getTotalCPUUserTime() - oldUtilizationSnapshot.getTotalCPUUserTime();
- systime = cputime - usrtime;
- waitime = newUtilizationSnaphot.getTotalCPUWaitTime() - oldUtilizationSnapshot.getTotalCPUWaitTime();
- blktime = newUtilizationSnaphot.getTotalCPUBlockTime() - oldUtilizationSnapshot.getTotalCPUBlockTime();
-
- sumUsrTime += (int) ((usrtime * PERCENT) / interval);
- sumSysTime += (int) ((systime * PERCENT) / interval);
- sumBlkTime += (int) ((blktime * PERCENT) / interval);
- sumWaiTime += (int) ((waitime * PERCENT) / interval);
-
- // Update snapshot
- this.userThreads.put(userThread, newUtilizationSnaphot);
- }
-
- sumUsrTime /= (divisor + 1);
- sumSysTime /= (divisor + 1);
- sumBlkTime /= (divisor + 1);
- sumWaiTime /= (divisor + 1);
- }
-
- return new InternalExecutionVertexThreadProfilingData(jobID, this.vertexId, this.subtask, this.executionId,
- (int) mainInterval, sumUsrTime, sumSysTime, sumBlkTime, sumWaiTime);
- }
- }
-
- // --------------------------------------------------------------------------------------------
-
- private class CPUUtilizationSnapshot {
-
- private final long timestamp;
-
- private final long totalCPUTime;
-
- private final long totalCPUUserTime;
-
- private final long totalCPUWaitTime;
-
- private final long totalCPUBlockTime;
-
- public CPUUtilizationSnapshot(long timestamp, long totalCPUTime, long totalCPUUserTime, long totalCPUWaitTime,
- long totalCPUBlockTime) {
- this.timestamp = timestamp;
- this.totalCPUTime = totalCPUTime;
- this.totalCPUUserTime = totalCPUUserTime;
- this.totalCPUWaitTime = totalCPUWaitTime;
- this.totalCPUBlockTime = totalCPUBlockTime;
- }
-
- public long getTimestamp() {
- return this.timestamp;
- }
-
- public long getTotalCPUTime() {
- return this.totalCPUTime;
- }
-
- public long getTotalCPUUserTime() {
- return this.totalCPUUserTime;
- }
-
- public long getTotalCPUWaitTime() {
- return this.totalCPUWaitTime;
- }
-
- public long getTotalCPUBlockTime() {
- return this.totalCPUBlockTime;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/InstanceProfiler.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/InstanceProfiler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/InstanceProfiler.java
deleted file mode 100644
index d973017..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/InstanceProfiler.java
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.profiling.impl;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.flink.runtime.profiling.ProfilingException;
-import org.apache.flink.runtime.profiling.impl.types.InternalInstanceProfilingData;
-import org.apache.flink.util.StringUtils;
-
-public class InstanceProfiler {
-
- static final String PROC_MEMINFO = "/proc/meminfo";
-
- static final String PROC_STAT = "/proc/stat";
-
- static final String PROC_NET_DEV = "/proc/net/dev";
-
- private static final String LOOPBACK_INTERFACE_NAME = "lo";
-
- private static final Pattern CPU_PATTERN = Pattern
- .compile("^cpu\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+).+$");
-
- private static final Pattern NETWORK_PATTERN = Pattern
- .compile("^\\s*(\\w+):\\s*(\\d+)\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+\\d+\\s+(\\d+).+$");
-
- private static final Pattern MEMORY_PATTERN = Pattern.compile("^\\w+:\\s*(\\d+)\\s+kB$");
-
- private static final int PERCENT = 100;
-
- private final String instancePath;
-
- private long lastTimestamp = 0;
-
- // CPU related variables
- private long lastCpuUser = 0;
-
- private long lastCpuNice = 0;
-
- private long lastCpuSys = 0;
-
- private long lastCpuIdle = 0;
-
- private long lastCpuIOWait = 0;
-
- private long lastCpuIrq = 0;
-
- private long lastCpuSoftirq = 0;
-
- // Network related variables
- private long lastReceivedBytes = 0;
-
- private long lastTramsmittedBytes = 0;
-
- private long firstTimestamp;
-
- public InstanceProfiler(String instancePath)
- throws ProfilingException {
-
- this.instancePath = instancePath;
- this.firstTimestamp = System.currentTimeMillis();
- // Initialize counters by calling generateProfilingData once and ignore the return value
- generateProfilingData(this.firstTimestamp);
- }
-
- public InternalInstanceProfilingData generateProfilingData(long timestamp) throws ProfilingException {
-
- final long profilingInterval = timestamp - lastTimestamp;
-
- final InternalInstanceProfilingData profilingData = new InternalInstanceProfilingData(
- this.instancePath, (int) profilingInterval);
-
- updateCPUUtilization(profilingData);
- updateMemoryUtilization(profilingData);
- updateNetworkUtilization(profilingData);
-
- // Update timestamp
- this.lastTimestamp = timestamp;
-
- return profilingData;
- }
-
- private void updateMemoryUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException {
-
- BufferedReader in = null;
-
- try {
-
- in = new BufferedReader(new FileReader(PROC_MEMINFO));
-
- long freeMemory = 0;
- long totalMemory = 0;
- long bufferedMemory = 0;
- long cachedMemory = 0;
- long cachedSwapMemory = 0;
-
- int count = 0;
- String output;
- while ((output = in.readLine()) != null) {
-
- switch (count) {
- case 0: // Total memory
- totalMemory = extractMemoryValue(output);
- break;
- case 1: // Free memory
- freeMemory = extractMemoryValue(output);
- break;
- case 2: // Buffers
- bufferedMemory = extractMemoryValue(output);
- break;
- case 3: // Cache
- cachedMemory = extractMemoryValue(output);
- break;
- case 4:
- cachedSwapMemory = extractMemoryValue(output);
- break;
- default:
- break;
- }
-
- ++count;
- }
-
- profilingData.setTotalMemory(totalMemory);
- profilingData.setFreeMemory(freeMemory);
- profilingData.setBufferedMemory(bufferedMemory);
- profilingData.setCachedMemory(cachedMemory);
- profilingData.setCachedSwapMemory(cachedSwapMemory);
-
- } catch (IOException ioe) {
- throw new ProfilingException("Error while reading network utilization: "
- + StringUtils.stringifyException(ioe));
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- }
- }
- }
-
- }
-
- private long extractMemoryValue(String line) throws ProfilingException {
-
- final Matcher matcher = MEMORY_PATTERN.matcher(line);
- if (!matcher.matches()) {
- throw new ProfilingException("Cannot extract memory data for profiling from line " + line);
- }
-
- return Long.parseLong(matcher.group(1));
- }
-
- private void updateNetworkUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException {
-
- BufferedReader in = null;
-
- try {
-
- in = new BufferedReader(new FileReader(PROC_NET_DEV));
-
- long receivedSum = 0;
- long transmittedSum = 0;
-
- String output;
- while ((output = in.readLine()) != null) {
- final Matcher networkMatcher = NETWORK_PATTERN.matcher(output);
- if (!networkMatcher.matches()) {
- continue;
- }
- /*
- * Extract information according to
- * http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html
- */
-
- if (LOOPBACK_INTERFACE_NAME.equals(networkMatcher.group(1))) {
- continue;
- }
-
- receivedSum += Long.parseLong(networkMatcher.group(2));
- transmittedSum += Long.parseLong(networkMatcher.group(3));
- }
-
- in.close();
- in = null;
-
- profilingData.setReceivedBytes(receivedSum - this.lastReceivedBytes);
- profilingData.setTransmittedBytes(transmittedSum - this.lastTramsmittedBytes);
-
- // Store values for next call
- this.lastReceivedBytes = receivedSum;
- this.lastTramsmittedBytes = transmittedSum;
-
- } catch (IOException ioe) {
- throw new ProfilingException("Error while reading network utilization: "
- + StringUtils.stringifyException(ioe));
- } catch (NumberFormatException nfe) {
- throw new ProfilingException("Error while reading network utilization: "
- + StringUtils.stringifyException(nfe));
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- }
- }
- }
- }
-
- private void updateCPUUtilization(InternalInstanceProfilingData profilingData) throws ProfilingException {
-
- BufferedReader in = null;
-
- try {
-
- in = new BufferedReader(new FileReader(PROC_STAT));
- final String output = in.readLine();
- if (output == null) {
- throw new ProfilingException("Cannot read CPU utilization, return value is null");
- }
-
- in.close();
- in = null;
-
- final Matcher cpuMatcher = CPU_PATTERN.matcher(output);
- if (!cpuMatcher.matches()) {
- throw new ProfilingException("Cannot extract CPU utilization from output \"" + output + "\"");
- }
-
- /*
- * Extract the information from the read line according to
- * http://www.linuxhowtos.org/System/procstat.htm
- */
-
- final long cpuUser = Long.parseLong(cpuMatcher.group(1));
- final long cpuNice = Long.parseLong(cpuMatcher.group(2));
- final long cpuSys = Long.parseLong(cpuMatcher.group(3));
- final long cpuIdle = Long.parseLong(cpuMatcher.group(4));
- final long cpuIOWait = Long.parseLong(cpuMatcher.group(5));
- final long cpuIrq = Long.parseLong(cpuMatcher.group(6));
- final long cpuSoftirq = Long.parseLong(cpuMatcher.group(7));
-
- // Calculate deltas
- final long deltaCpuUser = cpuUser - this.lastCpuUser;
- final long deltaCpuNice = cpuNice - this.lastCpuNice;
- final long deltaCpuSys = cpuSys - this.lastCpuSys;
- final long deltaCpuIdle = cpuIdle - this.lastCpuIdle;
- final long deltaCpuIOWait = cpuIOWait - this.lastCpuIOWait;
- final long deltaCpuIrq = cpuIrq - this.lastCpuIrq;
- final long deltaCpuSoftirq = cpuSoftirq - this.lastCpuSoftirq;
- final long deltaSum = deltaCpuUser + deltaCpuNice + deltaCpuSys + deltaCpuIdle + deltaCpuIOWait
- + deltaCpuIrq + deltaCpuSoftirq;
-
- // TODO: Fix deltaSum = 0 situation
-
- // Set the percentage values for the profiling data object
- /*
- * profilingData.setIdleCPU((int)((deltaCpuIdle*PERCENT)/deltaSum));
- * profilingData.setUserCPU((int)((deltaCpuUser*PERCENT)/deltaSum));
- * profilingData.setSystemCPU((int)((deltaCpuSys*PERCENT)/deltaSum));
- * profilingData.setIoWaitCPU((int)((deltaCpuIOWait*PERCENT)/deltaSum));
- * profilingData.setHardIrqCPU((int)((deltaCpuIrq*PERCENT)/deltaSum));
- * profilingData.setSoftIrqCPU((int)((deltaCpuSoftirq*PERCENT)/deltaSum));
- */
- // TODO: bad quick fix
- if (deltaSum > 0) {
- profilingData.setIdleCPU((int) ((deltaCpuIdle * PERCENT) / deltaSum));
- profilingData.setUserCPU((int) ((deltaCpuUser * PERCENT) / deltaSum));
- profilingData.setSystemCPU((int) ((deltaCpuSys * PERCENT) / deltaSum));
- profilingData.setIoWaitCPU((int) ((deltaCpuIOWait * PERCENT) / deltaSum));
- profilingData.setHardIrqCPU((int) ((deltaCpuIrq * PERCENT) / deltaSum));
- profilingData.setSoftIrqCPU((int) ((deltaCpuSoftirq * PERCENT) / deltaSum));
- } else {
- profilingData.setIdleCPU((int) (deltaCpuIdle));
- profilingData.setUserCPU((int) (deltaCpuUser));
- profilingData.setSystemCPU((int) (deltaCpuSys));
- profilingData.setIoWaitCPU((int) (deltaCpuIOWait));
- profilingData.setHardIrqCPU((int) (deltaCpuIrq));
- profilingData.setSoftIrqCPU((int) (deltaCpuSoftirq));
- }
- // Store values for next call
- this.lastCpuUser = cpuUser;
- this.lastCpuNice = cpuNice;
- this.lastCpuSys = cpuSys;
- this.lastCpuIdle = cpuIdle;
- this.lastCpuIOWait = cpuIOWait;
- this.lastCpuIrq = cpuIrq;
- this.lastCpuSoftirq = cpuSoftirq;
-
- } catch (IOException ioe) {
- throw new ProfilingException("Error while reading CPU utilization: " + StringUtils.stringifyException(ioe));
- } catch (NumberFormatException nfe) {
- throw new ProfilingException("Error while reading CPU utilization: " + StringUtils.stringifyException(nfe));
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- }
- }
- }
-
- }
-
- /**
- * @return InternalInstanceProfilingData ProfilingData for the instance from execution-start to currentTime
- * @throws ProfilingException
- */
- public InternalInstanceProfilingData generateCheckpointProfilingData() throws ProfilingException {
- final long profilingInterval = System.currentTimeMillis() - this.firstTimestamp;
-
- final InternalInstanceProfilingData profilingData = new InternalInstanceProfilingData(
- this.instancePath, (int) profilingInterval);
-
- updateCPUUtilization(profilingData);
- updateMemoryUtilization(profilingData);
- updateNetworkUtilization(profilingData);
-
- return profilingData;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/JobProfilingData.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/JobProfilingData.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/JobProfilingData.java
deleted file mode 100644
index ce7db25..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/JobProfilingData.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.impl;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.flink.runtime.executiongraph.ExecutionGraph;
-import org.apache.flink.runtime.executiongraph.ExecutionVertex;
-import org.apache.flink.runtime.instance.Instance;
-import org.apache.flink.runtime.instance.SimpleSlot;
-import org.apache.flink.runtime.profiling.impl.types.InternalInstanceProfilingData;
-import org.apache.flink.runtime.profiling.types.InstanceSummaryProfilingEvent;
-
-public class JobProfilingData {
-
- private final ExecutionGraph executionGraph;
-
- private final long profilingStart;
-
- private final Map<String, InternalInstanceProfilingData> collectedInstanceProfilingData = new
- HashMap<String, InternalInstanceProfilingData>();
-
-
- public JobProfilingData(ExecutionGraph executionGraph) {
- this.executionGraph = executionGraph;
- this.profilingStart = System.currentTimeMillis();
- }
-
-
- public long getProfilingStart() {
- return this.profilingStart;
- }
-
- public ExecutionGraph getExecutionGraph() {
- return this.executionGraph;
- }
-
- public boolean addIfInstanceIsAllocatedByJob(InternalInstanceProfilingData instanceProfilingData) {
-
- for (ExecutionVertex executionVertex : this.executionGraph.getAllExecutionVertices()) {
- SimpleSlot slot = executionVertex.getCurrentAssignedResource();
- if (slot != null && slot.getInstance().getPath().equals(
- instanceProfilingData.getInstancePath()))
- {
- this.collectedInstanceProfilingData.put(instanceProfilingData.getInstancePath(), instanceProfilingData);
- return true;
- }
- }
-
- return false;
- }
-
- public InstanceSummaryProfilingEvent getInstanceSummaryProfilingData(long timestamp) {
-
- final Set<Instance> tempSet = new HashSet<Instance>();
-
- for (ExecutionVertex executionVertex : this.executionGraph.getAllExecutionVertices()) {
- SimpleSlot slot = executionVertex.getCurrentAssignedResource();
- if (slot != null) {
- tempSet.add(slot.getInstance());
- }
- }
-
- // Now compare the size of the collected data set and the allocated instance set.
- // If their sizes are equal we can issue an instance summary.
- if (tempSet.size() != this.collectedInstanceProfilingData.size()) {
- return null;
- }
-
- return constructInstanceSummary(timestamp);
- }
-
- private InstanceSummaryProfilingEvent constructInstanceSummary(long timestamp) {
-
- final int numberOfInstances = this.collectedInstanceProfilingData.size();
-
- final Iterator<String> instanceIterator = this.collectedInstanceProfilingData.keySet().iterator();
-
- long freeMemorySum = 0;
- long totalMemorySum = 0;
- long bufferedMemorySum = 0;
- long cachedMemorySum = 0;
- long cachedSwapMemorySum = 0;
-
- int ioWaitCPUSum = 0;
- int idleCPUSum = 0;
- int profilingIntervalSum = 0;
- int systemCPUSum = 0;
- int hardIrqCPUSum = 0;
- int softIrqCPUSum = 0;
-
- int userCPUSum = 0;
- long receivedBytesSum = 0;
- long transmittedBytesSum = 0;
-
- // Sum up the individual values
- while (instanceIterator.hasNext()) {
-
- final InternalInstanceProfilingData profilingData = this.collectedInstanceProfilingData.get(instanceIterator.next());
-
- freeMemorySum += profilingData.getFreeMemory();
- ioWaitCPUSum += profilingData.getIOWaitCPU();
- idleCPUSum += profilingData.getIdleCPU();
- profilingIntervalSum += profilingData.getProfilingInterval();
- systemCPUSum += profilingData.getSystemCPU();
- hardIrqCPUSum += profilingData.getHardIrqCPU();
- softIrqCPUSum += profilingData.getSoftIrqCPU();
- totalMemorySum += profilingData.getTotalMemory();
- userCPUSum += profilingData.getUserCPU();
- receivedBytesSum += profilingData.getReceivedBytes();
- transmittedBytesSum += profilingData.getTransmittedBytes();
- bufferedMemorySum += profilingData.getBufferedMemory();
- cachedMemorySum += profilingData.getCachedMemory();
- cachedSwapMemorySum += profilingData.getCachedSwapMemory();
- }
-
- final InstanceSummaryProfilingEvent instanceSummary = new InstanceSummaryProfilingEvent(
- profilingIntervalSum / numberOfInstances,
- ioWaitCPUSum / numberOfInstances,
- idleCPUSum / numberOfInstances,
- userCPUSum / numberOfInstances,
- systemCPUSum / numberOfInstances,
- hardIrqCPUSum / numberOfInstances,
- softIrqCPUSum / numberOfInstances,
- totalMemorySum / numberOfInstances,
- freeMemorySum / numberOfInstances,
- bufferedMemorySum / numberOfInstances,
- cachedMemorySum / numberOfInstances,
- cachedSwapMemorySum / numberOfInstances,
- receivedBytesSum / numberOfInstances,
- transmittedBytesSum / numberOfInstances,
- this.executionGraph.getJobID(), timestamp, (timestamp - this.profilingStart));
-
- this.collectedInstanceProfilingData.clear();
-
- return instanceSummary;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexProfilingData.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexProfilingData.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexProfilingData.java
deleted file mode 100644
index 98ccfcf..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexProfilingData.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.impl.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-
-public abstract class InternalExecutionVertexProfilingData implements InternalProfilingData {
-
- private final JobID jobId;
-
- private final JobVertexID vertexId;
-
- private int subtask;
-
- private final ExecutionAttemptID executionId;
-
-
- public InternalExecutionVertexProfilingData() {
- this.jobId = new JobID();
- this.vertexId = new JobVertexID();
- this.executionId = new ExecutionAttemptID();
- this.subtask = -1;
- }
-
-
- public InternalExecutionVertexProfilingData(JobID jobId, JobVertexID vertexId, int subtask, ExecutionAttemptID executionId) {
- this.jobId = jobId;
- this.vertexId = vertexId;
- this.subtask = subtask;
- this.executionId = executionId;
- }
-
- // --------------------------------------------------------------------------------------------
-
- public JobID getJobID() {
- return this.jobId;
- }
-
- public JobVertexID getVertexId() {
- return vertexId;
- }
-
- public int getSubtask() {
- return subtask;
- }
-
- public ExecutionAttemptID getExecutionAttemptId() {
- return this.executionId;
- }
-
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void read(DataInputView in) throws IOException {
- this.jobId.read(in);
- this.vertexId.read(in);
- this.executionId.read(in);
- this.subtask = in.readInt();
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
- this.jobId.write(out);
- this.vertexId.write(out);
- this.executionId.write(out);
- out.writeInt(this.subtask);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexThreadProfilingData.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexThreadProfilingData.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexThreadProfilingData.java
deleted file mode 100644
index 92ce916..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalExecutionVertexThreadProfilingData.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.impl.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-
-public class InternalExecutionVertexThreadProfilingData extends InternalExecutionVertexProfilingData {
-
- private int profilingInterval = 0;
-
- private int userTime = 0;
-
- private int systemTime = 0;
-
- private int blockedTime = 0;
-
- private int waitedTime = 0;
-
- public InternalExecutionVertexThreadProfilingData(JobID jobID, JobVertexID vertexId, int subtask, ExecutionAttemptID executionId,
- int profilingInterval, int userTime, int systemTime, int blockedTime, int waitedTime)
- {
- super(jobID, vertexId, subtask, executionId);
-
- this.profilingInterval = profilingInterval;
- this.userTime = userTime;
- this.systemTime = systemTime;
- this.blockedTime = blockedTime;
- this.waitedTime = waitedTime;
- }
-
- public InternalExecutionVertexThreadProfilingData() {}
-
- @Override
- public void read(DataInputView in) throws IOException {
- super.read(in);
-
- this.profilingInterval = in.readInt();
- this.userTime = in.readInt();
- this.systemTime = in.readInt();
- this.blockedTime = in.readInt();
- this.waitedTime = in.readInt();
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
- super.write(out);
-
- out.writeInt(this.profilingInterval);
- out.writeInt(this.userTime);
- out.writeInt(this.systemTime);
- out.writeInt(this.blockedTime);
- out.writeInt(this.waitedTime);
- }
-
- public int getBlockedTime() {
- return this.blockedTime;
- }
-
- public int getProfilingInterval() {
- return this.profilingInterval;
- }
-
- public int getSystemTime() {
- return this.systemTime;
- }
-
- public int getUserTime() {
- return this.userTime;
- }
-
- public int getWaitedTime() {
- return this.waitedTime;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalInstanceProfilingData.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalInstanceProfilingData.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalInstanceProfilingData.java
deleted file mode 100644
index 145d11c..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalInstanceProfilingData.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.impl.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-
-public class InternalInstanceProfilingData implements InternalProfilingData {
-
- private String instancePath;
-
- private int profilingInterval;
-
- private int ioWaitCPU;
-
- private int idleCPU;
-
- private int userCPU;
-
- private int systemCPU;
-
- private int hardIrqCPU;
-
- private int softIrqCPU;
-
- private long totalMemory;
-
- private long freeMemory;
-
- private long bufferedMemory;
-
- private long cachedMemory;
-
- private long cachedSwapMemory;
-
- private long receivedBytes;
-
- private long transmittedBytes;
-
- public InternalInstanceProfilingData() {
- this.freeMemory = -1;
- this.ioWaitCPU = -1;
- this.idleCPU = -1;
- this.instancePath = "";
- this.profilingInterval = -1;
- this.systemCPU = -1;
- this.totalMemory = -1;
- this.bufferedMemory = -1;
- this.cachedMemory = -1;
- this.cachedSwapMemory = -1;
- this.userCPU = -1;
- this.receivedBytes = -1;
- this.transmittedBytes = -1;
- }
-
- public InternalInstanceProfilingData(String instancePath, int profilingInterval) {
-
- this.instancePath = instancePath;
- this.profilingInterval = profilingInterval;
- this.freeMemory = -1;
- this.ioWaitCPU = -1;
- this.idleCPU = -1;
- this.systemCPU = -1;
- this.totalMemory = -1;
- this.bufferedMemory = -1;
- this.cachedMemory = -1;
- this.cachedSwapMemory = -1;
- this.userCPU = -1;
- this.receivedBytes = -1;
- this.transmittedBytes = -1;
- }
-
- public long getFreeMemory() {
- return this.freeMemory;
- }
-
- public int getIOWaitCPU() {
- return this.ioWaitCPU;
- }
-
- public int getIdleCPU() {
- return this.idleCPU;
- }
-
- public int getHardIrqCPU() {
- return this.hardIrqCPU;
- }
-
- public int getSoftIrqCPU() {
- return this.softIrqCPU;
- }
-
- public String getInstancePath() {
- return this.instancePath;
- }
-
- public int getProfilingInterval() {
- return this.profilingInterval;
- }
-
- public int getSystemCPU() {
- return this.systemCPU;
- }
-
- public long getTotalMemory() {
- return this.totalMemory;
- }
-
- public long getBufferedMemory() {
- return this.bufferedMemory;
- }
-
- public long getCachedMemory() {
- return this.cachedMemory;
- }
-
- public long getCachedSwapMemory() {
- return this.cachedSwapMemory;
- }
-
- public int getUserCPU() {
- return this.userCPU;
- }
-
- public long getReceivedBytes() {
- return this.receivedBytes;
- }
-
- public long getTransmittedBytes() {
- return this.transmittedBytes;
- }
-
- @Override
- public void read(DataInputView in) throws IOException {
-
- this.freeMemory = in.readLong();
- this.ioWaitCPU = in.readInt();
- this.idleCPU = in.readInt();
- this.instancePath = in.readUTF();
- this.profilingInterval = in.readInt();
- this.systemCPU = in.readInt();
- this.totalMemory = in.readLong();
- this.bufferedMemory = in.readLong();
- this.cachedMemory = in.readLong();
- this.cachedSwapMemory = in.readLong();
- this.userCPU = in.readInt();
- this.receivedBytes = in.readLong();
- this.transmittedBytes = in.readLong();
- this.hardIrqCPU = in.readInt();
- this.softIrqCPU = in.readInt();
-
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
-
- out.writeLong(this.freeMemory);
- out.writeInt(this.ioWaitCPU);
- out.writeInt(this.idleCPU);
- out.writeUTF(instancePath);
- out.writeInt(this.profilingInterval);
- out.writeInt(this.systemCPU);
- out.writeLong(this.totalMemory);
- out.writeLong(this.bufferedMemory);
- out.writeLong(this.cachedMemory);
- out.writeLong(this.cachedSwapMemory);
- out.writeInt(this.userCPU);
- out.writeLong(this.receivedBytes);
- out.writeLong(this.transmittedBytes);
- out.writeInt(this.hardIrqCPU);
- out.writeInt(this.softIrqCPU);
-
- }
-
- public void setFreeMemory(long freeMemory) {
- this.freeMemory = freeMemory;
- }
-
- public void setIoWaitCPU(int ioWaitCPU) {
- this.ioWaitCPU = ioWaitCPU;
- }
-
- public void setIdleCPU(int idleCPU) {
- this.idleCPU = idleCPU;
- }
-
- public void setSystemCPU(int systemCPU) {
- this.systemCPU = systemCPU;
- }
-
- public void setHardIrqCPU(int hardIrqCPU) {
- this.hardIrqCPU = hardIrqCPU;
- }
-
- public void setSoftIrqCPU(int softIrqCPU) {
- this.softIrqCPU = softIrqCPU;
- }
-
- public void setTotalMemory(long totalMemory) {
- this.totalMemory = totalMemory;
- }
-
- public void setBufferedMemory(long bufferedMemory) {
- this.bufferedMemory = bufferedMemory;
- }
-
- public void setCachedMemory(long cachedMemory) {
- this.cachedMemory = cachedMemory;
- }
-
- public void setCachedSwapMemory(long cachedSwapMemory) {
- this.cachedSwapMemory = cachedSwapMemory;
- }
-
- public void setUserCPU(int userCPU) {
- this.userCPU = userCPU;
- }
-
- public void setReceivedBytes(long receivedBytes) {
- this.receivedBytes = receivedBytes;
- }
-
- public void setTransmittedBytes(long transmittedBytes) {
- this.transmittedBytes = transmittedBytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalProfilingData.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalProfilingData.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalProfilingData.java
deleted file mode 100644
index c6f53d1..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/InternalProfilingData.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.impl.types;
-
-import org.apache.flink.core.io.IOReadableWritable;
-
-public interface InternalProfilingData extends IOReadableWritable {}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/ProfilingDataContainer.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/ProfilingDataContainer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/ProfilingDataContainer.java
deleted file mode 100644
index b233b84..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/impl/types/ProfilingDataContainer.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.impl.types;
-
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Iterator;
-import java.util.Queue;
-
-import org.apache.flink.core.io.IOReadableWritable;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.util.StringUtils;
-
-public class ProfilingDataContainer implements IOReadableWritable {
-
- private final Queue<InternalProfilingData> queuedProfilingData = new ArrayDeque<InternalProfilingData>();
-
- public void addProfilingData(InternalProfilingData profilingData) {
-
- if (profilingData == null) {
- return;
- }
-
- this.queuedProfilingData.add(profilingData);
- }
-
- public void clear() {
- this.queuedProfilingData.clear();
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public void read(DataInputView in) throws IOException {
-
- final int numberOfRecords = in.readInt();
- for (int i = 0; i < numberOfRecords; i++) {
- final String className = StringUtils.readNullableString(in);
-
- Class<? extends InternalProfilingData> clazz;
- try {
- clazz = (Class<? extends InternalProfilingData>) Class.forName(className);
- } catch (Exception e) {
- throw new IOException(e);
- }
-
- InternalProfilingData profilingData ;
- try {
- profilingData = clazz.newInstance();
- } catch (Exception e) {
- throw new IOException(e);
- }
-
- // Restore internal state
- profilingData.read(in);
-
- this.queuedProfilingData.add(profilingData);
- }
- }
-
- public int size() {
- return this.queuedProfilingData.size();
- }
-
- public boolean isEmpty() {
- return this.queuedProfilingData.isEmpty();
- }
-
- public Iterator<InternalProfilingData> getIterator() {
- return this.queuedProfilingData.iterator();
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
-
- // Write the number of records
- out.writeInt(this.queuedProfilingData.size());
- // Write the records themselves
- for (InternalProfilingData profilingData : this.queuedProfilingData) {
- StringUtils.writeNullableString(profilingData.getClass().getName(), out);
- profilingData.write(out);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceProfilingEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceProfilingEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceProfilingEvent.java
deleted file mode 100644
index 7de9252..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceProfilingEvent.java
+++ /dev/null
@@ -1,399 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.api.common.JobID;
-
-/**
- * Instance profiling events are a special subclass of profiling events. They contain profiling information about the
- * utilization of a particular instance during a job execution.
- */
-public abstract class InstanceProfilingEvent extends ProfilingEvent {
-
- private static final long serialVersionUID = 5964092674722506040L;
-
- /**
- * The interval of time this profiling event covers in milliseconds.
- */
- private int profilingInterval;
-
- /**
- * The percentage of time the CPU(s) spent in state IOWAIT during the profiling interval.
- */
- private int ioWaitCPU;
-
- /**
- * The percentage of time the CPU(s) spent in state IDLE during the profiling interval.
- */
- private int idleCPU;
-
- /**
- * The percentage of time the CPU(s) spent in state USER during the profiling interval.
- */
- private int userCPU;
-
- /**
- * The percentage of time the CPU(s) spent in state SYSTEM during the profiling interval.
- */
- private int systemCPU;
-
- /**
- * The percentage of time the CPU(s) spent in state HARD_IRQ during the profiling interval.
- */
- private int hardIrqCPU;
-
- /**
- * The percentage of time the CPU(s) spent in state SOFT_IRQ during the profiling interval.
- */
- private int softIrqCPU;
-
- /**
- * The total amount of this instance's main memory in bytes.
- */
- private long totalMemory;
-
- /**
- * The free amount of this instance's main memory in bytes.
- */
- private long freeMemory;
-
- /**
- * The amount of main memory the instance uses for file buffers.
- */
- private long bufferedMemory;
-
- /**
- * The amount of main memory the instance uses as cache memory.
- */
- private long cachedMemory;
-
- /**
- * The amount of main memory the instance uses for cached swaps.
- */
- private long cachedSwapMemory;
-
- /**
- * The number of bytes received via network during the profiling interval.
- */
- private long receivedBytes;
-
- /**
- * The number of bytes transmitted via network during the profiling interval.
- */
- private long transmittedBytes;
-
- /**
- * Constructs a new instance profiling event.
- *
- * @param profilingInterval
- * the interval of time this profiling event covers in milliseconds
- * @param ioWaitCPU
- * the percentage of time the CPU(s) spent in state IOWAIT during the profiling interval
- * @param idleCPU
- * the percentage of time the CPU(s) spent in state IDLE during the profiling interval
- * @param userCPU
- * the percentage of time the CPU(s) spent in state USER during the profiling interval
- * @param systemCPU
- * the percentage of time the CPU(s) spent in state SYSTEM during the profiling interval
- * @param hardIrqCPU
- * the percentage of time the CPU(s) spent in state HARD_IRQ during the profiling interval
- * @param softIrqCPU
- * the percentage of time the CPU(s) spent in state SOFT_IRQ during the profiling interval
- * @param totalMemory
- * the total amount of this instance's main memory in bytes
- * @param freeMemory
- * the free amount of this instance's main memory in bytes
- * @param bufferedMemory
- * the amount of main memory the instance uses for file buffers
- * @param cachedMemory
- * the amount of main memory the instance uses as cache memory
- * @param cachedSwapMemory
- * The amount of main memory the instance uses for cached swaps
- * @param receivedBytes
- * the number of bytes received via network during the profiling interval
- * @param transmittedBytes
- * the number of bytes transmitted via network during the profiling interval
- * @param jobID
- * the ID of this job this profiling event belongs to
- * @param timestamp
- * the time stamp of this profiling event's creation
- * @param profilingTimestamp
- * the time stamp relative to the beginning of the job's execution
- */
- public InstanceProfilingEvent(final int profilingInterval, final int ioWaitCPU, final int idleCPU,
- final int userCPU, final int systemCPU, final int hardIrqCPU, final int softIrqCPU, final long totalMemory,
- final long freeMemory, final long bufferedMemory, final long cachedMemory, final long cachedSwapMemory,
- final long receivedBytes, final long transmittedBytes, final JobID jobID, final long timestamp,
- final long profilingTimestamp) {
-
- super(jobID, timestamp, profilingTimestamp);
-
- this.profilingInterval = profilingInterval;
-
- this.ioWaitCPU = ioWaitCPU;
- this.idleCPU = idleCPU;
- this.userCPU = userCPU;
- this.systemCPU = systemCPU;
- this.hardIrqCPU = hardIrqCPU;
- this.softIrqCPU = softIrqCPU;
-
- this.totalMemory = totalMemory;
- this.freeMemory = freeMemory;
- this.bufferedMemory = bufferedMemory;
- this.cachedMemory = cachedMemory;
- this.cachedSwapMemory = cachedSwapMemory;
-
- this.receivedBytes = receivedBytes;
- this.transmittedBytes = transmittedBytes;
- }
-
- /**
- * Default constructor for serialization/deserialization.
- */
- public InstanceProfilingEvent() {
- super();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Returns the interval of time this profiling event covers in milliseconds.
- *
- * @return the interval of time this profiling event covers in milliseconds
- */
- public final int getProfilingInterval() {
- return this.profilingInterval;
- }
-
- /**
- * Returns the total amount of memory of the corresponding instance.
- *
- * @return the total amount of memory in bytes
- */
- public final long getTotalMemory() {
- return this.totalMemory;
- }
-
- /**
- * Returns the amount of free memory of the corresponding instance.
- *
- * @return the amount of free memory in bytes.
- */
- public final long getFreeMemory() {
- return this.freeMemory;
- }
-
- /**
- * Returns the amount of memory, in bytes, used for file buffers.
- *
- * @return the amount of memory used for file buffers in bytes
- */
- public final long getBufferedMemory() {
- return this.bufferedMemory;
- }
-
- /**
- * Returns the amount of memory, in bytes, used as cache memory.
- *
- * @return the amount of memory used as cache memory in bytes
- */
- public final long getCachedMemory() {
- return this.cachedMemory;
- }
-
- /**
- * Returns the amount of swap, in bytes, used as cache memory.
- *
- * @return the amount of, in bytes, used as cache memory
- */
- public final long getCachedSwapMemory() {
- return this.cachedSwapMemory;
- }
-
- /**
- * Returns the percentage of time the CPU(s) spent in state USER during the profiling interval.
- *
- * @return the percentage of time the CPU(s) spent in state USER during the profiling interval
- */
- public final int getUserCPU() {
- return this.userCPU;
- }
-
- /**
- * Returns the percentage of time the CPU(s) spent in state SYSTEM during the profiling interval.
- *
- * @return the percentage of time the CPU(s) spent in state SYSTEM during the profiling interval
- */
- public final int getSystemCPU() {
- return this.systemCPU;
- }
-
- /**
- * Returns the percentage of time the CPU(s) spent in state IDLE during the profiling interval. Prior to Linux
- * 2.5.41, this includes IO-wait time.
- *
- * @return the percentage of time the CPU(s) spent in state IDLE during the profiling interval
- */
- public final int getIdleCPU() {
- return this.idleCPU;
- }
-
- /**
- * Returns the percentage of time the CPU(s) spent in state IOWAIT during the profiling interval. Prior to Linux
- * 2.5.41, included in idle.
- *
- * @return the percentage of time the CPU(s) spent in state IOWAIT during the profiling interval.
- */
- public final int getIOWaitCPU() {
- return this.ioWaitCPU;
- }
-
- /**
- * Returns the percentage of time the CPU(s) spent in state HARD_IRQ during the profiling interval.
- *
- * @return the percentage of time the CPU(s) spent in state HARD_IRQ during the profiling interval
- */
- public final int getHardIrqCPU() {
- return this.hardIrqCPU;
- }
-
- /**
- * Returns the percentage of time the CPU(s) spent in state SOFT_IRQ during the profiling interval.
- *
- * @return the percentage of time the CPU(s) spent in state SOFT_IRQ during the profiling interval
- */
- public final int getSoftIrqCPU() {
- return this.softIrqCPU;
- }
-
- /**
- * Returns the number of bytes received via network during the profiling interval.
- *
- * @return the number of bytes received via network during the profiling interval
- */
- public final long getReceivedBytes() {
- return this.receivedBytes;
- }
-
- /**
- * Returns the number of bytes transmitted via network during the profiling interval.
- *
- * @return the number of bytes transmitted via network during the profiling interval
- */
- public final long getTransmittedBytes() {
- return this.transmittedBytes;
- }
-
- // --------------------------------------------------------------------------------------------
- // Serialization
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void read(final DataInputView in) throws IOException {
- super.read(in);
-
- this.profilingInterval = in.readInt();
-
- this.ioWaitCPU = in.readInt();
- this.idleCPU = in.readInt();
- this.userCPU = in.readInt();
- this.systemCPU = in.readInt();
- this.hardIrqCPU = in.readInt();
- this.softIrqCPU = in.readInt();
-
- this.totalMemory = in.readLong();
- this.freeMemory = in.readLong();
- this.bufferedMemory = in.readLong();
- this.cachedMemory = in.readLong();
- this.cachedSwapMemory = in.readLong();
-
- this.receivedBytes = in.readLong();
- this.transmittedBytes = in.readLong();
- }
-
-
- @Override
- public void write(final DataOutputView out) throws IOException {
- super.write(out);
-
- out.writeInt(this.profilingInterval);
-
- out.writeInt(this.ioWaitCPU);
- out.writeInt(this.idleCPU);
- out.writeInt(this.userCPU);
- out.writeInt(this.systemCPU);
- out.writeInt(this.hardIrqCPU);
- out.writeInt(this.softIrqCPU);
-
- out.writeLong(totalMemory);
- out.writeLong(freeMemory);
- out.writeLong(bufferedMemory);
- out.writeLong(cachedMemory);
- out.writeLong(cachedSwapMemory);
-
- out.writeLong(receivedBytes);
- out.writeLong(transmittedBytes);
- }
-
-
- // --------------------------------------------------------------------------------------------
- // Utilities
- // --------------------------------------------------------------------------------------------
-
- @Override
- public boolean equals(final Object obj) {
- if (obj instanceof InstanceProfilingEvent) {
- final InstanceProfilingEvent other = (InstanceProfilingEvent) obj;
-
- return super.equals(obj) &&
- profilingInterval == other.profilingInterval &&
- ioWaitCPU == other.ioWaitCPU &&
- idleCPU == other.idleCPU &&
- userCPU == other.userCPU &&
- systemCPU == other.systemCPU &&
- hardIrqCPU == other.hardIrqCPU &&
- softIrqCPU == other.softIrqCPU &&
- totalMemory == other.totalMemory &&
- freeMemory == other.freeMemory &&
- bufferedMemory == other.bufferedMemory &&
- cachedMemory == other.cachedMemory &&
- cachedSwapMemory == other.cachedSwapMemory &&
- receivedBytes == other.receivedBytes &&
- transmittedBytes == other.transmittedBytes;
- }
- else {
- return false;
- }
- }
-
-
- @Override
- public int hashCode() {
- long hashCode = getJobID().hashCode() + getTimestamp() + getProfilingTimestamp();
- hashCode += this.profilingInterval + this.ioWaitCPU + this.idleCPU + this.userCPU + this.systemCPU
- + this.hardIrqCPU + this.softIrqCPU;
- hashCode += (this.totalMemory + this.freeMemory + this.bufferedMemory + this.cachedMemory + this.cachedSwapMemory);
- hashCode ^= hashCode >>> 32;
- return (int) hashCode;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceSummaryProfilingEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceSummaryProfilingEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceSummaryProfilingEvent.java
deleted file mode 100644
index 4bb052d..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/InstanceSummaryProfilingEvent.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.types;
-
-import org.apache.flink.api.common.JobID;
-
-/**
- * Instance summary profiling events summarize the profiling events of all instances involved in computing a job.
- */
-public final class InstanceSummaryProfilingEvent extends InstanceProfilingEvent {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Constructs a new instance summary profiling event.
- *
- * @param profilingInterval
- * the interval of time this profiling event covers in milliseconds
- * @param ioWaitCPU
- * the percentage of time the CPU(s) spent in state IOWAIT during the profiling interval
- * @param idleCPU
- * the percentage of time the CPU(s) spent in state IDLE during the profiling interval
- * @param userCPU
- * the percentage of time the CPU(s) spent in state USER during the profiling interval
- * @param systemCPU
- * the percentage of time the CPU(s) spent in state SYSTEM during the profiling interval
- * @param hardIrqCPU
- * the percentage of time the CPU(s) spent in state HARD_IRQ during the profiling interval
- * @param softIrqCPU
- * the percentage of time the CPU(s) spent in state SOFT_IRQ during the profiling interval
- * @param totalMemory
- * the total amount of this instance's main memory in bytes
- * @param freeMemory
- * the free amount of this instance's main memory in bytes
- * @param bufferedMemory
- * the amount of main memory the instance uses for file buffers
- * @param cachedMemory
- * the amount of main memory the instance uses as cache memory
- * @param cachedSwapMemory
- * The amount of main memory the instance uses for cached swaps
- * @param receivedBytes
- * the number of bytes received via network during the profiling interval
- * @param transmittedBytes
- * the number of bytes transmitted via network during the profiling interval
- * @param jobID
- * the ID of this job this profiling event belongs to
- * @param timestamp
- * the time stamp of this profiling event's creation
- * @param profilingTimestamp
- * the time stamp relative to the beginning of the job's execution
- */
- public InstanceSummaryProfilingEvent(final int profilingInterval, final int ioWaitCPU, final int idleCPU,
- final int userCPU, final int systemCPU, final int hardIrqCPU, final int softIrqCPU, final long totalMemory,
- final long freeMemory, final long bufferedMemory, final long cachedMemory, final long cachedSwapMemory,
- final long receivedBytes, final long transmittedBytes, final JobID jobID,
- final long timestamp, final long profilingTimestamp) {
- super(profilingInterval, ioWaitCPU, idleCPU, userCPU, systemCPU, hardIrqCPU, softIrqCPU, totalMemory,
- freeMemory, bufferedMemory, cachedMemory, cachedSwapMemory, receivedBytes, transmittedBytes, jobID,
- timestamp, profilingTimestamp);
- }
-
- /**
- * Default constructor for serialization/deserialization.
- */
- public InstanceSummaryProfilingEvent() {
- super();
- }
-
- @Override
- public boolean equals(Object obj) {
- return (obj instanceof InstanceSummaryProfilingEvent) && super.equals(obj);
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ProfilingEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ProfilingEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ProfilingEvent.java
deleted file mode 100644
index 13a9a99..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ProfilingEvent.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.event.job.AbstractEvent;
-import org.apache.flink.api.common.JobID;
-
-import com.google.common.base.Preconditions;
-
-/**
- * A profiling event is a special type of event. It is intended to transport profiling data of a Nephele job to external
- * components.
- */
-public abstract class ProfilingEvent extends AbstractEvent {
-
- private static final long serialVersionUID = 1L;
-
- /** The ID of the job the profiling data belongs to. */
- private final JobID jobID;
-
- /** The profiling time stamp. */
- private long profilingTimestamp;
-
- /**
- * Constructs a new profiling event.
- *
- * @param jobID
- * the ID of the job this profiling events belongs to
- * @param timestamp
- * the time stamp of the event
- * @param profilingTimestamp
- * the time stamp of the profiling data
- */
- public ProfilingEvent(JobID jobID, long timestamp, long profilingTimestamp) {
- super(timestamp);
-
- Preconditions.checkNotNull(jobID);
- this.jobID = jobID;
- this.profilingTimestamp = profilingTimestamp;
- }
-
- /**
- * Default constructor for serialization/deserialization.
- */
- public ProfilingEvent() {
- super();
- this.jobID = new JobID();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Returns the ID of the job this profiling information belongs to.
- *
- * @return the ID of the job this profiling information belongs to
- */
- public JobID getJobID() {
- return this.jobID;
- }
-
- /**
- * Returns the timestamp of this profiling information. The timestamp denotes
- * the time period in milliseconds between the creation of this profiling information
- * and the start of the corresponding vertex's execution.
- *
- * @return the timestamp of this profiling information.
- */
- public long getProfilingTimestamp() {
- return this.profilingTimestamp;
- }
-
- // --------------------------------------------------------------------------------------------
- // Serialization
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void read(DataInputView in) throws IOException {
- super.read(in);
-
- this.jobID.read(in);
- this.profilingTimestamp = in.readLong();
- }
-
-
- @Override
- public void write(DataOutputView out) throws IOException {
- super.write(out);
-
- this.jobID.write(out);
- out.writeLong(this.profilingTimestamp);
- }
-
- // --------------------------------------------------------------------------------------------
- // Utilities
- // --------------------------------------------------------------------------------------------
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof ProfilingEvent) {
- final ProfilingEvent other = (ProfilingEvent) obj;
-
- return super.equals(obj) && this.profilingTimestamp == other.profilingTimestamp &&
- this.jobID.equals(other.jobID);
- }
- else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return this.jobID.hashCode() ^ ((int) (profilingTimestamp >>> 32)) ^ ((int) (profilingTimestamp)) ^
- super.hashCode();
- }
-}
[7/8] flink git commit: [FLINK-1969] [runtime] Remove deprecated
profiler code
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/SingleInstanceProfilingEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/SingleInstanceProfilingEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/SingleInstanceProfilingEvent.java
deleted file mode 100644
index e07f144..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/SingleInstanceProfilingEvent.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.types.StringValue;
-
-import com.google.common.base.Preconditions;
-
-/**
- * A single instance profiling event encapsulates profiling information for one particular instance.
- */
-public final class SingleInstanceProfilingEvent extends InstanceProfilingEvent {
-
- private static final long serialVersionUID = 1L;
-
- private String instanceName;
-
- /**
- * Constructs a new instance profiling event.
- *
- * @param profilingInterval
- * the interval of time this profiling event covers in milliseconds
- * @param ioWaitCPU
- * the percentage of time the CPU(s) spent in state IOWAIT during the profiling interval
- * @param idleCPU
- * the percentage of time the CPU(s) spent in state IDLE during the profiling interval
- * @param userCPU
- * the percentage of time the CPU(s) spent in state USER during the profiling interval
- * @param systemCPU
- * the percentage of time the CPU(s) spent in state SYSTEM during the profiling interval
- * @param hardIrqCPU
- * the percentage of time the CPU(s) spent in state HARD_IRQ during the profiling interval
- * @param softIrqCPU
- * the percentage of time the CPU(s) spent in state SOFT_IRQ during the profiling interval
- * @param totalMemory
- * the total amount of this instance's main memory in bytes
- * @param freeMemory
- * the free amount of this instance's main memory in bytes
- * @param bufferedMemory
- * the amount of main memory the instance uses for file buffers
- * @param cachedMemory
- * the amount of main memory the instance uses as cache memory
- * @param cachedSwapMemory
- * The amount of main memory the instance uses for cached swaps
- * @param receivedBytes
- * the number of bytes received via network during the profiling interval
- * @param transmittedBytes
- * the number of bytes transmitted via network during the profiling interval
- * @param jobID
- * the ID of this job this profiling event belongs to
- * @param timestamp
- * the time stamp of this profiling event's creation
- * @param profilingTimestamp
- * the time stamp relative to the beginning of the job's execution
- * @param instanceName
- * the name of the instance this profiling event refers to
- */
- public SingleInstanceProfilingEvent(final int profilingInterval, final int ioWaitCPU, final int idleCPU,
- final int userCPU, final int systemCPU, final int hardIrqCPU, final int softIrqCPU, final long totalMemory,
- final long freeMemory, final long bufferedMemory, final long cachedMemory, final long cachedSwapMemory,
- final long receivedBytes, final long transmittedBytes, final JobID jobID, final long timestamp,
- final long profilingTimestamp, final String instanceName)
- {
- super(profilingInterval, ioWaitCPU, idleCPU, userCPU, systemCPU, hardIrqCPU, softIrqCPU, totalMemory,
- freeMemory, bufferedMemory, cachedMemory, cachedSwapMemory, receivedBytes, transmittedBytes, jobID,
- timestamp, profilingTimestamp);
-
- Preconditions.checkNotNull(instanceName);
- this.instanceName = instanceName;
- }
-
- /**
- * Default constructor for serialization/deserialization.
- */
- public SingleInstanceProfilingEvent() {
- super();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Returns the name of the instance.
- *
- * @return the name of the instance
- */
- public String getInstanceName() {
- return this.instanceName;
- }
-
- // --------------------------------------------------------------------------------------------
- // Serialization
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void read(DataInputView in) throws IOException {
- super.read(in);
- this.instanceName = StringValue.readString(in);
- }
-
- @Override
- public void write(DataOutputView out) throws IOException {
- super.write(out);
- StringValue.writeString(this.instanceName, out);
- }
-
- // --------------------------------------------------------------------------------------------
- // Utilities
- // --------------------------------------------------------------------------------------------
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof SingleInstanceProfilingEvent) {
- SingleInstanceProfilingEvent other = (SingleInstanceProfilingEvent) obj;
- return super.equals(obj) && this.instanceName.equals(other.instanceName);
-
- }
- else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return super.hashCode() + 31*instanceName.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ThreadProfilingEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ThreadProfilingEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ThreadProfilingEvent.java
deleted file mode 100644
index 08b932a..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/ThreadProfilingEvent.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-
-/**
- * Through this interface it is possible to access profiling data about the CPU utilization
- * of the corresponding execution thread during its execution.
- */
-public class ThreadProfilingEvent extends VertexProfilingEvent {
-
- private static final long serialVersionUID = -3006867830244444710L;
-
- private int userTime;
-
- private int systemTime;
-
- private int blockedTime;
-
- private int waitedTime;
-
- public ThreadProfilingEvent(int userTime, int systemTime, int blockedTime, int waitedTime,
- JobVertexID vertexId, int subtask, ExecutionAttemptID executionId,
- int profilingInterval, JobID jobID, long timestamp, long profilingTimestamp)
- {
- super(vertexId, subtask, executionId, profilingInterval, jobID, timestamp, profilingTimestamp);
-
- this.userTime = userTime;
- this.systemTime = systemTime;
- this.blockedTime = blockedTime;
- this.waitedTime = waitedTime;
- }
-
- public ThreadProfilingEvent() {
- super();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Returns the percentage of time the execution thread spent in
- * user mode in the given profiling interval.
- *
- * @return the percentage of time spent in user mode
- */
- public int getUserTime() {
- return this.userTime;
- }
-
- /**
- * Returns the percentage of time the execution thread spent in
- * system mode in the given profiling interval.
- *
- * @return the percentage of time spent in system mode
- */
- public int getSystemTime() {
- return this.systemTime;
- }
-
- /**
- * Returns the percentage of time the execution thread has been
- * blocked to enter or reenter a monitor in the given profiling interval.
- *
- * @return the percentage of time the thread has been blocked
- */
- public int getBlockedTime() {
- return this.blockedTime;
- }
-
- /**
- * Returns the percentage of time the execution thread spent in
- * either <code>WAITING</code> or <code>TIMED_WAITING</code> state in the given profiling interval.
- *
- * @return the percentage of time the thread spent waiting
- */
- public int getWaitedTime() {
- return this.waitedTime;
- }
-
- // --------------------------------------------------------------------------------------------
- // Serialization
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void read(DataInputView in) throws IOException {
- super.read(in);
-
- this.userTime = in.readInt();
- this.systemTime = in.readInt();
- this.blockedTime = in.readInt();
- this.waitedTime = in.readInt();
- }
-
-
- @Override
- public void write(DataOutputView out) throws IOException {
- super.write(out);
-
- out.writeInt(this.userTime);
- out.writeInt(this.systemTime);
- out.writeInt(this.blockedTime);
- out.writeInt(this.waitedTime);
- }
-
-
- // --------------------------------------------------------------------------------------------
- // Utilities
- // --------------------------------------------------------------------------------------------
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof ThreadProfilingEvent) {
- final ThreadProfilingEvent other = (ThreadProfilingEvent) obj;
-
- return this.userTime == other.userTime &&
- this.systemTime == other.systemTime &&
- this.blockedTime == other.blockedTime &&
- this.waitedTime == other.waitedTime &&
- super.equals(obj);
- }
- else {
- return false;
- }
- }
-
- // hash code is inherited from the superclass
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/VertexProfilingEvent.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/VertexProfilingEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/VertexProfilingEvent.java
deleted file mode 100644
index fc18758..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/profiling/types/VertexProfilingEvent.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.profiling.types;
-
-import java.io.IOException;
-
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-
-/**
- * This interface is a base interface for profiling data which
- * pertains to the execution of tasks.
- */
-public abstract class VertexProfilingEvent extends ProfilingEvent {
-
- private static final long serialVersionUID = -5364961557518174880L;
-
- private final JobVertexID vertexId;
-
- private int subtask;
-
- private final ExecutionAttemptID executionId;
-
- private int profilingInterval;
-
-
- public VertexProfilingEvent(JobVertexID vertexId, int subtask, ExecutionAttemptID executionId,
- int profilingInterval, JobID jobID, long timestamp, long profilingTimestamp)
- {
- super(jobID, timestamp, profilingTimestamp);
-
- this.vertexId = vertexId;
- this.subtask = subtask;
- this.executionId = executionId;
- this.profilingInterval = profilingInterval;
- }
-
- public VertexProfilingEvent() {
- super();
- this.vertexId = new JobVertexID();
- this.executionId = new ExecutionAttemptID();
- }
-
- // --------------------------------------------------------------------------------------------
-
- /**
- * Returns the ID of the vertex this profiling information
- * belongs to.
- *
- * @return the ID of the vertex this profiling information belongs to
- */
- public JobVertexID getVertexID() {
- return this.vertexId;
- }
-
- /**
- * The interval in milliseconds to which the rest
- * of the profiling data relates to.
- *
- * @return the profiling interval given in milliseconds
- */
- public int getProfilingInterval() {
- return this.profilingInterval;
- }
-
- public int getSubtask() {
- return subtask;
- }
-
- public ExecutionAttemptID getExecutionId() {
- return executionId;
- }
-
- // --------------------------------------------------------------------------------------------
- // Serialization
- // --------------------------------------------------------------------------------------------
-
- @Override
- public void read(DataInputView in) throws IOException {
- super.read(in);
-
- this.vertexId.read(in);
- this.executionId.read(in);
- this.subtask = in.readInt();
- this.profilingInterval = in.readInt();
- }
-
-
- @Override
- public void write(DataOutputView out) throws IOException {
- super.write(out);
-
- this.vertexId.write(out);
- this.executionId.write(out);
- out.writeInt(subtask);
- out.writeInt(this.profilingInterval);
- }
-
- // --------------------------------------------------------------------------------------------
- // Utilities
- // --------------------------------------------------------------------------------------------
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof VertexProfilingEvent) {
- final VertexProfilingEvent other = (VertexProfilingEvent) obj;
-
- return super.equals(other) && this.subtask == other.subtask &&
- this.profilingInterval == other.profilingInterval &&
- this.vertexId.equals(other.vertexId) &&
- this.executionId.equals(other.executionId);
- }
- else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return super.hashCode() ^ vertexId.hashCode() ^ (31*subtask) ^ executionId.hashCode();
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
index 15a9446..75463fd 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManager.scala
@@ -53,11 +53,8 @@ import org.apache.flink.runtime.jobmanager.scheduler.{Scheduler => FlinkSchedule
import org.apache.flink.runtime.messages.JobManagerMessages._
import org.apache.flink.runtime.messages.RegistrationMessages._
import org.apache.flink.runtime.messages.TaskManagerMessages.{SendStackTrace, Heartbeat}
-import org.apache.flink.runtime.profiling.ProfilingUtils
import org.apache.flink.util.{ExceptionUtils, InstantiationUtil}
-import org.slf4j.LoggerFactory
-
import akka.actor._
import scala.concurrent._
@@ -98,7 +95,6 @@ class JobManager(val flinkConfiguration: Configuration,
val libraryCacheManager: BlobLibraryCacheManager,
val archive: ActorRef,
val accumulatorManager: AccumulatorManager,
- val profiler: Option[ActorRef],
val defaultExecutionRetries: Int,
val delayBetweenRetries: Long,
val timeout: FiniteDuration)
@@ -124,7 +120,6 @@ class JobManager(val flinkConfiguration: Configuration,
}
archive ! PoisonPill
- profiler.foreach( ref => ref ! PoisonPill )
for((e,_) <- currentJobs.values) {
e.fail(new Exception("The JobManager is shutting down."))
@@ -935,7 +930,7 @@ object JobManager {
/**
* Create the job manager components as (instanceManager, scheduler, libraryCacheManager,
- * archiverProps, accumulatorManager, profiler, defaultExecutionRetries,
+ * archiverProps, accumulatorManager, defaultExecutionRetries,
* delayBetweenRetries, timeout)
*
* @param configuration The configuration from which to parse the config values.
@@ -943,15 +938,13 @@ object JobManager {
*/
def createJobManagerComponents(configuration: Configuration) :
(InstanceManager, FlinkScheduler, BlobLibraryCacheManager,
- Props, AccumulatorManager, Option[Props], Int, Long, FiniteDuration, Int) = {
+ Props, AccumulatorManager, Int, Long, FiniteDuration, Int) = {
val timeout: FiniteDuration = AkkaUtils.getTimeout(configuration)
val archiveCount = configuration.getInteger(ConfigConstants.JOB_MANAGER_WEB_ARCHIVE_COUNT,
ConfigConstants.DEFAULT_JOB_MANAGER_WEB_ARCHIVE_COUNT)
- val profilingEnabled = configuration.getBoolean(ProfilingUtils.PROFILE_JOB_KEY, false)
-
val cleanupInterval = configuration.getLong(
ConfigConstants.LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL,
ConfigConstants.DEFAULT_LIBRARY_CACHE_MANAGER_CLEANUP_INTERVAL) * 1000
@@ -978,12 +971,6 @@ object JobManager {
val archiveProps: Props = Props(classOf[MemoryArchivist], archiveCount)
- val profilerProps: Option[Props] = if (profilingEnabled) {
- Some(Props(classOf[JobManagerProfiler]))
- } else {
- None
- }
-
val accumulatorManager: AccumulatorManager = new AccumulatorManager(Math.min(1, archiveCount))
var blobServer: BlobServer = null
@@ -1018,7 +1005,7 @@ object JobManager {
}
(instanceManager, scheduler, libraryCacheManager, archiveProps, accumulatorManager,
- profilerProps, executionRetries, delayBetweenRetries, timeout, archiveCount)
+ executionRetries, delayBetweenRetries, timeout, archiveCount)
}
/**
@@ -1052,12 +1039,9 @@ object JobManager {
archiverActorName: Option[String]): (ActorRef, ActorRef) = {
val (instanceManager, scheduler, libraryCacheManager, archiveProps, accumulatorManager,
- profilerProps, executionRetries, delayBetweenRetries,
+ executionRetries, delayBetweenRetries,
timeout, _) = createJobManagerComponents(configuration)
- val profiler: Option[ActorRef] =
- profilerProps.map( props => actorSystem.actorOf(props, PROFILER_NAME) )
-
// start the archiver wither with the given name, or without (avoid name conflicts)
val archiver: ActorRef = archiverActorName match {
case Some(actorName) => actorSystem.actorOf(archiveProps, actorName)
@@ -1065,7 +1049,7 @@ object JobManager {
}
val jobManagerProps = Props(classOf[JobManager], configuration, instanceManager, scheduler,
- libraryCacheManager, archiver, accumulatorManager, profiler, executionRetries,
+ libraryCacheManager, archiver, accumulatorManager, executionRetries,
delayBetweenRetries, timeout)
val jobManager: ActorRef = jobMangerActorName match {
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala
deleted file mode 100644
index dd3a1b7..0000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/jobmanager/JobManagerProfiler.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.jobmanager
-
-import akka.actor.Actor
-import org.apache.flink.runtime.{ActorSynchronousLogging, ActorLogMessages}
-import org.apache.flink.runtime.messages.JobManagerProfilerMessages.ReportProfilingData
-import org.apache.flink.runtime.profiling.impl.types.{InternalInstanceProfilingData, InternalExecutionVertexThreadProfilingData}
-
-import scala.collection.convert.WrapAsScala
-
-/**
- * Basic skeleton for the JobManager profiler. Currently, it simply logs the received messages.
- */
-class JobManagerProfiler
- extends Actor
- with ActorLogMessages
- with ActorSynchronousLogging
- with WrapAsScala {
- override def receiveWithLogMessages: Receive = {
- case ReportProfilingData(profilingContainer) =>
- profilingContainer.getIterator foreach {
- case x: InternalExecutionVertexThreadProfilingData =>
- log.info(s"Received InternalExecutionVertexThreadProfilingData $x.")
- case x: InternalInstanceProfilingData =>
- log.info(s"Received InternalInstanceProfilingData $x.")
- case x =>
- log.error(s"Received unknown profiling data: ${x.getClass.getName}" )
- }
- }
-
- /**
- * Handle unmatched messages with an exception.
- */
- override def unhandled(message: Any): Unit = {
- // let the actor crash
- throw new RuntimeException("Received unknown message " + message)
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerProfilerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerProfilerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerProfilerMessages.scala
deleted file mode 100644
index 85e0fc3..0000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/JobManagerProfilerMessages.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.messages
-
-import org.apache.flink.runtime.profiling.impl.types.ProfilingDataContainer
-
-/**
- * This object contains the job manager profiler messages
- */
-object JobManagerProfilerMessages {
-
- /**
- * Reports profiling data to the profiler.
- * @param profilingDataContainer
- */
- case class ReportProfilingData(profilingDataContainer: ProfilingDataContainer)
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerProfilerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerProfilerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerProfilerMessages.scala
deleted file mode 100644
index 0662d48..0000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerProfilerMessages.scala
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.messages
-
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
-import org.apache.flink.runtime.taskmanager.Task
-
-object TaskManagerProfilerMessages {
-
- /**
- * Requests to monitor the specified [[task]].
- *
- * @param task
- */
- case class MonitorTask(task: Task)
-
- /**
- * Requests to unmonitor the task associated to [[executionID]].
- *
- * @param executionID
- */
- case class UnmonitorTask(executionID: ExecutionAttemptID)
-
- /**
- * Registers the sender as a profiling event listener.
- */
- case object RegisterProfilingListener
-
- /**
- * Unregisters the sender as a profiling event listener.
- */
- case object UnregisterProfilingListener
-
- /**
- * Makes the task manager profiling the running tasks.
- */
- case object ProfileTasks
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala
deleted file mode 100644
index f0079f8..0000000
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManagerProfiler.scala
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.taskmanager
-
-import java.lang.management.ManagementFactory
-import java.util.concurrent.TimeUnit
-
-import akka.actor.{Cancellable, ActorRef, Actor, ActorLogging}
-import org.apache.flink.runtime.{ActorSynchronousLogging, ActorLogMessages}
-import org.apache.flink.runtime.execution.{RuntimeEnvironment, ExecutionState}
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
-import org.apache.flink.runtime.jobgraph.JobVertexID
-import org.apache.flink.runtime.messages.ExecutionGraphMessages.ExecutionStateChanged
-import org.apache.flink.runtime.messages.JobManagerProfilerMessages.ReportProfilingData
-import org.apache.flink.runtime.messages.TaskManagerProfilerMessages._
-import org.apache.flink.runtime.profiling.ProfilingException
-import org.apache.flink.runtime.profiling.impl.types.ProfilingDataContainer
-import org.apache.flink.runtime.profiling.impl.{EnvironmentThreadSet, InstanceProfiler}
-
-import scala.concurrent.duration.FiniteDuration
-
-/**
- * Actor which is responsible for profiling task threads on the [[TaskManager]]. The monitoring
- * is triggered by the self-addressed message [[ProfileTasks]] which is scheduled to be sent
- * repeatedly.
- *
- * @param instancePath Akka URL to [[TaskManager]] instance
- * @param reportInterval Interval of profiling action
- */
-class TaskManagerProfiler(val instancePath: String, val reportInterval: Int)
- extends Actor with ActorLogMessages with ActorSynchronousLogging {
-
- import context.dispatcher
-
- val tmx = ManagementFactory.getThreadMXBean
- val instanceProfiler = new InstanceProfiler(instancePath)
- val listeners = scala.collection.mutable.Set[ActorRef]()
- val environments = scala.collection.mutable.HashMap[ExecutionAttemptID, RuntimeEnvironment]()
- val monitoredThreads = scala.collection.mutable.HashMap[RuntimeEnvironment,
- EnvironmentThreadSet]()
-
- var monitoringScheduler: Option[Cancellable] = None
-
- if (tmx.isThreadContentionMonitoringSupported) {
- tmx.setThreadContentionMonitoringEnabled(true)
- } else {
- throw new ProfilingException("The thread contention monitoring is not supported.")
- }
-
-
- override def receiveWithLogMessages: Receive = {
- case MonitorTask(task) =>
- task.registerExecutionListener(self)
- environments += task.getExecutionId -> task.getEnvironment
-
- case UnmonitorTask(executionAttemptID) =>
- environments.remove(executionAttemptID)
-
- case RegisterProfilingListener =>
- listeners += sender
- if (monitoringScheduler.isEmpty) {
- startMonitoring()
- }
-
- case UnregisterProfilingListener =>
- listeners -= sender
- if (listeners.isEmpty) {
- stopMonitoring()
- }
-
- case ProfileTasks =>
- val timestamp = System.currentTimeMillis()
-
- val profilingDataContainer = new ProfilingDataContainer()
-
- for ((env, set) <- monitoredThreads) {
- val threadProfilingData = set.captureCPUUtilization(env.getJobID, tmx, timestamp)
-
- if (threadProfilingData != null) {
- profilingDataContainer.addProfilingData(threadProfilingData)
- }
-
- if (monitoredThreads.nonEmpty) {
- val instanceProfilingData = try {
- Some(instanceProfiler.generateProfilingData(timestamp))
- } catch {
- case e: ProfilingException =>
- log.error("Error while retrieving instance profiling data.", e)
- None
- }
-
- instanceProfilingData foreach {
- profilingDataContainer.addProfilingData(_)
- }
-
- if (!profilingDataContainer.isEmpty) {
- for (listener <- listeners) {
- listener ! ReportProfilingData(profilingDataContainer)
- }
- }
-
- profilingDataContainer.clear()
- }
- }
-
- case ExecutionStateChanged(_, vertexID, _, _, subtaskIndex, executionID, newExecutionState,
- _, _) =>
- import ExecutionState._
-
- environments.get(executionID) match {
- case Some(environment) =>
- newExecutionState match {
- case RUNNING => registerMainThreadForCPUProfiling(environment, vertexID,
- subtaskIndex, executionID)
- case FINISHED | CANCELING | CANCELED | FAILED =>
- unregisterMainThreadFromCPUProfiling(environment)
- case _ =>
- }
- case None =>
- log.warn(s"Could not find environment for execution id $executionID.")
- }
- }
-
- /**
- * Handle unmatched messages with an exception.
- */
- override def unhandled(message: Any): Unit = {
- // let the actor crash
- throw new RuntimeException("Received unknown message " + message)
- }
-
- def startMonitoring(): Unit = {
- val interval = new FiniteDuration(reportInterval, TimeUnit.MILLISECONDS)
- val delay = new FiniteDuration((reportInterval * Math.random()).toLong, TimeUnit.MILLISECONDS)
-
- // schedule ProfileTasks message to be sent repeatedly to oneself
- monitoringScheduler = Some(context.system.scheduler.schedule(delay, interval, self,
- ProfileTasks))
- }
-
- def stopMonitoring(): Unit = {
- monitoringScheduler.foreach {
- _.cancel()
- }
- monitoringScheduler = None
- }
-
- def registerMainThreadForCPUProfiling(environment: RuntimeEnvironment, vertexID: JobVertexID,
- subtask: Int,
- executionID: ExecutionAttemptID): Unit = {
- monitoredThreads += environment -> new EnvironmentThreadSet(tmx,
- environment.getExecutingThread, vertexID,
- subtask, executionID)
- }
-
- def unregisterMainThreadFromCPUProfiling(environment: RuntimeEnvironment): Unit = {
- monitoredThreads.remove(environment) match {
- case Some(set) =>
- if (set.getMainThread != environment.getExecutingThread) {
- log.error(s"The thread ${environment.getExecutingThread.getName} is not the main thread" +
- s" of this environment.")
- }
- case None =>
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/impl/InstanceProfilerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/impl/InstanceProfilerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/impl/InstanceProfilerTest.java
deleted file mode 100644
index 85a4a37..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/impl/InstanceProfilerTest.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.runtime.profiling.impl;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.when;
-import static org.mockito.MockitoAnnotations.initMocks;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.net.InetAddress;
-
-import org.apache.flink.runtime.instance.InstanceConnectionInfo;
-import org.apache.flink.runtime.profiling.ProfilingException;
-import org.apache.flink.runtime.profiling.impl.types.InternalInstanceProfilingData;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-/**
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest(InstanceProfiler.class)
-public class InstanceProfilerTest {
-
- @Mock
- private InstanceConnectionInfo infoMock;
-
- @Mock
- private InetAddress addressMock;
-
- @Mock
- private BufferedReader cpuBufferMock;
-
- @Mock
- private BufferedReader networkBufferMock;
-
- @Mock
- private BufferedReader memoryBufferMock;
-
- @Mock
- private FileReader cpuReaderMock;
-
- @Mock
- private FileReader networkReaderMock;
-
- @Mock
- private FileReader memoryReaderMock;
-
- // object under test
- InstanceProfiler out;
-
- @Before
- public void setUp() throws Exception {
- initMocks(this);
- when(this.infoMock.address()).thenReturn(this.addressMock);
- when(this.addressMock.getHostAddress()).thenReturn("192.168.1.1");
-
- whenNew(FileReader.class).withArguments(InstanceProfiler.PROC_STAT).thenReturn(this.cpuReaderMock);
- whenNew(BufferedReader.class).withArguments(this.cpuReaderMock).thenReturn(this.cpuBufferMock);
-
- when(this.cpuBufferMock.readLine()).thenReturn(
- "cpu 222875 20767 209704 3782096 209864 0 1066 0 0 0"
- );
-
- whenNew(FileReader.class).withArguments(InstanceProfiler.PROC_NET_DEV).thenReturn(this.networkReaderMock);
- whenNew(BufferedReader.class).withArguments(this.networkReaderMock).thenReturn(this.networkBufferMock);
-
- when(this.networkBufferMock.readLine())
- .thenReturn(
- " eth0: 364729203 286442 0 0 0 0 0 1060 14483806 191563 0 0 0 0 0 0",
- (String) null,
- " eth0: 364729203 286442 0 0 0 0 0 1060 14483806 191563 0 0 0 0 0 0",
- (String) null,
- " eth0: 364729203 286442 0 0 0 0 0 1060 14483806 191563 0 0 0 0 0 0",
- (String) null
- );
-
- whenNew(FileReader.class).withArguments(InstanceProfiler.PROC_MEMINFO).thenReturn(this.memoryReaderMock);
- whenNew(BufferedReader.class).withArguments(this.memoryReaderMock).thenReturn(this.memoryBufferMock);
-
- when(this.memoryBufferMock.readLine()).thenReturn(
- "MemTotal: 8052956 kB",
- "MemFree: 3999880 kB",
- "Buffers: 77216 kB",
- "Cached: 1929640 kB",
- null,
- "MemTotal: 8052956 kB",
- "MemFree: 3999880 kB",
- "Buffers: 77216 kB",
- "Cached: 1929640 kB",
- null,
- "MemTotal: 8052956 kB",
- "MemFree: 3999880 kB",
- "Buffers: 77216 kB",
- "Cached: 1929640 kB",
- null
- );
-
- PowerMockito.mockStatic(System.class);
- when(System.currentTimeMillis()).thenReturn(0L);
-
- this.out = new InstanceProfiler("InstanceProfilerTest");
- }
-
- @Test
- public void shouldHaveNetworkTraffic() {
-
- try {
- final InternalInstanceProfilingData generateProfilingData = out.generateProfilingData(0L);
- assertEquals(0L, generateProfilingData.getReceivedBytes());
- assertEquals(0L, generateProfilingData.getTransmittedBytes());
- } catch (ProfilingException e) {
- fail(e.getMessage());
- }
- }
-
- @Test
- public void shouldHaveMemSettingsMeasured() {
-
- try {
- final InternalInstanceProfilingData generateProfilingData = out.generateProfilingData(0L);
-
- final long totalMemory = generateProfilingData.getTotalMemory();
- assertThat(totalMemory, is(equalTo(8052956L)));
-
- long freeMemory = generateProfilingData.getFreeMemory();
- assertThat(freeMemory, is(equalTo(3999880L)));
-
- long buffers = generateProfilingData.getBufferedMemory();
- assertThat(buffers, is(equalTo(77216L)));
-
- long cached = generateProfilingData.getCachedMemory();
- assertThat(cached, is(equalTo(1929640L)));
- } catch (ProfilingException e) {
- fail(e.getMessage());
- }
- }
-
- @Test
- public void shouldMeasureCPUUtilization() {
-
- try {
- final InternalInstanceProfilingData generateProfilingData = out.generateProfilingData(0L);
-
- assertEquals(0L, generateProfilingData.getUserCPU());
- assertEquals(0L, generateProfilingData.getIdleCPU());
- assertEquals(0L, generateProfilingData.getSystemCPU());
- assertEquals(0L, generateProfilingData.getHardIrqCPU());
- assertEquals(0L, generateProfilingData.getSoftIrqCPU());
- assertEquals(0L, generateProfilingData.getIOWaitCPU());
- } catch (ProfilingException e) {
- fail(e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/types/ProfilingTypesTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/types/ProfilingTypesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/types/ProfilingTypesTest.java
deleted file mode 100644
index c3c75ce..0000000
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/profiling/types/ProfilingTypesTest.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.profiling.types;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.testutils.CommonTestUtils;
-import org.apache.flink.runtime.testutils.ManagementTestUtils;
-import org.junit.Test;
-
-/**
- * This test checks the proper serialization and deserialization of profiling events.
- */
-public class ProfilingTypesTest {
-
- private static final int PROFILING_INTERVAL = 4321;
-
- private static final int IOWAIT_CPU = 10;
-
- private static final int IDLE_CPU = 11;
-
- private static final int USER_CPU = 12;
-
- private static final int SYSTEM_CPU = 13;
-
- private static final int HARD_IRQ_CPU = 14;
-
- private static final int SOFT_IRQ_CPU = 15;
-
- private static final long TOTAL_MEMORY = 10001L;
-
- private static final long FREE_MEMORY = 10002L;
-
- private static final long BUFFERED_MEMORY = 10003L;
-
- private static final long CACHED_MEMORY = 10004L;
-
- private static final long CACHED_SWAP_MEMORY = 10005L;
-
- private static final long RECEIVED_BYTES = 100006L;
-
- private static final long TRANSMITTED_BYTES = 100007L;
-
- private static final long TIMESTAMP = 100008L;
-
- private static final long PROFILING_TIMESTAMP = 100009L;
-
- private static final String INSTANCE_NAME = "Test Instance";
-
- private static final int USER_TIME = 17;
-
- private static final int SYSTEM_TIME = 18;
-
- private static final int BLOCKED_TIME = 19;
-
- private static final int WAITED_TIME = 20;
-
- /**
- * Tests serialization/deserialization for {@link InstanceSummaryProfilingEvent}.
- */
- @Test
- public void testInstanceSummaryProfilingEvent() {
-
- final InstanceSummaryProfilingEvent orig = new InstanceSummaryProfilingEvent(PROFILING_INTERVAL, IOWAIT_CPU,
- IDLE_CPU, USER_CPU, SYSTEM_CPU, HARD_IRQ_CPU, SOFT_IRQ_CPU, TOTAL_MEMORY, FREE_MEMORY, BUFFERED_MEMORY,
- CACHED_MEMORY, CACHED_SWAP_MEMORY, RECEIVED_BYTES, TRANSMITTED_BYTES, new JobID(), TIMESTAMP,
- PROFILING_TIMESTAMP);
-
- final InstanceSummaryProfilingEvent copy = (InstanceSummaryProfilingEvent) ManagementTestUtils.createCopy(orig);
-
- assertEquals(orig.getProfilingInterval(), copy.getProfilingInterval());
- assertEquals(orig.getIOWaitCPU(), copy.getIOWaitCPU());
- assertEquals(orig.getIdleCPU(), copy.getIdleCPU());
- assertEquals(orig.getUserCPU(), copy.getUserCPU());
- assertEquals(orig.getSystemCPU(), copy.getSystemCPU());
- assertEquals(orig.getHardIrqCPU(), copy.getHardIrqCPU());
- assertEquals(orig.getSoftIrqCPU(), copy.getSoftIrqCPU());
- assertEquals(orig.getTotalMemory(), copy.getTotalMemory());
- assertEquals(orig.getFreeMemory(), copy.getFreeMemory());
- assertEquals(orig.getBufferedMemory(), copy.getBufferedMemory());
- assertEquals(orig.getCachedMemory(), copy.getCachedMemory());
- assertEquals(orig.getCachedSwapMemory(), copy.getCachedSwapMemory());
- assertEquals(orig.getReceivedBytes(), copy.getReceivedBytes());
- assertEquals(orig.getTransmittedBytes(), copy.getTransmittedBytes());
- assertEquals(orig.getJobID(), copy.getJobID());
- assertEquals(orig.getTimestamp(), copy.getTimestamp());
- assertEquals(orig.getProfilingTimestamp(), copy.getProfilingTimestamp());
- assertEquals(orig.hashCode(), copy.hashCode());
- assertTrue(orig.equals(copy));
- }
-
- /**
- * Tests serialization/deserialization for {@link SingleInstanceProfilingEvent}.
- */
- @Test
- public void testSingleInstanceProfilingEvent() {
- try {
- final SingleInstanceProfilingEvent orig = new SingleInstanceProfilingEvent(PROFILING_INTERVAL, IOWAIT_CPU,
- IDLE_CPU, USER_CPU, SYSTEM_CPU, HARD_IRQ_CPU, SOFT_IRQ_CPU, TOTAL_MEMORY, FREE_MEMORY, BUFFERED_MEMORY,
- CACHED_MEMORY, CACHED_SWAP_MEMORY, RECEIVED_BYTES, TRANSMITTED_BYTES, new JobID(), TIMESTAMP,
- PROFILING_TIMESTAMP, INSTANCE_NAME);
-
- final SingleInstanceProfilingEvent copy = (SingleInstanceProfilingEvent) CommonTestUtils.createCopyWritable(orig);
-
- assertEquals(orig.getProfilingInterval(), copy.getProfilingInterval());
- assertEquals(orig.getIOWaitCPU(), copy.getIOWaitCPU());
- assertEquals(orig.getIdleCPU(), copy.getIdleCPU());
- assertEquals(orig.getUserCPU(), copy.getUserCPU());
- assertEquals(orig.getSystemCPU(), copy.getSystemCPU());
- assertEquals(orig.getHardIrqCPU(), copy.getHardIrqCPU());
- assertEquals(orig.getSoftIrqCPU(), copy.getSoftIrqCPU());
- assertEquals(orig.getTotalMemory(), copy.getTotalMemory());
- assertEquals(orig.getFreeMemory(), copy.getFreeMemory());
- assertEquals(orig.getBufferedMemory(), copy.getBufferedMemory());
- assertEquals(orig.getCachedMemory(), copy.getCachedMemory());
- assertEquals(orig.getCachedSwapMemory(), copy.getCachedSwapMemory());
- assertEquals(orig.getReceivedBytes(), copy.getReceivedBytes());
- assertEquals(orig.getTransmittedBytes(), copy.getTransmittedBytes());
- assertEquals(orig.getJobID(), copy.getJobID());
- assertEquals(orig.getTimestamp(), copy.getTimestamp());
- assertEquals(orig.getProfilingTimestamp(), copy.getProfilingTimestamp());
- assertEquals(orig.getInstanceName(), copy.getInstanceName());
- assertEquals(orig.hashCode(), copy.hashCode());
- assertTrue(orig.equals(copy));
- }
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
- }
-
- }
-
- /**
- * Tests serialization/deserialization for {@link ThreadProfilingEvent}.
- */
- @Test
- public void testThreadProfilingEvent() {
-
- final ThreadProfilingEvent orig = new ThreadProfilingEvent(USER_TIME, SYSTEM_TIME, BLOCKED_TIME, WAITED_TIME,
- new JobVertexID(), 17, new ExecutionAttemptID(), PROFILING_INTERVAL, new JobID(), TIMESTAMP, PROFILING_TIMESTAMP);
-
- final ThreadProfilingEvent copy = (ThreadProfilingEvent) ManagementTestUtils.createCopy(orig);
-
- assertEquals(orig.getUserTime(), copy.getUserTime());
- assertEquals(orig.getSystemTime(), copy.getSystemTime());
- assertEquals(orig.getBlockedTime(), copy.getBlockedTime());
- assertEquals(orig.getWaitedTime(), copy.getWaitedTime());
- assertEquals(orig.getVertexID(), copy.getVertexID());
- assertEquals(orig.getProfilingInterval(), copy.getProfilingInterval());
- assertEquals(orig.getJobID(), copy.getJobID());
- assertEquals(orig.getTimestamp(), copy.getTimestamp());
- assertEquals(orig.getProfilingTimestamp(), copy.getProfilingTimestamp());
- assertEquals(orig.hashCode(), copy.hashCode());
- assertTrue(orig.equals(copy));
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
index f87e151..9f9fe93 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingCluster.scala
@@ -49,7 +49,7 @@ class TestingCluster(userConfiguration: Configuration, singleActorSystem: Boolea
override def startJobManager(actorSystem: ActorSystem): ActorRef = {
- val (instanceManager, scheduler, libraryCacheManager, _, accumulatorManager, _,
+ val (instanceManager, scheduler, libraryCacheManager, _, accumulatorManager,
executionRetries, delayBetweenRetries,
timeout, archiveCount) = JobManager.createJobManagerComponents(configuration)
@@ -57,7 +57,7 @@ class TestingCluster(userConfiguration: Configuration, singleActorSystem: Boolea
val archive = actorSystem.actorOf(testArchiveProps, JobManager.ARCHIVE_NAME)
val jobManagerProps = Props(new JobManager(configuration, instanceManager, scheduler,
- libraryCacheManager, archive, accumulatorManager, None, executionRetries,
+ libraryCacheManager, archive, accumulatorManager, executionRetries,
delayBetweenRetries, timeout) with TestingJobManager)
actorSystem.actorOf(jobManagerProps, JobManager.JOB_MANAGER_NAME)
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
----------------------------------------------------------------------
diff --git a/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala b/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
index ddfffee..05093b5 100644
--- a/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
+++ b/flink-test-utils/src/main/scala/org/apache/flink/test/util/ForkableFlinkMiniCluster.scala
@@ -71,7 +71,7 @@ class ForkableFlinkMiniCluster(userConfiguration: Configuration, singleActorSyst
override def startJobManager(actorSystem: ActorSystem): ActorRef = {
- val (instanceManager, scheduler, libraryCacheManager, _, accumulatorManager, _,
+ val (instanceManager, scheduler, libraryCacheManager, _, accumulatorManager,
executionRetries, delayBetweenRetries,
timeout, archiveCount) = JobManager.createJobManagerComponents(configuration)
@@ -79,7 +79,7 @@ class ForkableFlinkMiniCluster(userConfiguration: Configuration, singleActorSyst
val archive = actorSystem.actorOf(testArchiveProps, JobManager.ARCHIVE_NAME)
val jobManagerProps = Props(new JobManager(configuration, instanceManager, scheduler,
- libraryCacheManager, archive, accumulatorManager, None, executionRetries,
+ libraryCacheManager, archive, accumulatorManager, executionRetries,
delayBetweenRetries, timeout) with TestingJobManager)
val jobManager = actorSystem.actorOf(jobManagerProps, JobManager.JOB_MANAGER_NAME)
http://git-wip-us.apache.org/repos/asf/flink/blob/fbea2da2/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala
----------------------------------------------------------------------
diff --git a/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala b/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala
index 06e16dc..7884edd 100644
--- a/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala
+++ b/flink-yarn/src/main/scala/org/apache/flink/yarn/ApplicationMaster.scala
@@ -32,7 +32,7 @@ import org.apache.flink.yarn.Messages.StartYarnSession
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.slf4j.LoggerFactory
+
import scala.io.Source
@@ -228,18 +228,14 @@ object ApplicationMaster {
// start all the components inside the job manager
LOG.debug("Starting JobManager components")
val (instanceManager, scheduler, libraryCacheManager, archiveProps, accumulatorManager,
- profilerProps, executionRetries, delayBetweenRetries,
+ executionRetries, delayBetweenRetries,
timeout, _) = JobManager.createJobManagerComponents(configuration)
- // start the profiler, if needed
- val profiler: Option[ActorRef] =
- profilerProps.map( props => jobManagerSystem.actorOf(props, JobManager.PROFILER_NAME) )
-
// start the archiver
val archiver: ActorRef = jobManagerSystem.actorOf(archiveProps, JobManager.ARCHIVE_NAME)
val jobManagerProps = Props(new JobManager(configuration, instanceManager, scheduler,
- libraryCacheManager, archiver, accumulatorManager, profiler, executionRetries,
+ libraryCacheManager, archiver, accumulatorManager, executionRetries,
delayBetweenRetries, timeout) with ApplicationMasterActor)
LOG.debug("Starting JobManager actor")
[5/8] flink git commit: [FLINK-1672] [runtime] Unify Task and
RuntimeEnvironment into one class.
Posted by se...@apache.org.
[FLINK-1672] [runtime] Unify Task and RuntimeEnvironment into one class.
- This simplifies and hardens the failure handling during task startup
- Guarantees that no actor system threads are blocked by task bootstrap, or task canceling
- Corrects some previously erroneous corner case state transitions
- Adds simple and robust tests
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/8e613014
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/8e613014
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/8e613014
Branch: refs/heads/master
Commit: 8e61301452218e6d279b013beb7bbd02a7c2e3f9
Parents: 1d368a4
Author: Stephan Ewen <se...@apache.org>
Authored: Sun May 3 04:41:03 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon May 11 21:13:41 2015 +0200
----------------------------------------------------------------------
.../flink/runtime/execution/Environment.java | 3 +
.../flink/runtime/execution/ExecutionState.java | 6 +-
.../runtime/execution/RuntimeEnvironment.java | 458 ---------
.../runtime/io/network/NetworkEnvironment.java | 11 +-
.../io/network/partition/ResultPartition.java | 14 +-
.../partition/consumer/SingleInputGate.java | 15 +-
.../runtime/operators/RegularPactTask.java | 3 +-
.../runtime/taskmanager/RuntimeEnvironment.java | 230 +++++
.../apache/flink/runtime/taskmanager/Task.java | 990 ++++++++++++++-----
.../runtime/messages/TaskManagerMessages.scala | 12 +-
.../flink/runtime/messages/TaskMessages.scala | 6 +-
.../flink/runtime/taskmanager/TaskManager.scala | 321 ++----
.../consumer/LocalInputChannelTest.java | 7 +-
.../partition/consumer/SingleInputGateTest.java | 6 +-
.../partition/consumer/TestSingleInputGate.java | 4 +-
.../partition/consumer/UnionInputGateTest.java | 9 +-
.../operators/testutils/MockEnvironment.java | 6 +
.../runtime/taskmanager/ForwardingActor.java | 41 +
.../taskmanager/TaskExecutionStateTest.java | 33 +
.../runtime/taskmanager/TaskManagerTest.java | 152 ++-
.../flink/runtime/taskmanager/TaskTest.java | 874 ++++++++++++----
.../testingUtils/TestingTaskManager.scala | 6 +-
22 files changed, 1972 insertions(+), 1235 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
index 7ab3bc9..081e3ca 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.execution;
+import akka.actor.ActorRef;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
@@ -159,4 +160,6 @@ public interface Environment {
InputGate[] getAllInputGates();
+ // this should go away
+ ActorRef getJobManager();
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java
index 2fcaea1..9f4a5a7 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/ExecutionState.java
@@ -35,10 +35,10 @@ package org.apache.flink.runtime.execution;
* ... -> FAILED
* </pre>
*
- * It is possible to enter the {@code FAILED} state from any other state.
+ * <p>It is possible to enter the {@code FAILED} state from any other state.</p>
*
- * The states {@code FINISHED}, {@code CANCELED}, and {@code FAILED} are
- * considered terminal states.
+ * <p>The states {@code FINISHED}, {@code CANCELED}, and {@code FAILED} are
+ * considered terminal states.</p>
*/
public enum ExecutionState {
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
deleted file mode 100644
index 081d4bc..0000000
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/RuntimeEnvironment.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.execution;
-
-import akka.actor.ActorRef;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.accumulators.AccumulatorEvent;
-import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
-import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
-import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
-import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
-import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.network.NetworkEnvironment;
-import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
-import org.apache.flink.runtime.io.network.partition.ResultPartition;
-import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
-import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
-import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
-import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.jobgraph.JobVertexID;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
-import org.apache.flink.runtime.memorymanager.MemoryManager;
-import org.apache.flink.runtime.messages.accumulators.ReportAccumulatorResult;
-import org.apache.flink.runtime.taskmanager.Task;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static com.google.common.base.Preconditions.checkElementIndex;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-public class RuntimeEnvironment implements Environment, Runnable {
-
- private static final Logger LOG = LoggerFactory.getLogger(RuntimeEnvironment.class);
-
- private static final ThreadGroup TASK_THREADS = new ThreadGroup("Task Threads");
-
- /** The ActorRef to the job manager */
- private final ActorRef jobManager;
-
- /** The task that owns this environment */
- private final Task owner;
-
- /** The job configuration encapsulated in the environment object. */
- private final Configuration jobConfiguration;
-
- /** The task configuration encapsulated in the environment object. */
- private final Configuration taskConfiguration;
-
- /** ClassLoader for all user code classes */
- private final ClassLoader userCodeClassLoader;
-
- /** Instance of the class to be run in this environment. */
- private final AbstractInvokable invokable;
-
- /** The memory manager of the current environment (currently the one associated with the executing TaskManager). */
- private final MemoryManager memoryManager;
-
- /** The I/O manager of the current environment (currently the one associated with the executing TaskManager). */
- private final IOManager ioManager;
-
- /** The input split provider that can be queried for new input splits. */
- private final InputSplitProvider inputSplitProvider;
-
- /** The thread executing the task in the environment. */
- private Thread executingThread;
-
- private final BroadcastVariableManager broadcastVariableManager;
-
- private final Map<String, FutureTask<Path>> cacheCopyTasks = new HashMap<String, FutureTask<Path>>();
-
- private final AtomicBoolean canceled = new AtomicBoolean();
-
- private final ResultPartition[] producedPartitions;
- private final ResultPartitionWriter[] writers;
-
- private final SingleInputGate[] inputGates;
-
- private final Map<IntermediateDataSetID, SingleInputGate> inputGatesById = new HashMap<IntermediateDataSetID, SingleInputGate>();
-
- public RuntimeEnvironment(
- ActorRef jobManager, Task owner, TaskDeploymentDescriptor tdd, ClassLoader userCodeClassLoader,
- MemoryManager memoryManager, IOManager ioManager, InputSplitProvider inputSplitProvider,
- BroadcastVariableManager broadcastVariableManager, NetworkEnvironment networkEnvironment) throws Exception {
-
- this.owner = checkNotNull(owner);
-
- this.memoryManager = checkNotNull(memoryManager);
- this.ioManager = checkNotNull(ioManager);
- this.inputSplitProvider = checkNotNull(inputSplitProvider);
- this.jobManager = checkNotNull(jobManager);
-
- this.broadcastVariableManager = checkNotNull(broadcastVariableManager);
-
- try {
- // Produced intermediate result partitions
- final List<ResultPartitionDeploymentDescriptor> partitions = tdd.getProducedPartitions();
-
- this.producedPartitions = new ResultPartition[partitions.size()];
- this.writers = new ResultPartitionWriter[partitions.size()];
-
- for (int i = 0; i < this.producedPartitions.length; i++) {
- ResultPartitionDeploymentDescriptor desc = partitions.get(i);
- ResultPartitionID partitionId = new ResultPartitionID(desc.getPartitionId(), owner.getExecutionId());
-
- this.producedPartitions[i] = new ResultPartition(
- this,
- owner.getJobID(),
- partitionId,
- desc.getPartitionType(),
- desc.getNumberOfSubpartitions(),
- networkEnvironment.getPartitionManager(),
- networkEnvironment.getPartitionConsumableNotifier(),
- ioManager,
- networkEnvironment.getDefaultIOMode());
-
- writers[i] = new ResultPartitionWriter(this.producedPartitions[i]);
- }
-
- // Consumed intermediate result partitions
- final List<InputGateDeploymentDescriptor> consumedPartitions = tdd.getInputGates();
-
- this.inputGates = new SingleInputGate[consumedPartitions.size()];
-
- for (int i = 0; i < inputGates.length; i++) {
- inputGates[i] = SingleInputGate.create(
- this, consumedPartitions.get(i), networkEnvironment);
-
- // The input gates are organized by key for task updates/channel updates at runtime
- inputGatesById.put(inputGates[i].getConsumedResultId(), inputGates[i]);
- }
-
- this.jobConfiguration = tdd.getJobConfiguration();
- this.taskConfiguration = tdd.getTaskConfiguration();
-
- // ----------------------------------------------------------------
- // Invokable setup
- // ----------------------------------------------------------------
- // Note: This has to be done *after* the readers and writers have
- // been setup, because the invokable relies on them for I/O.
- // ----------------------------------------------------------------
-
- // Load and instantiate the invokable class
- this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
- // Class of the task to run in this environment
- Class<? extends AbstractInvokable> invokableClass;
- try {
- final String className = tdd.getInvokableClassName();
- invokableClass = Class.forName(className, true, userCodeClassLoader).asSubclass(AbstractInvokable.class);
- }
- catch (Throwable t) {
- throw new Exception("Could not load invokable class.", t);
- }
-
- try {
- this.invokable = invokableClass.newInstance();
- }
- catch (Throwable t) {
- throw new Exception("Could not instantiate the invokable class.", t);
- }
-
- this.invokable.setEnvironment(this);
- this.invokable.registerInputOutput();
- }
- catch (Throwable t) {
- throw new Exception("Error setting up runtime environment: " + t.getMessage(), t);
- }
- }
-
- /**
- * Returns the task invokable instance.
- */
- public AbstractInvokable getInvokable() {
- return this.invokable;
- }
-
- @Override
- public JobID getJobID() {
- return this.owner.getJobID();
- }
-
- @Override
- public JobVertexID getJobVertexId() {
- return this.owner.getVertexID();
- }
-
- @Override
- public void run() {
- // quick fail in case the task was cancelled while the thread was started
- if (owner.isCanceledOrFailed()) {
- owner.cancelingDone();
- return;
- }
-
- try {
- Thread.currentThread().setContextClassLoader(userCodeClassLoader);
- invokable.invoke();
-
- // Make sure, we enter the catch block when the task has been canceled
- if (owner.isCanceledOrFailed()) {
- throw new CancelTaskException("Task has been canceled or failed");
- }
-
- // Finish the produced partitions
- if (producedPartitions != null) {
- for (ResultPartition partition : producedPartitions) {
- if (partition != null) {
- partition.finish();
- }
- }
- }
-
- if (owner.isCanceledOrFailed()) {
- throw new CancelTaskException();
- }
-
- // Finally, switch execution state to FINISHED and report to job manager
- if (!owner.markAsFinished()) {
- throw new Exception("Could *not* notify job manager that the task is finished.");
- }
- }
- catch (Throwable t) {
- if (!owner.isCanceledOrFailed()) {
- // Perform clean up when the task failed and has been not canceled by the user
- try {
- invokable.cancel();
- }
- catch (Throwable t2) {
- LOG.error("Error while canceling the task", t2);
- }
- }
-
- // if we are already set as cancelled or failed (when failure is triggered externally),
- // mark that the thread is done.
- if (owner.isCanceledOrFailed() || t instanceof CancelTaskException) {
- owner.cancelingDone();
- }
- else {
- // failure from inside the task thread. notify the task of the failure
- owner.markFailed(t);
- }
- }
- }
-
- /**
- * Returns the thread, which is assigned to execute the user code.
- */
- public Thread getExecutingThread() {
- synchronized (this) {
- if (executingThread == null) {
- String name = owner.getTaskNameWithSubtasks();
-
- if (LOG.isDebugEnabled()) {
- name = name + " (" + owner.getExecutionId() + ")";
- }
-
- executingThread = new Thread(TASK_THREADS, this, name);
- }
-
- return executingThread;
- }
- }
-
- public void cancelExecution() {
- if (!canceled.compareAndSet(false, true)) {
- return;
- }
-
- LOG.info("Canceling {} ({}).", owner.getTaskNameWithSubtasks(), owner.getExecutionId());
-
- // Request user code to shut down
- if (invokable != null) {
- try {
- invokable.cancel();
- }
- catch (Throwable e) {
- LOG.error("Error while canceling the task.", e);
- }
- }
-
- final Thread executingThread = this.executingThread;
- if (executingThread != null) {
- // interrupt the running thread and wait for it to die
- executingThread.interrupt();
- try {
- executingThread.join(5000);
- }
- catch (InterruptedException e) {
- }
- if (!executingThread.isAlive()) {
- return;
- }
- // Continuously interrupt the user thread until it changed to state CANCELED
- while (executingThread != null && executingThread.isAlive()) {
- LOG.warn("Task " + owner.getTaskNameWithSubtasks() + " did not react to cancelling signal. Sending repeated interrupt.");
- if (LOG.isDebugEnabled()) {
- StringBuilder bld = new StringBuilder("Task ").append(owner.getTaskNameWithSubtasks()).append(" is stuck in method:\n");
- StackTraceElement[] stack = executingThread.getStackTrace();
- for (StackTraceElement e : stack) {
- bld.append(e).append('\n');
- }
- LOG.debug(bld.toString());
- }
- executingThread.interrupt();
- try {
- executingThread.join(1000);
- }
- catch (InterruptedException e) {
- }
- }
- }
- }
-
- @Override
- public ActorRef getJobManager() {
- return jobManager;
- }
-
- @Override
- public IOManager getIOManager() {
- return ioManager;
- }
-
- @Override
- public MemoryManager getMemoryManager() {
- return memoryManager;
- }
-
- @Override
- public BroadcastVariableManager getBroadcastVariableManager() {
- return broadcastVariableManager;
- }
-
- @Override
- public void reportAccumulators(Map<String, Accumulator<?, ?>> accumulators) {
- AccumulatorEvent evt;
- try {
- evt = new AccumulatorEvent(getJobID(), accumulators);
- }
- catch (IOException e) {
- throw new RuntimeException("Cannot serialize accumulators to send them to JobManager", e);
- }
-
- ReportAccumulatorResult accResult = new ReportAccumulatorResult(getJobID(), owner.getExecutionId(), evt);
- jobManager.tell(accResult, ActorRef.noSender());
- }
-
- @Override
- public ResultPartitionWriter getWriter(int index) {
- checkElementIndex(index, writers.length, "Illegal environment writer request.");
-
- return writers[checkElementIndex(index, writers.length)];
- }
-
- @Override
- public ResultPartitionWriter[] getAllWriters() {
- return writers;
- }
-
- @Override
- public InputGate getInputGate(int index) {
- checkElementIndex(index, inputGates.length);
-
- return inputGates[index];
- }
-
- @Override
- public SingleInputGate[] getAllInputGates() {
- return inputGates;
- }
-
- public ResultPartition[] getProducedPartitions() {
- return producedPartitions;
- }
-
- public SingleInputGate getInputGateById(IntermediateDataSetID id) {
- return inputGatesById.get(id);
- }
-
- @Override
- public Configuration getTaskConfiguration() {
- return taskConfiguration;
- }
-
- @Override
- public Configuration getJobConfiguration() {
- return jobConfiguration;
- }
-
- @Override
- public int getNumberOfSubtasks() {
- return owner.getNumberOfSubtasks();
- }
-
- @Override
- public int getIndexInSubtaskGroup() {
- return owner.getSubtaskIndex();
- }
-
- @Override
- public String getTaskName() {
- return owner.getTaskName();
- }
-
- @Override
- public InputSplitProvider getInputSplitProvider() {
- return inputSplitProvider;
- }
-
- @Override
- public String getTaskNameWithSubtasks() {
- return owner.getTaskNameWithSubtasks();
- }
-
- @Override
- public ClassLoader getUserClassLoader() {
- return userCodeClassLoader;
- }
-
- public void addCopyTasksForCacheFile(Map<String, FutureTask<Path>> copyTasks) {
- cacheCopyTasks.putAll(copyTasks);
- }
-
- public void addCopyTaskForCacheFile(String name, FutureTask<Path> copyTask) {
- cacheCopyTasks.put(name, copyTask);
- }
-
- @Override
- public Map<String, FutureTask<Path>> getCopyTask() {
- return cacheCopyTasks;
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
index af55ebf..259ea55 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NetworkEnvironment.java
@@ -243,7 +243,7 @@ public class NetworkEnvironment {
public void registerTask(Task task) throws IOException {
final ResultPartition[] producedPartitions = task.getProducedPartitions();
- final ResultPartitionWriter[] writers = task.getWriters();
+ final ResultPartitionWriter[] writers = task.getAllWriters();
if (writers.length != producedPartitions.length) {
throw new IllegalStateException("Unequal number of writers and partitions.");
@@ -288,7 +288,7 @@ public class NetworkEnvironment {
}
// Setup the buffer pool for each buffer reader
- final SingleInputGate[] inputGates = task.getInputGates();
+ final SingleInputGate[] inputGates = task.getAllInputGates();
for (SingleInputGate gate : inputGates) {
BufferPool bufferPool = null;
@@ -329,10 +329,9 @@ public class NetworkEnvironment {
partitionManager.releasePartitionsProducedBy(executionId);
}
- ResultPartitionWriter[] writers = task.getWriters();
-
+ ResultPartitionWriter[] writers = task.getAllWriters();
if (writers != null) {
- for (ResultPartitionWriter writer : task.getWriters()) {
+ for (ResultPartitionWriter writer : writers) {
taskEventDispatcher.unregisterWriter(writer);
}
}
@@ -344,7 +343,7 @@ public class NetworkEnvironment {
}
}
- final SingleInputGate[] inputGates = task.getInputGates();
+ final SingleInputGate[] inputGates = task.getAllInputGates();
if (inputGates != null) {
for (SingleInputGate gate : inputGates) {
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
index f06c8fb..df1f254 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartition.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.io.network.partition;
-import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.executiongraph.IntermediateResultPartition;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.disk.iomanager.IOManager.IOMode;
@@ -76,9 +75,8 @@ import static com.google.common.base.Preconditions.checkState;
public class ResultPartition implements BufferPoolOwner {
private static final Logger LOG = LoggerFactory.getLogger(ResultPartition.class);
-
- /** The owning environment. Mainly for debug purposes. */
- private final Environment owner;
+
+ private final String owningTaskName;
private final JobID jobId;
@@ -120,7 +118,7 @@ public class ResultPartition implements BufferPoolOwner {
private long totalNumberOfBytes;
public ResultPartition(
- Environment owner,
+ String owningTaskName,
JobID jobId,
ResultPartitionID partitionId,
ResultPartitionType partitionType,
@@ -130,7 +128,7 @@ public class ResultPartition implements BufferPoolOwner {
IOManager ioManager,
IOMode defaultIoMode) {
- this.owner = checkNotNull(owner);
+ this.owningTaskName = checkNotNull(owningTaskName);
this.jobId = checkNotNull(jobId);
this.partitionId = checkNotNull(partitionId);
this.partitionType = checkNotNull(partitionType);
@@ -162,7 +160,7 @@ public class ResultPartition implements BufferPoolOwner {
// Initially, partitions should be consumed once before release.
pin();
- LOG.debug("{}: Initialized {}", owner.getTaskNameWithSubtasks(), this);
+ LOG.debug("{}: Initialized {}", owningTaskName, this);
}
/**
@@ -281,7 +279,7 @@ public class ResultPartition implements BufferPoolOwner {
*/
public void release() {
if (isReleased.compareAndSet(false, true)) {
- LOG.debug("{}: Releasing {}.", owner.getTaskNameWithSubtasks(), this);
+ LOG.debug("{}: Releasing {}.", owningTaskName, this);
// Release all subpartitions
for (ResultSubpartition subpartition : subpartitions) {
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
index b0d138a..acda1d8 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
@@ -24,7 +24,6 @@ import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionLocation;
import org.apache.flink.runtime.event.task.AbstractEvent;
import org.apache.flink.runtime.event.task.TaskEvent;
-import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
@@ -101,8 +100,8 @@ public class SingleInputGate implements InputGate {
/** Lock object to guard partition requests and runtime channel updates. */
private final Object requestLock = new Object();
- /** The owning environment. Mainly for debug purposes. */
- private final Environment owner;
+ /** The name of the owning task, for logging purposes. */
+ private final String owningTaskName;
/**
* The ID of the consumed intermediate result. Each input gate consumes partitions of the
@@ -153,12 +152,12 @@ public class SingleInputGate implements InputGate {
private int numberOfUninitializedChannels;
public SingleInputGate(
- Environment owner,
+ String owningTaskName,
IntermediateDataSetID consumedResultId,
int consumedSubpartitionIndex,
int numberOfInputChannels) {
- this.owner = checkNotNull(owner);
+ this.owningTaskName = checkNotNull(owningTaskName);
this.consumedResultId = checkNotNull(consumedResultId);
checkArgument(consumedSubpartitionIndex >= 0);
@@ -265,7 +264,7 @@ public class SingleInputGate implements InputGate {
synchronized (requestLock) {
if (!isReleased) {
try {
- LOG.debug("{}: Releasing {}.", owner.getTaskNameWithSubtasks(), this);
+ LOG.debug("{}: Releasing {}.", owningTaskName, this);
for (InputChannel inputChannel : inputChannels.values()) {
try {
@@ -410,7 +409,7 @@ public class SingleInputGate implements InputGate {
* Creates an input gate and all of its input channels.
*/
public static SingleInputGate create(
- Environment owner,
+ String owningTaskName,
InputGateDeploymentDescriptor igdd,
NetworkEnvironment networkEnvironment) {
@@ -422,7 +421,7 @@ public class SingleInputGate implements InputGate {
final InputChannelDeploymentDescriptor[] icdd = checkNotNull(igdd.getInputChannelDeploymentDescriptors());
final SingleInputGate inputGate = new SingleInputGate(
- owner, consumedResultId, consumedSubpartitionIndex, icdd.length);
+ owningTaskName, consumedResultId, consumedSubpartitionIndex, icdd.length);
// Create the input channels. There is one input channel for each consumed partition.
final InputChannel[] inputChannels = new InputChannel[icdd.length];
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
index b528f75..2bee094 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/RegularPactTask.java
@@ -1067,7 +1067,8 @@ public class RegularPactTask<S extends Function, OT> extends AbstractInvokable i
public DistributedRuntimeUDFContext createRuntimeContext(String taskName) {
Environment env = getEnvironment();
return new DistributedRuntimeUDFContext(taskName, env.getNumberOfSubtasks(),
- env.getIndexInSubtaskGroup(), getUserCodeClassLoader(), getExecutionConfig(), env.getCopyTask());
+ env.getIndexInSubtaskGroup(), getUserCodeClassLoader(), getExecutionConfig(),
+ env.getDistributedCacheEntries());
}
// --------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
new file mode 100644
index 0000000..1321336
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/RuntimeEnvironment.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+import akka.actor.ActorRef;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.accumulators.Accumulator;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.accumulators.AccumulatorEvent;
+import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
+import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
+import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
+import org.apache.flink.runtime.memorymanager.MemoryManager;
+import org.apache.flink.runtime.messages.accumulators.ReportAccumulatorResult;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.Future;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * In implementation of the {@link Environment}.
+ */
+public class RuntimeEnvironment implements Environment {
+
+ private final JobID jobId;
+ private final JobVertexID jobVertexId;
+ private final ExecutionAttemptID executionId;
+
+ private final String taskName;
+ private final String taskNameWithSubtasks;
+ private final int subtaskIndex;
+ private final int parallelism;
+
+ private final Configuration jobConfiguration;
+ private final Configuration taskConfiguration;
+
+ private final ClassLoader userCodeClassLoader;
+
+ private final MemoryManager memManager;
+ private final IOManager ioManager;
+ private final BroadcastVariableManager bcVarManager;
+ private final InputSplitProvider splitProvider;
+
+ private final Map<String, Future<Path>> distCacheEntries;
+
+ private final ResultPartitionWriter[] writers;
+ private final InputGate[] inputGates;
+
+ private final ActorRef jobManagerActor;
+
+ // ------------------------------------------------------------------------
+
+ public RuntimeEnvironment(JobID jobId, JobVertexID jobVertexId, ExecutionAttemptID executionId,
+ String taskName, String taskNameWithSubtasks,
+ int subtaskIndex, int parallelism,
+ Configuration jobConfiguration, Configuration taskConfiguration,
+ ClassLoader userCodeClassLoader,
+ MemoryManager memManager, IOManager ioManager,
+ BroadcastVariableManager bcVarManager,
+ InputSplitProvider splitProvider,
+ Map<String, Future<Path>> distCacheEntries,
+ ResultPartitionWriter[] writers,
+ InputGate[] inputGates,
+ ActorRef jobManagerActor) {
+
+ checkArgument(parallelism > 0 && subtaskIndex >= 0 && subtaskIndex < parallelism);
+
+ this.jobId = checkNotNull(jobId);
+ this.jobVertexId = checkNotNull(jobVertexId);
+ this.executionId = checkNotNull(executionId);
+ this.taskName = checkNotNull(taskName);
+ this.taskNameWithSubtasks = checkNotNull(taskNameWithSubtasks);
+ this.subtaskIndex = subtaskIndex;
+ this.parallelism = parallelism;
+ this.jobConfiguration = checkNotNull(jobConfiguration);
+ this.taskConfiguration = checkNotNull(taskConfiguration);
+ this.userCodeClassLoader = checkNotNull(userCodeClassLoader);
+ this.memManager = checkNotNull(memManager);
+ this.ioManager = checkNotNull(ioManager);
+ this.bcVarManager = checkNotNull(bcVarManager);
+ this.splitProvider = checkNotNull(splitProvider);
+ this.distCacheEntries = checkNotNull(distCacheEntries);
+ this.writers = checkNotNull(writers);
+ this.inputGates = checkNotNull(inputGates);
+ this.jobManagerActor = checkNotNull(jobManagerActor);
+ }
+
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public JobID getJobID() {
+ return jobId;
+ }
+
+ @Override
+ public JobVertexID getJobVertexId() {
+ return jobVertexId;
+ }
+
+ @Override
+ public ExecutionAttemptID getExecutionId() {
+ return executionId;
+ }
+
+ @Override
+ public String getTaskName() {
+ return taskName;
+ }
+
+ @Override
+ public String getTaskNameWithSubtasks() {
+ return taskNameWithSubtasks;
+ }
+
+ @Override
+ public int getNumberOfSubtasks() {
+ return parallelism;
+ }
+
+ @Override
+ public int getIndexInSubtaskGroup() {
+ return subtaskIndex;
+ }
+
+ @Override
+ public Configuration getJobConfiguration() {
+ return jobConfiguration;
+ }
+
+ @Override
+ public Configuration getTaskConfiguration() {
+ return taskConfiguration;
+ }
+
+ @Override
+ public ClassLoader getUserClassLoader() {
+ return userCodeClassLoader;
+ }
+
+ @Override
+ public MemoryManager getMemoryManager() {
+ return memManager;
+ }
+
+ @Override
+ public IOManager getIOManager() {
+ return ioManager;
+ }
+
+ @Override
+ public BroadcastVariableManager getBroadcastVariableManager() {
+ return bcVarManager;
+ }
+
+ @Override
+ public InputSplitProvider getInputSplitProvider() {
+ return splitProvider;
+ }
+
+ @Override
+ public Map<String, Future<Path>> getDistributedCacheEntries() {
+ return distCacheEntries;
+ }
+
+ @Override
+ public ResultPartitionWriter getWriter(int index) {
+ return writers[index];
+ }
+
+ @Override
+ public ResultPartitionWriter[] getAllWriters() {
+ return writers;
+ }
+
+ @Override
+ public InputGate getInputGate(int index) {
+ return inputGates[index];
+ }
+
+ @Override
+ public InputGate[] getAllInputGates() {
+ return inputGates;
+ }
+
+ @Override
+ public void reportAccumulators(Map<String, Accumulator<?, ?>> accumulators) {
+ AccumulatorEvent evt;
+ try {
+ evt = new AccumulatorEvent(getJobID(), accumulators);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Cannot serialize accumulators to send them to JobManager", e);
+ }
+
+ ReportAccumulatorResult accResult = new ReportAccumulatorResult(jobId, executionId, evt);
+ jobManagerActor.tell(accResult, ActorRef.noSender());
+ }
+
+ @Override
+ public ActorRef getJobManager() {
+ return jobManagerActor;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
index e6eee5b..f12344b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/Task.java
@@ -19,382 +19,755 @@
package org.apache.flink.runtime.taskmanager;
import akka.actor.ActorRef;
+import akka.util.Timeout;
+
+import org.apache.flink.api.common.JobID;
+import org.apache.flink.api.common.cache.DistributedCache;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.blob.BlobKey;
+import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
+import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
+import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
+import org.apache.flink.runtime.execution.CancelTaskException;
+import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.execution.ExecutionState;
-import org.apache.flink.runtime.execution.RuntimeEnvironment;
+import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.filecache.FileCache;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+import org.apache.flink.runtime.io.network.NetworkEnvironment;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
import org.apache.flink.runtime.io.network.partition.ResultPartition;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate;
-import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
+import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
+import org.apache.flink.runtime.jobgraph.tasks.BarrierTransceiver;
+import org.apache.flink.runtime.jobgraph.tasks.OperatorStateCarrier;
import org.apache.flink.runtime.memorymanager.MemoryManager;
-import org.apache.flink.runtime.messages.ExecutionGraphMessages;
-import org.apache.flink.runtime.messages.TaskMessages.UnregisterTask;
-import org.apache.flink.runtime.profiling.TaskManagerProfiler;
-import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.runtime.messages.TaskMessages;
+import org.apache.flink.runtime.messages.TaskMessages.TaskInFinalState;
+import org.apache.flink.runtime.messages.TaskManagerMessages.FatalError;
+import org.apache.flink.runtime.state.StateHandle;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import scala.concurrent.duration.FiniteDuration;
+
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-public class Task {
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * The Task represents one execution of a parallel subtask on a TaskManager.
+ * A Task wraps a Flink operator (which may be a user function) and
+ * runs it, providing all service necessary for example to consume input data,
+ * produce its results (intermediate result partitions) and communicate
+ * with the JobManager.
+ *
+ * <p>The Flink operators (implemented as subclasses of
+ * {@link org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable} have only data
+ * readers, -writers, and certain event callbacks. The task connects those to the
+ * network stack and actor messages, and tracks the state of the execution and
+ * handles exceptions.</p>
+ *
+ * <p>Tasks have no knowledge about how they relate to other tasks, or whether they
+ * are the first attempt to execute the task, or a repeated attempt. All of that
+ * is only known to the JobManager. All the task knows are its own runnable code,
+ * the task's configuration, and the IDs of the intermediate results to consume and
+ * produce (if any).</p>
+ *
+ * <p>Each Task is run by one dedicated thread.</p>
+ */
+public class Task implements Runnable {
+ /** The class logger. */
+ private static final Logger LOG = LoggerFactory.getLogger(Task.class);
+
+ /** The tread group that contains all task threads */
+ private static final ThreadGroup TASK_THREADS_GROUP = new ThreadGroup("Flink Task Threads");
+
/** For atomic state updates */
private static final AtomicReferenceFieldUpdater<Task, ExecutionState> STATE_UPDATER =
AtomicReferenceFieldUpdater.newUpdater(Task.class, ExecutionState.class, "executionState");
- /** The log object used for debugging. */
- private static final Logger LOG = LoggerFactory.getLogger(Task.class);
-
- // --------------------------------------------------------------------------------------------
+ // ------------------------------------------------------------------------
+ // Constant fields that are part of the initial Task construction
+ // ------------------------------------------------------------------------
+ /** The job that the task belongs to */
private final JobID jobId;
+ /** The vertex in the JobGraph whose code the task executes */
private final JobVertexID vertexId;
+ /** The execution attempt of the parallel subtask */
+ private final ExecutionAttemptID executionId;
+
+ /** The index of the parallel subtask, in [0, numberOfSubtasks) */
private final int subtaskIndex;
- private final int numberOfSubtasks;
-
- private final ExecutionAttemptID executionId;
+ /** The number of parallel subtasks for the JobVertex/ExecutionJobVertex that this task belongs to */
+ private final int parallelism;
+ /** The name of the task */
private final String taskName;
+ /** The name of the task, including the subtask index and the parallelism */
+ private final String taskNameWithSubtask;
+
+ /** The job-wide configuration object */
+ private final Configuration jobConfiguration;
+
+ /** The task-specific configuration */
+ private final Configuration taskConfiguration;
+
+ /** The jar files used by this task */
+ private final List<BlobKey> requiredJarFiles;
+
+ /** The name of the class that holds the invokable code */
+ private final String nameOfInvokableClass;
+
+ /** The handle to the state that the operator was initialized with */
+ private final StateHandle operatorState;
+
+ /** The memory manager to be used by this task */
+ private final MemoryManager memoryManager;
+
+ /** The I/O manager to be used by this task */
+ private final IOManager ioManager;
+
+ /** The BroadcastVariableManager to be used by this task */
+ private final BroadcastVariableManager broadcastVariableManager;
+
+ private final ResultPartition[] producedPartitions;
+
+ private final ResultPartitionWriter[] writers;
+
+ private final SingleInputGate[] inputGates;
+
+ private final Map<IntermediateDataSetID, SingleInputGate> inputGatesById;
+
+ /** The TaskManager actor that spawned this task */
private final ActorRef taskManager;
- private final List<ActorRef> executionListenerActors = new CopyOnWriteArrayList<ActorRef>();
+ /** The JobManager actor */
+ private final ActorRef jobManager;
+
+ /** All actors that want to be notified about changes in the task's execution state */
+ private final List<ActorRef> executionListenerActors;
+
+ /** The timeout for all ask operations on actors */
+ private final Timeout actorAskTimeout;
+
+ private final LibraryCacheManager libraryCache;
+
+ private final FileCache fileCache;
+
+ private final NetworkEnvironment network;
- /** The environment (with the invokable) executed by this task */
- private volatile RuntimeEnvironment environment;
+ /** The thread that executes the task */
+ private final Thread executingThread;
+
+ // ------------------------------------------------------------------------
+ // Fields that control the task execution
+ // ------------------------------------------------------------------------
+
+ private final AtomicBoolean invokableHasBeenCanceled = new AtomicBoolean(false);
+
+ /** The invokable of this task, if initialized */
+ private volatile AbstractInvokable invokable;
+
/** The current execution state of the task */
- private volatile ExecutionState executionState = ExecutionState.DEPLOYING;
+ private volatile ExecutionState executionState = ExecutionState.CREATED;
+ /** The observed exception, in case the task execution failed */
private volatile Throwable failureCause;
- // --------------------------------------------------------------------------------------------
+
+ /**
+ * <p><b>IMPORTANT:</b> This constructor may not start any work that would need to
+ * be undone in the case of a failing task deployment.</p>
+ */
+ public Task(TaskDeploymentDescriptor tdd,
+ MemoryManager memManager,
+ IOManager ioManager,
+ NetworkEnvironment networkEnvironment,
+ BroadcastVariableManager bcVarManager,
+ ActorRef taskManagerActor,
+ ActorRef jobManagerActor,
+ FiniteDuration actorAskTimeout,
+ LibraryCacheManager libraryCache,
+ FileCache fileCache)
+ {
+ checkArgument(tdd.getNumberOfSubtasks() > 0);
+ checkArgument(tdd.getIndexInSubtaskGroup() >= 0);
+ checkArgument(tdd.getIndexInSubtaskGroup() < tdd.getNumberOfSubtasks());
+
+ this.jobId = checkNotNull(tdd.getJobID());
+ this.vertexId = checkNotNull(tdd.getVertexID());
+ this.executionId = checkNotNull(tdd.getExecutionId());
+ this.subtaskIndex = tdd.getIndexInSubtaskGroup();
+ this.parallelism = tdd.getNumberOfSubtasks();
+ this.taskName = checkNotNull(tdd.getTaskName());
+ this.taskNameWithSubtask = getTaskNameWithSubtask(taskName, subtaskIndex, parallelism);
+ this.jobConfiguration = checkNotNull(tdd.getJobConfiguration());
+ this.taskConfiguration = checkNotNull(tdd.getTaskConfiguration());
+ this.requiredJarFiles = checkNotNull(tdd.getRequiredJarFiles());
+ this.nameOfInvokableClass = checkNotNull(tdd.getInvokableClassName());
+ this.operatorState = tdd.getOperatorStates();
+
+ this.memoryManager = checkNotNull(memManager);
+ this.ioManager = checkNotNull(ioManager);
+ this.broadcastVariableManager =checkNotNull(bcVarManager);
+
+ this.jobManager = checkNotNull(jobManagerActor);
+ this.taskManager = checkNotNull(taskManagerActor);
+ this.actorAskTimeout = new Timeout(checkNotNull(actorAskTimeout));
+
+ this.libraryCache = checkNotNull(libraryCache);
+ this.fileCache = checkNotNull(fileCache);
+ this.network = checkNotNull(networkEnvironment);
+
+ this.executionListenerActors = new CopyOnWriteArrayList<ActorRef>();
+
+ // create the reader and writer structures
+
+ final String taskNameWithSubtasksAndId =
+ Task.getTaskNameWithSubtaskAndID(taskName, subtaskIndex, parallelism, executionId);
+
+ List<ResultPartitionDeploymentDescriptor> partitions = tdd.getProducedPartitions();
+ List<InputGateDeploymentDescriptor> consumedPartitions = tdd.getInputGates();
+
+ // Produced intermediate result partitions
+ this.producedPartitions = new ResultPartition[partitions.size()];
+ this.writers = new ResultPartitionWriter[partitions.size()];
+
+ for (int i = 0; i < this.producedPartitions.length; i++) {
+ ResultPartitionDeploymentDescriptor desc = partitions.get(i);
+ ResultPartitionID partitionId = new ResultPartitionID(desc.getPartitionId(), executionId);
+
+ this.producedPartitions[i] = new ResultPartition(
+ taskNameWithSubtasksAndId,
+ jobId,
+ partitionId,
+ desc.getPartitionType(),
+ desc.getNumberOfSubpartitions(),
+ networkEnvironment.getPartitionManager(),
+ networkEnvironment.getPartitionConsumableNotifier(),
+ ioManager,
+ networkEnvironment.getDefaultIOMode());
+
+ this.writers[i] = new ResultPartitionWriter(this.producedPartitions[i]);
+ }
+
+ // Consumed intermediate result partitions
+ this.inputGates = new SingleInputGate[consumedPartitions.size()];
+ this.inputGatesById = new HashMap<IntermediateDataSetID, SingleInputGate>();
- public Task(JobID jobId, JobVertexID vertexId, int taskIndex, int parallelism,
- ExecutionAttemptID executionId, String taskName, ActorRef taskManager) {
+ for (int i = 0; i < this.inputGates.length; i++) {
+ SingleInputGate gate = SingleInputGate.create(
+ taskNameWithSubtasksAndId, consumedPartitions.get(i), networkEnvironment);
- this.jobId = jobId;
- this.vertexId = vertexId;
- this.subtaskIndex = taskIndex;
- this.numberOfSubtasks = parallelism;
- this.executionId = executionId;
- this.taskName = taskName;
- this.taskManager = taskManager;
+ this.inputGates[i] = gate;
+ inputGatesById.put(gate.getConsumedResultId(), gate);
+ }
+
+ // finally, create the executing thread, but do not start it
+ executingThread = new Thread(TASK_THREADS_GROUP, this, taskNameWithSubtask);
}
- /**
- * Returns the ID of the job this task belongs to.
- */
+ // ------------------------------------------------------------------------
+ // Accessors
+ // ------------------------------------------------------------------------
+
public JobID getJobID() {
- return this.jobId;
+ return jobId;
}
- /**
- * Returns the ID of this task vertex.
- */
- public JobVertexID getVertexID() {
- return this.vertexId;
+ public JobVertexID getJobVertexId() {
+ return vertexId;
}
- /**
- * Gets the index of the parallel subtask [0, parallelism).
- */
- public int getSubtaskIndex() {
+ public ExecutionAttemptID getExecutionId() {
+ return executionId;
+ }
+
+ public int getIndexInSubtaskGroup() {
return subtaskIndex;
}
- /**
- * Gets the total number of subtasks of the task that this subtask belongs to.
- */
public int getNumberOfSubtasks() {
- return numberOfSubtasks;
+ return parallelism;
}
- /**
- * Gets the ID of the execution attempt.
- */
- public ExecutionAttemptID getExecutionId() {
- return executionId;
+ public String getTaskName() {
+ return taskName;
+ }
+
+ public String getTaskNameWithSubtasks() {
+ return taskNameWithSubtask;
+ }
+
+ public Configuration getJobConfiguration() {
+ return jobConfiguration;
+ }
+
+ public Configuration getTaskConfiguration() {
+ return this.taskConfiguration;
+ }
+
+ public ResultPartitionWriter[] getAllWriters() {
+ return writers;
+ }
+
+ public SingleInputGate[] getAllInputGates() {
+ return inputGates;
+ }
+
+ public ResultPartition[] getProducedPartitions() {
+ return producedPartitions;
+ }
+
+ public SingleInputGate getInputGateById(IntermediateDataSetID id) {
+ return inputGatesById.get(id);
+ }
+
+ public Thread getExecutingThread() {
+ return executingThread;
}
+ // ------------------------------------------------------------------------
+ // Task Execution
+ // ------------------------------------------------------------------------
+
/**
* Returns the current execution state of the task.
+ * @return The current execution state of the task.
*/
public ExecutionState getExecutionState() {
return this.executionState;
}
- public void setEnvironment(RuntimeEnvironment environment) {
- this.environment = environment;
- }
-
- public RuntimeEnvironment getEnvironment() {
- return environment;
- }
-
+ /**
+ * Checks whether the task has failed, is canceled, or is being canceled at the moment.
+ * @return True is the task in state FAILED, CANCELING, or CANCELED, false otherwise.
+ */
public boolean isCanceledOrFailed() {
return executionState == ExecutionState.CANCELING ||
executionState == ExecutionState.CANCELED ||
executionState == ExecutionState.FAILED;
}
- public String getTaskName() {
- if (LOG.isDebugEnabled()) {
- return taskName + " (" + executionId + ")";
- } else {
- return taskName;
- }
- }
-
- public String getTaskNameWithSubtasks() {
- if (LOG.isDebugEnabled()) {
- return this.taskName + " (" + (this.subtaskIndex + 1) + "/" + this.numberOfSubtasks +
- ") (" + executionId + ")";
- } else {
- return this.taskName + " (" + (this.subtaskIndex + 1) + "/" + this.numberOfSubtasks + ")";
- }
- }
-
+ /**
+ * If the task has failed, this method gets the exception that caused this task to fail.
+ * Otherwise this method returns null.
+ *
+ * @return The exception that caused the task to fail, or null, if the task has not failed.
+ */
public Throwable getFailureCause() {
return failureCause;
}
- // ----------------------------------------------------------------------------------------------------------------
- // States and Transitions
- // ----------------------------------------------------------------------------------------------------------------
-
/**
- * Marks the task as finished. This succeeds, if the task was previously in the state
- * "RUNNING", otherwise it fails. Failure indicates that the task was either
- * canceled, or set to failed.
- *
- * @return True, if the task correctly enters the state FINISHED.
+ * Starts the task's thread.
*/
- public boolean markAsFinished() {
- if (STATE_UPDATER.compareAndSet(this, ExecutionState.RUNNING, ExecutionState.FINISHED)) {
- notifyObservers(ExecutionState.FINISHED, null);
- unregisterTask();
- return true;
- }
- else {
- return false;
- }
+ public void startTaskThread() {
+ executingThread.start();
}
- public void markFailed(Throwable error) {
+ /**
+ * The core work method that bootstraps the task and executes it code
+ */
+ public void run() {
+
+ // ----------------------------
+ // Initial State transition
+ // ----------------------------
while (true) {
ExecutionState current = this.executionState;
-
- // if canceled, fine. we are done, and the jobmanager has been told
- if (current == ExecutionState.CANCELED) {
+ if (current == ExecutionState.CREATED) {
+ if (STATE_UPDATER.compareAndSet(this, ExecutionState.CREATED, ExecutionState.DEPLOYING)) {
+ // success, we can start our work
+ break;
+ }
+ }
+ else if (current == ExecutionState.FAILED) {
+ // we were immediately failed. tell the TaskManager that we reached our final state
+ notifyFinalState();
return;
}
+ else if (current == ExecutionState.CANCELING) {
+ if (STATE_UPDATER.compareAndSet(this, ExecutionState.CANCELING, ExecutionState.CANCELED)) {
+ // we were immediately canceled. tell the TaskManager that we reached our final state
+ notifyFinalState();
+ return;
+ }
+ }
+ else {
+ throw new IllegalStateException("Invalid state for beginning of task operation");
+ }
+ }
- // if canceling, we are done, but we cannot be sure that the jobmanager has been told.
- // after all, we may have recognized our failure state before the cancelling and never sent a canceled
- // message back
- else if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
- this.failureCause = error;
+ // all resource acquisitions and registrations from here on
+ // need to be undone in the end
- notifyObservers(ExecutionState.FAILED, ExceptionUtils.stringifyException(error));
- unregisterTask();
+ Map<String, Future<Path>> distributedCacheEntries = new HashMap<String, Future<Path>>();
- return;
- }
- }
- }
+ AbstractInvokable invokable = null;
- public void cancelExecution() {
- while (true) {
- ExecutionState current = this.executionState;
+ try {
+ // ----------------------------
+ // Task Bootstrap - We periodically
+ // check for canceling as a shortcut
+ // ----------------------------
- // if the task is already canceled (or canceling) or finished or failed,
- // then we need not do anything
- if (current == ExecutionState.FINISHED || current == ExecutionState.CANCELED ||
- current == ExecutionState.CANCELING || current == ExecutionState.FAILED) {
- return;
- }
+ // first of all, get a user-code classloader
+ // this may involve downloading the job's JAR files and/or classes
+ LOG.info("Loading JAR files for task " + taskNameWithSubtask);
+ final ClassLoader userCodeClassLoader = createUserCodeClassloader(libraryCache);
- if (current == ExecutionState.DEPLOYING) {
- // directly set to canceled
- if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELED)) {
+ // now load the task's invokable code
+ invokable = loadAndInstantiateInvokable(userCodeClassLoader, nameOfInvokableClass);
- notifyObservers(ExecutionState.CANCELED, null);
- unregisterTask();
- return;
- }
+ if (isCanceledOrFailed()) {
+ throw new CancelTaskException();
}
- else if (current == ExecutionState.RUNNING) {
- // go to canceling and perform the actual task canceling
- if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELING)) {
-
- notifyObservers(ExecutionState.CANCELING, null);
- try {
- this.environment.cancelExecution();
- }
- catch (Throwable e) {
- LOG.error("Error while cancelling the task.", e);
- }
- return;
+ // ----------------------------------------------------------------
+ // register the task with the network stack
+ // this operation may fail if the system does not have enough
+ // memory to run the necessary data exchanges
+ // the registration must also strictly be undone
+ // ----------------------------------------------------------------
+
+ LOG.info("Registering task at network: " + this);
+ network.registerTask(this);
+
+ // next, kick off the background copying of files for the distributed cache
+ try {
+ for (Map.Entry<String, DistributedCache.DistributedCacheEntry> entry :
+ DistributedCache.readFileInfoFromConfig(jobConfiguration))
+ {
+ LOG.info("Obtaining local cache file for '" + entry.getKey() + '\'');
+ Future<Path> cp = fileCache.createTmpFile(entry.getKey(), entry.getValue(), jobId);
+ distributedCacheEntries.put(entry.getKey(), cp);
}
}
- else {
- throw new RuntimeException("unexpected state for cancelling: " + current);
+ catch (Exception e) {
+ throw new Exception("Exception while adding files to distributed cache.", e);
}
- }
- }
- /**
- * Sets the tasks to be cancelled and reports a failure back to the master.
- */
- public void failExternally(Throwable cause) {
- while (true) {
- ExecutionState current = this.executionState;
-
- // if the task is already canceled (or canceling) or finished or failed,
- // then we need not do anything
- if (current == ExecutionState.CANCELED || current == ExecutionState.CANCELING || current == ExecutionState.FAILED) {
- return;
+ if (isCanceledOrFailed()) {
+ throw new CancelTaskException();
}
- if (current == ExecutionState.FINISHED) {
- // Set state to failed in order to correctly unregister task from network environment
- if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
- notifyObservers(ExecutionState.FAILED, null);
+ // ----------------------------------------------------------------
+ // call the user code initialization methods
+ // ----------------------------------------------------------------
- return;
+ TaskInputSplitProvider splitProvider = new TaskInputSplitProvider(jobManager,
+ jobId, vertexId, executionId, userCodeClassLoader, actorAskTimeout);
+
+ Environment env = new RuntimeEnvironment(jobId, vertexId, executionId,
+ taskName, taskNameWithSubtask, subtaskIndex, parallelism,
+ jobConfiguration, taskConfiguration,
+ userCodeClassLoader, memoryManager, ioManager, broadcastVariableManager,
+ splitProvider, distributedCacheEntries,
+ writers, inputGates, jobManager);
+
+ // let the task code create its readers and writers
+ invokable.setEnvironment(env);
+ try {
+ invokable.registerInputOutput();
+ }
+ catch (Exception e) {
+ throw new Exception("Call to registerInputOutput() of invokable failed", e);
+ }
+
+ // the very last thing before the actual execution starts running is to inject
+ // the state into the task. the state is non-empty if this is an execution
+ // of a task that failed but had backuped state from a checkpoint
+ if (operatorState != null) {
+ if (invokable instanceof OperatorStateCarrier) {
+ ((OperatorStateCarrier) invokable).injectState(operatorState);
+ }
+ else {
+ throw new IllegalStateException("Found operator state for a non-stateful task invokable");
}
}
- if (current == ExecutionState.DEPLOYING) {
- // directly set to canceled
- if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
- this.failureCause = cause;
+ // ----------------------------------------------------------------
+ // actual task core work
+ // ----------------------------------------------------------------
- notifyObservers(ExecutionState.FAILED, null);
- unregisterTask();
- return;
+ // we must make strictly sure that the invokable is accessible to teh cancel() call
+ // by the time we switched to running.
+ this.invokable = invokable;
+
+ // switch to the RUNNING state, if that fails, we have been canceled/failed in the meantime
+ if (!STATE_UPDATER.compareAndSet(this, ExecutionState.DEPLOYING, ExecutionState.RUNNING)) {
+ throw new CancelTaskException();
+ }
+
+ // notify everyone that we switched to running. especially the TaskManager needs
+ // to know this!
+ notifyObservers(ExecutionState.RUNNING, null);
+ taskManager.tell(new TaskMessages.UpdateTaskExecutionState(
+ new TaskExecutionState(jobId, executionId, ExecutionState.RUNNING)), ActorRef.noSender());
+
+ // make sure the user code classloader is accessible thread-locally
+ executingThread.setContextClassLoader(userCodeClassLoader);
+
+ // run the invokable
+ invokable.invoke();
+
+ // make sure, we enter the catch block if the task leaves the invoke() method due
+ // to the fact that it has been canceled
+ if (isCanceledOrFailed()) {
+ throw new CancelTaskException();
+ }
+
+ // ----------------------------------------------------------------
+ // finalization of a successful execution
+ // ----------------------------------------------------------------
+
+ // finish the produced partitions. if this fails, we consider the execution failed.
+ for (ResultPartition partition : producedPartitions) {
+ if (partition != null) {
+ partition.finish();
}
}
- else if (current == ExecutionState.RUNNING) {
- // go to canceling and perform the actual task canceling
- if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
- try {
- this.environment.cancelExecution();
+
+ // try to mark the task as finished
+ // if that fails, the task was canceled/failed in the meantime
+ if (STATE_UPDATER.compareAndSet(this, ExecutionState.RUNNING, ExecutionState.FINISHED)) {
+ notifyObservers(ExecutionState.FINISHED, null);
+ }
+ else {
+ throw new CancelTaskException();
+ }
+ }
+ catch (Throwable t) {
+
+ // ----------------------------------------------------------------
+ // the execution failed. either the invokable code properly failed, or
+ // an exception was thrown as a side effect of cancelling
+ // ----------------------------------------------------------------
+
+ try {
+ // transition into our final state. we should be either in RUNNING, CANCELING, or FAILED
+ // loop for multiple retries during concurrent state changes via calls to cancel() or
+ // to failExternally()
+ while (true) {
+ ExecutionState current = this.executionState;
+ if (current == ExecutionState.RUNNING || current == ExecutionState.DEPLOYING) {
+ if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
+ // proper failure of the task. record the exception as the root cause
+ failureCause = t;
+ notifyObservers(ExecutionState.FAILED, t);
+
+ // in case of an exception during execution, we still call "cancel()" on the task
+ if (invokable != null && this.invokable != null && invokableHasBeenCanceled.compareAndSet(false, true)) {
+ try {
+ invokable.cancel();
+ }
+ catch (Throwable t2) {
+ LOG.error("Error while canceling task " + taskNameWithSubtask, t2);
+ }
+ }
+ break;
+ }
}
- catch (Throwable e) {
- LOG.error("Error while cancelling the task.", e);
+ else if (current == ExecutionState.CANCELING) {
+ if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELED)) {
+ notifyObservers(ExecutionState.CANCELED, null);
+ break;
+ }
}
+ else if (current == ExecutionState.FAILED) {
+ // in state failed already, no transition necessary any more
+ break;
+ }
+ // unexpected state, go to failed
+ else if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.FAILED)) {
+ LOG.error("Unexpected state in Task during an exception: " + current);
+ break;
+ }
+ // else fall through the loop and
+ }
+ }
+ catch (Throwable tt) {
+ String message = "FATAL - exception in task exception handler";
+ LOG.error(message, tt);
+ notifyFatalError(message, tt);
+ }
+ }
+ finally {
+ try {
+ LOG.info("Freeing task resources for " + taskNameWithSubtask);
+
+ // free the network resources
+ network.unregisterTask(this);
+
+ if (invokable != null) {
+ memoryManager.releaseAll(invokable);
+ }
- this.failureCause = cause;
+ // remove all of the tasks library resources
+ libraryCache.unregisterTask(jobId, executionId);
- notifyObservers(ExecutionState.FAILED, null);
- unregisterTask();
+ // remove all files in the distributed cache
+ removeCachedFiles(distributedCacheEntries, fileCache);
- return;
- }
+ notifyFinalState();
}
- else {
- throw new RuntimeException("unexpected state for failing the task: " + current);
+ catch (Throwable t) {
+ // an error in the resource cleanup is fatal
+ String message = "FATAL - exception in task resource cleanup";
+ LOG.error(message, t);
+ notifyFatalError(message, t);
}
}
}
- public void cancelingDone() {
- while (true) {
- ExecutionState current = this.executionState;
+ private ClassLoader createUserCodeClassloader(LibraryCacheManager libraryCache) throws Exception {
+ long startDownloadTime = System.currentTimeMillis();
- if (current == ExecutionState.CANCELED || current == ExecutionState.FAILED) {
- return;
- }
- if (!(current == ExecutionState.RUNNING || current == ExecutionState.CANCELING)) {
- LOG.error(String.format("Unexpected state transition in Task: %s -> %s", current, ExecutionState.CANCELED));
- }
+ // triggers the download of all missing jar files from the job manager
+ libraryCache.registerTask(jobId, executionId, requiredJarFiles);
- if (STATE_UPDATER.compareAndSet(this, current, ExecutionState.CANCELED)) {
- notifyObservers(ExecutionState.CANCELED, null);
- unregisterTask();
- return;
- }
+ LOG.debug("Register task {} at library cache manager took {} milliseconds",
+ executionId, System.currentTimeMillis() - startDownloadTime);
+
+ ClassLoader userCodeClassLoader = libraryCache.getClassLoader(jobId);
+ if (userCodeClassLoader == null) {
+ throw new Exception("No user code classloader available.");
}
+ return userCodeClassLoader;
}
- /**
- * Starts the execution of this task.
- */
- public boolean startExecution() {
- LOG.info("Starting execution of task {}", this.getTaskName());
- if (STATE_UPDATER.compareAndSet(this, ExecutionState.DEPLOYING, ExecutionState.RUNNING)) {
- final Thread thread = this.environment.getExecutingThread();
- thread.start();
- return true;
+ private AbstractInvokable loadAndInstantiateInvokable(ClassLoader classLoader, String className) throws Exception {
+ Class<? extends AbstractInvokable> invokableClass;
+ try {
+ invokableClass = Class.forName(className, true, classLoader)
+ .asSubclass(AbstractInvokable.class);
}
- else {
- return false;
+ catch (Throwable t) {
+ throw new Exception("Could not load the task's invokable class.", t);
+ }
+ try {
+ return invokableClass.newInstance();
+ }
+ catch (Throwable t) {
+ throw new Exception("Could not instantiate the task's invokable class.", t);
}
}
- /**
- * Unregisters the task from the central memory manager.
- */
- public void unregisterMemoryManager(MemoryManager memoryManager) {
- RuntimeEnvironment env = this.environment;
- if (memoryManager != null && env != null) {
- memoryManager.releaseAll(env.getInvokable());
+ private void removeCachedFiles(Map<String, Future<Path>> entries, FileCache fileCache) {
+ // cancel and release all distributed cache files
+ try {
+ for (Map.Entry<String, Future<Path>> entry : entries.entrySet()) {
+ String name = entry.getKey();
+ try {
+ fileCache.deleteTmpFile(name, jobId);
+ }
+ catch (Exception e) {
+ // unpleasant, but we continue
+ LOG.error("Distributed Cache could not remove cached file registered under '"
+ + name + "'.", e);
+ }
+ }
+ }
+ catch (Throwable t) {
+ LOG.error("Error while removing cached local files from distributed cache.");
}
}
- protected void unregisterTask() {
- taskManager.tell(new UnregisterTask(executionId), ActorRef.noSender());
+ private void notifyFinalState() {
+ taskManager.tell(new TaskInFinalState(executionId), ActorRef.noSender());
}
- // -----------------------------------------------------------------------------------------------------------------
- // Task Profiling
- // -----------------------------------------------------------------------------------------------------------------
+ private void notifyFatalError(String message, Throwable cause) {
+ taskManager.tell(new FatalError(message, cause), ActorRef.noSender());
+ }
- /**
- * Registers the task manager profiler with the task.
- */
- public void registerProfiler(TaskManagerProfiler taskManagerProfiler, Configuration jobConfiguration) {
- taskManagerProfiler.registerTask(this, jobConfiguration);
+ // ----------------------------------------------------------------------------------------------------------------
+ // Canceling / Failing the task from the outside
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public void cancelExecution() {
+ LOG.info("Attempting to cancel task " + taskNameWithSubtask);
+ if (cancelOrFailAndCancelInvokable(ExecutionState.CANCELING)) {
+ notifyObservers(ExecutionState.CANCELING, null);
+ }
}
/**
- * Unregisters the task from the task manager profiler.
+ * Sets the tasks to be cancelled and reports a failure back to the master.
*/
- public void unregisterProfiler(TaskManagerProfiler taskManagerProfiler) {
- if (taskManagerProfiler != null) {
- taskManagerProfiler.unregisterTask(this.executionId);
+ public void failExternally(Throwable cause) {
+ LOG.info("Attempting to fail task externally " + taskNameWithSubtask);
+ if (cancelOrFailAndCancelInvokable(ExecutionState.FAILED)) {
+ failureCause = cause;
+ notifyObservers(ExecutionState.FAILED, cause);
}
}
- // ------------------------------------------------------------------------
- // Intermediate result partitions
- // ------------------------------------------------------------------------
-
- public SingleInputGate[] getInputGates() {
- return environment != null ? environment.getAllInputGates() : null;
- }
+ private boolean cancelOrFailAndCancelInvokable(ExecutionState targetState) {
+ while (true) {
+ ExecutionState current = this.executionState;
- public ResultPartitionWriter[] getWriters() {
- return environment != null ? environment.getAllWriters() : null;
- }
+ // if the task is already canceled (or canceling) or finished or failed,
+ // then we need not do anything
+ if (current.isTerminal() || current == ExecutionState.CANCELING) {
+ return false;
+ }
- public ResultPartition[] getProducedPartitions() {
- return environment != null ? environment.getProducedPartitions() : null;
+ if (current == ExecutionState.DEPLOYING || current == ExecutionState.CREATED) {
+ if (STATE_UPDATER.compareAndSet(this, current, targetState)) {
+ // if we manage this state transition, then the invokable gets never called
+ // we need not call cancel on it
+ return true;
+ }
+ }
+ else if (current == ExecutionState.RUNNING) {
+ if (STATE_UPDATER.compareAndSet(this, ExecutionState.RUNNING, targetState)) {
+ // we are canceling / failing out of the running state
+ // we need to cancel the invokable
+ if (invokable != null && invokableHasBeenCanceled.compareAndSet(false, true)) {
+ LOG.info("Triggering cancellation of task code {} ({}).", taskNameWithSubtask, executionId);
+
+ // because the canceling may block on user code, we cancel from a separate thread
+ Runnable canceler = new TaskCanceler(LOG, invokable, executingThread, taskNameWithSubtask);
+ Thread cancelThread = new Thread(executingThread.getThreadGroup(), canceler,
+ "Canceler for " + taskNameWithSubtask);
+ cancelThread.start();
+ }
+ return true;
+ }
+ }
+ else {
+ throw new IllegalStateException("Unexpected task state: " + current);
+ }
+ }
}
- // --------------------------------------------------------------------------------------------
- // State Listeners
- // --------------------------------------------------------------------------------------------
+ // ------------------------------------------------------------------------
+ // State Listeners
+ // ------------------------------------------------------------------------
public void registerExecutionListener(ActorRef listener) {
executionListenerActors.add(listener);
@@ -404,25 +777,146 @@ public class Task {
executionListenerActors.remove(listener);
}
- private void notifyObservers(ExecutionState newState, String message) {
- if (LOG.isInfoEnabled()) {
- LOG.info(getTaskNameWithSubtasks() + " switched to " + newState + (message == null ? "" : " : " + message));
+ private void notifyObservers(ExecutionState newState, Throwable error) {
+ if (error == null) {
+ LOG.info(taskNameWithSubtask + " switched to " + newState);
+ }
+ else {
+ LOG.info(taskNameWithSubtask + " switched to " + newState + " with exception.", error);
}
+ TaskExecutionState stateUpdate = new TaskExecutionState(jobId, executionId, newState, error);
+ TaskMessages.UpdateTaskExecutionState actorMessage = new
+ TaskMessages.UpdateTaskExecutionState(stateUpdate);
+
for (ActorRef listener : executionListenerActors) {
- listener.tell(new ExecutionGraphMessages.ExecutionStateChanged(
- jobId, vertexId, taskName, numberOfSubtasks, subtaskIndex,
- executionId, newState, System.currentTimeMillis(), message),
- ActorRef.noSender());
+ listener.tell(actorMessage, ActorRef.noSender());
}
}
- // --------------------------------------------------------------------------------------------
- // Utilities
- // --------------------------------------------------------------------------------------------
+ // ------------------------------------------------------------------------
+ // Notifications on the invokable
+ // ------------------------------------------------------------------------
+
+ public void triggerCheckpointBarrier(final long checkpointID) {
+ AbstractInvokable invokabe = this.invokable;
+
+ if (executionState == ExecutionState.RUNNING && invokabe != null) {
+ if (invokabe instanceof BarrierTransceiver) {
+ final BarrierTransceiver barrierTransceiver = (BarrierTransceiver) invokabe;
+ final Logger logger = LOG;
+
+ Thread caller = new Thread("Barrier emitter") {
+ @Override
+ public void run() {
+ try {
+ barrierTransceiver.broadcastBarrierFromSource(checkpointID);
+ }
+ catch (Throwable t) {
+ logger.error("Error while triggering checkpoint barriers", t);
+ }
+ }
+ };
+ caller.setDaemon(true);
+ caller.start();
+ }
+ else {
+ LOG.error("Task received a checkpoint request, but is not a checkpointing task - "
+ + taskNameWithSubtask);
+ }
+ }
+ else {
+ LOG.debug("Ignoring request to trigger a checkpoint barrier");
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
@Override
public String toString() {
return getTaskNameWithSubtasks() + " [" + executionState + ']';
}
+
+ // ------------------------------------------------------------------------
+ // Task Names
+ // ------------------------------------------------------------------------
+
+ public static String getTaskNameWithSubtask(String name, int subtask, int numSubtasks) {
+ return name + " (" + (subtask+1) + '/' + numSubtasks + ')';
+ }
+
+ public static String getTaskNameWithSubtaskAndID(String name, int subtask, int numSubtasks, ExecutionAttemptID id) {
+ return name + " (" + (subtask+1) + '/' + numSubtasks + ") (" + id + ')';
+ }
+
+ /**
+ * This runner calls cancel() on the invokable and periodically interrupts the
+ * thread until it has terminated.
+ */
+ private static class TaskCanceler implements Runnable {
+
+ private final Logger logger;
+ private final AbstractInvokable invokable;
+ private final Thread executer;
+ private final String taskName;
+
+ public TaskCanceler(Logger logger, AbstractInvokable invokable, Thread executer, String taskName) {
+ this.logger = logger;
+ this.invokable = invokable;
+ this.executer = executer;
+ this.taskName = taskName;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // the user-defined cancel method may throw errors.
+ // we need do continue despite that
+ try {
+ invokable.cancel();
+ }
+ catch (Throwable t) {
+ logger.error("Error while canceling the task", t);
+ }
+
+ // interrupt the running thread initially
+ executer.interrupt();
+ try {
+ executer.join(10000);
+ }
+ catch (InterruptedException e) {
+ // we can ignore this
+ }
+
+ // it is possible that the user code does not react immediately. for that
+ // reason, we spawn a separate thread that repeatedly interrupts the user code until
+ // it exits
+ while (executer.isAlive()) {
+
+ // build the stack trace of where the thread is stuck, for the log
+ StringBuilder bld = new StringBuilder();
+ StackTraceElement[] stack = executer.getStackTrace();
+ for (StackTraceElement e : stack) {
+ bld.append(e).append('\n');
+ }
+
+ logger.warn("Task '{}' did not react to cancelling signal, but is stuck in method:\n {}",
+ taskName, bld.toString());
+
+ executer.interrupt();
+ try {
+ executer.join(5000);
+ }
+ catch (InterruptedException e) {
+ // we can ignore this
+ }
+ }
+ }
+ catch (Throwable t) {
+ logger.error("Error in the task canceler", t);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
index c81830c..b12f1b5 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskManagerMessages.scala
@@ -24,7 +24,15 @@ import org.apache.flink.runtime.instance.InstanceID
* Miscellaneous actor messages exchanged with the TaskManager.
*/
object TaskManagerMessages {
-
+
+ /**
+ * This message informs the TaskManager about a fatal error that prevents
+ * it from continuing.
+ *
+ * @param description The description of the problem
+ */
+ case class FatalError(description: String, cause: Throwable)
+
/**
* Tells the task manager to send a heartbeat message to the job manager.
*/
@@ -49,7 +57,7 @@ object TaskManagerMessages {
// --------------------------------------------------------------------------
- // Utility messages used for notifications during TaskManager startup
+ // Reporting the current TaskManager stack trace
// --------------------------------------------------------------------------
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskMessages.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskMessages.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskMessages.scala
index c8c5726..b1a08ca 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskMessages.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/messages/TaskMessages.scala
@@ -67,12 +67,12 @@ object TaskMessages {
extends TaskMessage
/**
- * Unregister the task identified by [[executionID]] from the TaskManager.
- * Sent to the TaskManager by futures and callbacks.
+ * Notifies the TaskManager that the task has reached its final state,
+ * either FINISHED, CANCELED, or FAILED.
*
* @param executionID The task's execution attempt ID.
*/
- case class UnregisterTask(executionID: ExecutionAttemptID)
+ case class TaskInFinalState(executionID: ExecutionAttemptID)
extends TaskMessage
[4/8] flink git commit: [FLINK-1672] [runtime] Unify Task and
RuntimeEnvironment into one class.
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 747cc85..bdefea6 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.taskmanager
import java.io.{File, IOException}
import java.net.{InetAddress, InetSocketAddress}
import java.util
-import java.util.concurrent.{TimeUnit, FutureTask}
+import java.util.concurrent.TimeUnit
import java.lang.reflect.Method
import java.lang.management.{GarbageCollectorMXBean, ManagementFactory, MemoryMXBean}
@@ -36,16 +36,13 @@ import com.codahale.metrics.jvm.{MemoryUsageGaugeSet, GarbageCollectorMetricSet}
import com.fasterxml.jackson.databind.ObjectMapper
import grizzled.slf4j.Logger
-import org.apache.flink.api.common.cache.DistributedCache
import org.apache.flink.configuration._
-import org.apache.flink.core.fs.Path
import org.apache.flink.runtime.{ActorSynchronousLogging, ActorLogMessages}
import org.apache.flink.runtime.akka.AkkaUtils
import org.apache.flink.runtime.blob.{BlobService, BlobCache}
import org.apache.flink.runtime.broadcast.BroadcastVariableManager
import org.apache.flink.runtime.deployment.{InputChannelDeploymentDescriptor, TaskDeploymentDescriptor}
import org.apache.flink.runtime.execution.librarycache.{BlobLibraryCacheManager, FallbackLibraryCacheManager, LibraryCacheManager}
-import org.apache.flink.runtime.execution.{CancelTaskException, ExecutionState, RuntimeEnvironment}
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID
import org.apache.flink.runtime.filecache.FileCache
import org.apache.flink.runtime.instance.{HardwareDescription, InstanceConnectionInfo, InstanceID}
@@ -54,7 +51,6 @@ import org.apache.flink.runtime.io.disk.iomanager.{IOManager, IOManagerAsync}
import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.io.network.netty.NettyConfig
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID
-import org.apache.flink.runtime.jobgraph.tasks.{OperatorStateCarrier,BarrierTransceiver}
import org.apache.flink.runtime.jobmanager.JobManager
import org.apache.flink.runtime.memorymanager.{MemoryManager, DefaultMemoryManager}
import org.apache.flink.runtime.messages.CheckpointingMessages.{CheckpointingMessage, BarrierReq}
@@ -67,9 +63,6 @@ import org.apache.flink.runtime.process.ProcessReaper
import org.apache.flink.runtime.security.SecurityUtils
import org.apache.flink.runtime.security.SecurityUtils.FlinkSecuredRunner
import org.apache.flink.runtime.util.{MathUtils, EnvironmentInformation}
-import org.apache.flink.util.ExceptionUtils
-
-import org.slf4j.LoggerFactory
import scala.concurrent._
import scala.concurrent.duration._
@@ -136,13 +129,13 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
protected val resources = HardwareDescription.extractFromSystem(memoryManager.getMemorySize)
/** Registry of all tasks currently executed by this TaskManager */
- protected val runningTasks = new util.concurrent.ConcurrentHashMap[ExecutionAttemptID, Task]()
+ protected val runningTasks = new java.util.HashMap[ExecutionAttemptID, Task]()
/** Handler for shared broadcast variables (shared between multiple Tasks) */
protected val bcVarManager = new BroadcastVariableManager()
/** Handler for distributed files cached by this TaskManager */
- protected val fileCache = new FileCache()
+ protected val fileCache = new FileCache(config.configuration)
/** Registry of metrics periodically transmitted to the JobManager */
private val metricRegistry = TaskManager.createMetricsRegistry()
@@ -282,6 +275,9 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
case Disconnect(msg) =>
handleJobManagerDisconnect(sender(), "JobManager requested disconnect: " + msg)
+
+ case FatalError(message, cause) =>
+ killTaskManagerFatal(message, cause)
}
/**
@@ -344,12 +340,13 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
// state transition
case updateMsg @ UpdateTaskExecutionState(taskExecutionState: TaskExecutionState) =>
+
+ // we receive these from our tasks and forward them to the JobManager
currentJobManager foreach {
jobManager => {
val futureResponse = (jobManager ? updateMsg)(askTimeout)
-
+
val executionID = taskExecutionState.getID
- val executionState = taskExecutionState.getExecutionState
futureResponse.mapTo[Boolean].onComplete {
// IMPORTANT: In the future callback, we cannot directly modify state
@@ -359,21 +356,16 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
self ! FailTask(executionID,
new Exception("Task has been cancelled on the JobManager."))
}
-
- if (!result || executionState.isTerminal) {
- self ! UnregisterTask(executionID)
- }
+
case Failure(t) =>
self ! FailTask(executionID, new Exception(
"Failed to send ExecutionStateChange notification to JobManager"))
-
- self ! UnregisterTask(executionID)
}(context.dispatcher)
}
}
// removes the task from the TaskManager and frees all its resources
- case UnregisterTask(executionID) =>
+ case TaskInFinalState(executionID) =>
unregisterTaskAndNotifyFinalState(executionID)
// starts a new task on the TaskManager
@@ -383,35 +375,22 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
// marks a task as failed for an external reason
// external reasons are reasons other than the task code itself throwing an exception
case FailTask(executionID, cause) =>
- Option(runningTasks.get(executionID)) match {
- case Some(task) =>
-
- // execute failing operation concurrently
- implicit val executor = context.dispatcher
- Future {
- task.failExternally(cause)
- }.onFailure{
- case t: Throwable => log.error(s"Could not fail task ${task} externally.", t)
- }
- case None =>
+ val task = runningTasks.get(executionID)
+ if (task != null) {
+ task.failExternally(cause)
+ } else {
+ log.debug(s"Cannot find task to fail for execution ${executionID})")
}
// cancels a task
case CancelTask(executionID) =>
- Option(runningTasks.get(executionID)) match {
- case Some(task) =>
- // execute cancel operation concurrently
- implicit val executor = context.dispatcher
- Future {
- task.cancelExecution()
- }.onFailure{
- case t: Throwable => log.error("Could not cancel task " + task, t)
- }
-
- sender ! new TaskOperationResult(executionID, true)
-
- case None =>
- sender ! new TaskOperationResult(executionID, false,
+ val task = runningTasks.get(executionID)
+ if (task != null) {
+ task.cancelExecution()
+ sender ! new TaskOperationResult(executionID, true)
+ } else {
+ log.debug(s"Cannot find task to cancel for execution ${executionID})")
+ sender ! new TaskOperationResult(executionID, false,
"No task with that execution ID was found.")
}
}
@@ -430,24 +409,11 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
log.debug(s"[FT-TaskManager] Barrier $checkpointID request received " +
s"for attempt $attemptID.")
- Option(runningTasks.get(attemptID)) match {
- case Some(i) =>
- if (i.getExecutionState == ExecutionState.RUNNING) {
- i.getEnvironment.getInvokable match {
- case barrierTransceiver: BarrierTransceiver =>
- new Thread(new Runnable {
- override def run(): Unit =
- barrierTransceiver.broadcastBarrierFromSource(checkpointID)
- }).start()
-
- case _ => log.error("Taskmanager received a checkpoint request for " +
- s"non-checkpointing task $attemptID.")
- }
- }
-
- case None =>
- // may always happen in case of canceled/finished tasks
- log.debug(s"Taskmanager received a checkpoint request for unknown task $attemptID.")
+ val task = runningTasks.get(attemptID)
+ if (task != null) {
+ task.triggerCheckpointBarrier(checkpointID)
+ } else {
+ log.debug(s"Taskmanager received a checkpoint request for unknown task $attemptID.")
}
// unknown checkpoint message
@@ -770,8 +736,7 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
}
}
}
-
-
+
// --------------------------------------------------------------------------
// Task Operations
// --------------------------------------------------------------------------
@@ -784,130 +749,46 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
* @param tdd TaskDeploymentDescriptor describing the task to be executed on this [[TaskManager]]
*/
private def submitTask(tdd: TaskDeploymentDescriptor): Unit = {
- val slot = tdd.getTargetSlotNumber
-
- if (!isConnected) {
- sender ! Failure(
- new IllegalStateException("TaskManager is not associated with a JobManager.")
- )
- } else if (slot < 0 || slot >= numberOfSlots) {
- sender ! Failure(new Exception(s"Target slot $slot does not exist on TaskManager."))
- } else {
- sender ! Acknowledge
-
- Future {
- initializeTask(tdd)
- }(context.dispatcher)
- }
- }
-
- /** Sets up a [[org.apache.flink.runtime.execution.RuntimeEnvironment]] for the task and starts
- * its execution in a separate thread.
- *
- * @param tdd TaskDeploymentDescriptor describing the task to be executed on this [[TaskManager]]
- */
- private def initializeTask(tdd: TaskDeploymentDescriptor): Unit ={
- val jobID = tdd.getJobID
- val vertexID = tdd.getVertexID
- val executionID = tdd.getExecutionId
- val taskIndex = tdd.getIndexInSubtaskGroup
- val numSubtasks = tdd.getNumberOfSubtasks
- var startRegisteringTask = 0L
- var task: Task = null
-
try {
- val userCodeClassLoader = libraryCacheManager match {
- case Some(manager) =>
- if (log.isDebugEnabled) {
- startRegisteringTask = System.currentTimeMillis()
- }
-
- // triggers the download of all missing jar files from the job manager
- manager.registerTask(jobID, executionID, tdd.getRequiredJarFiles)
-
- if (log.isDebugEnabled) {
- log.debug(s"Register task $executionID at library cache manager " +
- s"took ${(System.currentTimeMillis() - startRegisteringTask) / 1000.0}s")
- }
-
- manager.getClassLoader(jobID)
- case None => throw new IllegalStateException("There is no valid library cache manager.")
- }
-
- if (userCodeClassLoader == null) {
- throw new RuntimeException("No user code Classloader available.")
- }
-
- task = new Task(jobID, vertexID, taskIndex, numSubtasks, executionID,
- tdd.getTaskName, self)
-
- Option(runningTasks.put(executionID, task)) match {
- case Some(_) => throw new RuntimeException(
- s"TaskManager contains already a task with executionID $executionID.")
+ // grab some handles and sanity check on the fly
+ val jobManagerActor = currentJobManager match {
+ case Some(jm) => jm
case None =>
+ throw new IllegalStateException("TaskManager is not associated with a JobManager.")
}
-
- val env = currentJobManager match {
- case Some(jobManager) =>
- val splitProvider = new TaskInputSplitProvider(jobManager, jobID, vertexID,
- executionID, userCodeClassLoader, askTimeout)
-
- new RuntimeEnvironment(jobManager, task, tdd, userCodeClassLoader,
- memoryManager, ioManager, splitProvider, bcVarManager, network)
-
- case None => throw new IllegalStateException(
- "TaskManager has not yet been registered at a JobManager.")
- }
-
- task.setEnvironment(env)
-
- //inject operator state
- if (tdd.getOperatorStates != null) {
- task.getEnvironment.getInvokable match {
- case opStateCarrier: OperatorStateCarrier =>
- opStateCarrier.injectState(tdd.getOperatorStates)
- }
+ val libCache = libraryCacheManager match {
+ case Some(manager) => manager
+ case None => throw new IllegalStateException("There is no valid library cache manager.")
}
- // register the task with the network stack and profiles
- log.info(s"Register task $task.")
- network.registerTask(task)
-
- val cpTasks = new util.HashMap[String, FutureTask[Path]]()
-
- for (entry <- DistributedCache.readFileInfoFromConfig(tdd.getJobConfiguration).asScala) {
- val cp = fileCache.createTmpFile(entry.getKey, entry.getValue, jobID)
- cpTasks.put(entry.getKey, cp)
+ val slot = tdd.getTargetSlotNumber
+ if (slot < 0 || slot >= numberOfSlots) {
+ throw new IllegalArgumentException(s"Target slot $slot does not exist on TaskManager.")
}
- env.addCopyTasksForCacheFile(cpTasks)
- if (!task.startExecution()) {
- throw new RuntimeException("Cannot start task. Task was canceled or failed.")
+ // create the task. this does not grab any TaskManager resources or download
+ // and libraries - the operation does not block
+ val execId = tdd.getExecutionId
+ val task = new Task(tdd, memoryManager, ioManager, network, bcVarManager,
+ self, jobManagerActor, config.timeout, libCache, fileCache)
+
+ // add the task to the map
+ val prevTask = runningTasks.put(execId, task)
+ if (prevTask != null) {
+ // already have a task for that ID, put if back and report an error
+ runningTasks.put(execId, prevTask)
+ throw new IllegalStateException("TaskManager already contains a task for id " + execId)
}
-
- self ! UpdateTaskExecutionState(
- new TaskExecutionState(jobID, executionID, ExecutionState.RUNNING)
- )
- } catch {
- case t: Throwable =>
- if (!t.isInstanceOf[CancelTaskException]) {
- log.error("Could not instantiate task with execution ID " + executionID, t)
- }
-
- try {
- if (task != null) {
- task.failExternally(t)
- removeAllTaskResources(task)
- }
-
- libraryCacheManager foreach { _.unregisterTask(jobID, executionID) }
- } catch {
- case t: Throwable => log.error("Error during cleanup of task deployment.", t)
- }
-
- self ! UpdateTaskExecutionState(
- new TaskExecutionState(jobID, executionID, ExecutionState.FAILED, t)
- )
+
+ // all good, we kick off the task, which performs its own initialization
+ task.startTaskThread()
+
+ sender ! Acknowledge
+ }
+ catch {
+ case t: Throwable =>
+ log.error("SubmitTask failed", t)
+ sender ! Failure(t)
}
}
@@ -927,19 +808,20 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
val errors: Seq[String] = partitionInfos.flatMap { info =>
val (resultID, partitionInfo) = info
- val reader = task.getEnvironment.getInputGateById(resultID)
+ val reader = task.getInputGateById(resultID)
if (reader != null) {
Future {
try {
reader.updateInputChannel(partitionInfo)
- } catch {
+ }
+ catch {
case t: Throwable =>
log.error(s"Could not update input data location for task " +
s"${task.getTaskName}. Trying to fail task.", t)
try {
- task.markFailed(t)
+ task.failExternally(t)
}
catch {
case t: Throwable =>
@@ -977,20 +859,20 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
private def cancelAndClearEverything(cause: Throwable) {
if (runningTasks.size > 0) {
log.info("Cancelling all computations and discarding all cached data.")
-
+
for (t <- runningTasks.values().asScala) {
t.failExternally(cause)
- unregisterTaskAndNotifyFinalState(t.getExecutionId)
}
+ runningTasks.clear()
}
}
private def unregisterTaskAndNotifyFinalState(executionID: ExecutionAttemptID): Unit = {
- Option(runningTasks.remove(executionID)) match {
- case Some(task) =>
-
- // mark the task as failed if it is not yet in a final state
+ val task = runningTasks.remove(executionID)
+ if (task != null) {
+
+ // the task must be in a terminal state
if (!task.getExecutionState.isTerminal) {
try {
task.failExternally(new Exception("Task is being removed from TaskManager"))
@@ -999,66 +881,15 @@ extends Actor with ActorLogMessages with ActorSynchronousLogging {
}
}
- log.info(s"Unregister task with execution ID $executionID.")
- removeAllTaskResources(task)
- libraryCacheManager foreach { _.unregisterTask(task.getJobID, executionID) }
-
- log.info(s"Updating FINAL execution state of ${task.getTaskName} " +
- s"(${task.getExecutionId}) to ${task.getExecutionState}.")
+ log.info(s"Unregistering task and sending final execution state " +
+ s"${task.getExecutionState} to JobManager for task ${task.getTaskName} " +
+ s"(${task.getExecutionId})")
self ! UpdateTaskExecutionState(new TaskExecutionState(
task.getJobID, task.getExecutionId, task.getExecutionState, task.getFailureCause))
-
- case None =>
- log.debug(s"Cannot find task with ID $executionID to unregister.")
}
- }
-
- /**
- * This method cleans up the resources of a task in the distributed cache,
- * network stack and the memory manager.
- *
- * If the cleanup in the network stack or memory manager fails, this is considered
- * a fatal problem (critical resource leak) and causes the TaskManager to quit.
- * A TaskManager JVM restart is the best safe way to fix that error.
- *
- * @param task The Task whose resources should be cleared.
- */
- private def removeAllTaskResources(task: Task): Unit = {
-
- // release the critical things first, and fail fatally if it does not work
-
- // this releases all task resources, like buffer pools and intermediate result
- // partitions being built. If this fails, the TaskManager is in serious trouble,
- // as this is a massive resource leak. We kill the TaskManager in that case,
- // to recover through a clean JVM start
- try {
- network.unregisterTask(task)
- } catch {
- case t: Throwable =>
- killTaskManagerFatal("Failed to unregister task resources from network stack", t)
- }
-
- // safety net to release all the task's memory
- try {
- task.unregisterMemoryManager(memoryManager)
- } catch {
- case t: Throwable =>
- killTaskManagerFatal("Failed to unregister task memory from memory manager", t)
- }
-
- // release temp files from the distributed cache
- if (task.getEnvironment != null) {
- try {
- for (entry <- DistributedCache.readFileInfoFromConfig(
- task.getEnvironment.getJobConfiguration).asScala) {
- fileCache.deleteTmpFile(entry.getKey, entry.getValue, task.getJobID)
- }
- } catch {
- // this is pretty unpleasant, but not a reason to give up immediately
- case e: Exception => log.error(
- "Error cleaning up local temp files from the distributed cache.", e)
- }
+ else {
+ log.error(s"Cannot find task with ID $executionID to unregister.")
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
index d13db05..fdf41f0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
@@ -19,8 +19,8 @@
package org.apache.flink.runtime.io.network.partition.consumer;
import com.google.common.collect.Lists;
+
import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.TaskEventDispatcher;
import org.apache.flink.runtime.io.network.buffer.BufferPool;
@@ -36,6 +36,7 @@ import org.apache.flink.runtime.io.network.util.TestPartitionProducer;
import org.apache.flink.runtime.io.network.util.TestProducerSource;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
+
import org.junit.Test;
import java.util.Collections;
@@ -93,7 +94,7 @@ public class LocalInputChannelTest {
partitionIds[i] = new ResultPartitionID();
final ResultPartition partition = new ResultPartition(
- mock(Environment.class),
+ "Test Name",
jobId,
partitionIds[i],
ResultPartitionType.PIPELINED,
@@ -222,7 +223,7 @@ public class LocalInputChannelTest {
checkArgument(numberOfExpectedBuffersPerChannel >= 1);
this.inputGate = new SingleInputGate(
- mock(Environment.class),
+ "Test Name",
new IntermediateDataSetID(),
subpartitionIndex,
numberOfInputChannels);
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
index be66aeb..9a7ffe5 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
@@ -22,8 +22,6 @@ import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.deployment.InputChannelDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionLocation;
import org.apache.flink.runtime.event.task.TaskEvent;
-import org.apache.flink.runtime.execution.Environment;
-import org.apache.flink.runtime.execution.RuntimeEnvironment;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.network.ConnectionManager;
import org.apache.flink.runtime.io.network.TaskEventDispatcher;
@@ -61,7 +59,7 @@ public class SingleInputGateTest {
public void testBasicGetNextLogic() throws Exception {
// Setup
final SingleInputGate inputGate = new SingleInputGate(
- mock(Environment.class), new IntermediateDataSetID(), 0, 2);
+ "Test Task Name", new IntermediateDataSetID(), 0, 2);
final TestInputChannel[] inputChannels = new TestInputChannel[]{
new TestInputChannel(inputGate, 0),
@@ -107,7 +105,7 @@ public class SingleInputGateTest {
// Setup reader with one local and one unknown input channel
final IntermediateDataSetID resultId = new IntermediateDataSetID();
- final SingleInputGate inputGate = new SingleInputGate(mock(Environment.class), resultId, 0, 2);
+ final SingleInputGate inputGate = new SingleInputGate("Test Task Name", resultId, 0, 2);
final BufferPool bufferPool = mock(BufferPool.class);
when(bufferPool.getNumberOfRequiredMemorySegments()).thenReturn(2);
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestSingleInputGate.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestSingleInputGate.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestSingleInputGate.java
index efc02de..2dafaa2 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestSingleInputGate.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestSingleInputGate.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.io.network.partition.consumer;
-import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
@@ -30,7 +29,6 @@ import java.util.List;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkElementIndex;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
/**
@@ -50,7 +48,7 @@ public class TestSingleInputGate {
checkArgument(numberOfInputChannels >= 1);
this.inputGate = spy(new SingleInputGate(
- mock(Environment.class), new IntermediateDataSetID(), 0, numberOfInputChannels));
+ "Test Task Name", new IntermediateDataSetID(), 0, numberOfInputChannels));
this.inputChannels = new TestInputChannel[numberOfInputChannels];
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java
index 90132ff..050f43a 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java
@@ -18,14 +18,13 @@
package org.apache.flink.runtime.io.network.partition.consumer;
-import org.apache.flink.runtime.execution.Environment;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
+
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
public class UnionInputGateTest {
@@ -39,9 +38,9 @@ public class UnionInputGateTest {
@Test(timeout = 120 * 1000)
public void testBasicGetNextLogic() throws Exception {
// Setup
- final Environment env = mock(Environment.class);
- final SingleInputGate ig1 = new SingleInputGate(env, new IntermediateDataSetID(), 0, 3);
- final SingleInputGate ig2 = new SingleInputGate(env, new IntermediateDataSetID(), 0, 5);
+ final String testTaskName = "Test Task";
+ final SingleInputGate ig1 = new SingleInputGate(testTaskName, new IntermediateDataSetID(), 0, 3);
+ final SingleInputGate ig2 = new SingleInputGate(testTaskName, new IntermediateDataSetID(), 0, 5);
final UnionInputGate union = new UnionInputGate(new SingleInputGate[]{ig1, ig2});
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
index 16cd66e..735f67e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.operators.testutils;
+import akka.actor.ActorRef;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
@@ -261,4 +262,9 @@ public class MockEnvironment implements Environment {
public void reportAccumulators(Map<String, Accumulator<?, ?>> accumulators) {
// discard, this is only for testing
}
+
+ @Override
+ public ActorRef getJobManager() {
+ return ActorRef.noSender();
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/ForwardingActor.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/ForwardingActor.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/ForwardingActor.java
new file mode 100644
index 0000000..70e6f22
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/ForwardingActor.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.taskmanager;
+
+import akka.actor.UntypedActor;
+
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * Actor for testing that simply puts all its messages into a
+ * blocking queue.
+ */
+class ForwardingActor extends UntypedActor {
+
+ private final BlockingQueue<Object> queue;
+
+ public ForwardingActor(BlockingQueue<Object> queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public void onReceive(Object message) {
+ queue.add(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskExecutionStateTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskExecutionStateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskExecutionStateTest.java
index 9d10591..f4c7a57 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskExecutionStateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskExecutionStateTest.java
@@ -21,6 +21,8 @@ package org.apache.flink.runtime.taskmanager;
import static org.junit.Assert.*;
import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
@@ -84,4 +86,35 @@ public class TaskExecutionStateTest {
fail(e.getMessage());
}
}
+
+ @Test
+ public void hanleNonSerializableException() {
+ try {
+ @SuppressWarnings({"ThrowableInstanceNeverThrown", "serial"})
+ Exception hostile = new Exception() {
+ // should be non serializable, because it contains the outer class reference
+
+ @Override
+ public String getMessage() {
+ throw new RuntimeException("Cannot get Message");
+ }
+
+ @Override
+ public void printStackTrace(PrintStream s) {
+ throw new RuntimeException("Cannot print");
+ }
+
+ @Override
+ public void printStackTrace(PrintWriter s) {
+ throw new RuntimeException("Cannot print");
+ }
+ };
+
+ new TaskExecutionState(new JobID(), new ExecutionAttemptID(), ExecutionState.FAILED, hostile);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
index 056ed37..d84cb37 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskManagerTest.java
@@ -29,6 +29,7 @@ import akka.testkit.JavaTestKit;
import akka.util.Timeout;
import org.apache.flink.configuration.ConfigConstants;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
@@ -56,11 +57,14 @@ import org.apache.flink.runtime.messages.TaskMessages.SubmitTask;
import org.apache.flink.runtime.messages.TaskMessages.TaskOperationResult;
import org.apache.flink.runtime.testingUtils.TestingTaskManager;
import org.apache.flink.runtime.testingUtils.TestingTaskManagerMessages;
-import org.apache.flink.runtime.testingUtils.TestingUtils;
+
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import scala.Option;
import scala.concurrent.Await;
import scala.concurrent.Future;
@@ -75,42 +79,61 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
@SuppressWarnings("serial")
public class TaskManagerTest {
- private static ActorSystem system;
-
- private static Timeout timeout = new Timeout(1, TimeUnit.MINUTES);
+ private static final Logger LOG = LoggerFactory.getLogger(TaskManagerTest.class);
+
+ private static final Timeout timeout = new Timeout(1, TimeUnit.MINUTES);
- private final FiniteDuration d = new FiniteDuration(20, TimeUnit.SECONDS);
+ private static final FiniteDuration d = new FiniteDuration(20, TimeUnit.SECONDS);
+ private static ActorSystem system;
+
@BeforeClass
public static void setup() {
- system = ActorSystem.create("TestActorSystem", TestingUtils.testConfig());
+ system = AkkaUtils.createLocalActorSystem(new Configuration());
}
@AfterClass
public static void teardown() {
JavaTestKit.shutdownActorSystem(system);
- system = null;
}
@Test
- public void testSetupTaskManager() {
+ public void testSubmitAndExecuteTask() {
+
+ LOG.info( "--------------------------------------------------------------------\n" +
+ " Starting testSubmitAndExecuteTask() \n" +
+ "--------------------------------------------------------------------");
+
+
new JavaTestKit(system){{
- ActorRef jobManager = null;
+
ActorRef taskManager = null;
+
try {
- jobManager = system.actorOf(Props.create(SimpleJobManager.class));
-
- taskManager = createTaskManager(jobManager);
+ taskManager = createTaskManager(getTestActor(), false);
+ final ActorRef tmClosure = taskManager;
+
+ // handle the registration
+ new Within(d) {
+ @Override
+ protected void run() {
+ expectMsgClass(RegistrationMessages.RegisterTaskManager.class);
+
+ final InstanceID iid = new InstanceID();
+ assertEquals(tmClosure, getLastSender());
+ tmClosure.tell(new RegistrationMessages.AcknowledgeRegistration(
+ getTestActor(), iid, 12345), getTestActor());
+ }
+ };
- JobID jid = new JobID();
- JobVertexID vid = new JobVertexID();
+ final JobID jid = new JobID();
+ final JobVertexID vid = new JobVertexID();
final ExecutionAttemptID eid = new ExecutionAttemptID();
final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, vid, eid, "TestTask", 2, 7,
@@ -119,13 +142,54 @@ public class TaskManagerTest {
Collections.<InputGateDeploymentDescriptor>emptyList(),
new ArrayList<BlobKey>(), 0);
- final ActorRef tmClosure = taskManager;
+
new Within(d) {
@Override
protected void run() {
tmClosure.tell(new SubmitTask(tdd), getRef());
- expectMsgEquals(Messages.getAcknowledge());
+
+ // TaskManager should acknowledge the submission
+ // heartbeats may be interleaved
+ long deadline = System.currentTimeMillis() + 10000;
+ do {
+ Object message = receiveOne(d);
+ if (message == Messages.getAcknowledge()) {
+ break;
+ }
+ } while (System.currentTimeMillis() < deadline);
+
+ // task should have switched to running
+ Object toRunning = new TaskMessages.UpdateTaskExecutionState(
+ new TaskExecutionState(jid, eid, ExecutionState.RUNNING));
+
+ // task should have switched to finished
+ Object toFinished = new TaskMessages.UpdateTaskExecutionState(
+ new TaskExecutionState(jid, eid, ExecutionState.FINISHED));
+
+ deadline = System.currentTimeMillis() + 10000;
+ do {
+ Object message = receiveOne(d);
+ if (message.equals(toRunning)) {
+ break;
+ }
+ else if (!(message instanceof TaskManagerMessages.Heartbeat)) {
+ fail("Unexpected message: " + message);
+ }
+ } while (System.currentTimeMillis() < deadline);
+
+ deadline = System.currentTimeMillis() + 10000;
+ do {
+ Object message = receiveOne(d);
+ if (message.equals(toFinished)) {
+ break;
+ }
+ else if (!(message instanceof TaskManagerMessages.Heartbeat)) {
+ fail("Unexpected message: " + message);
+ }
+ } while (System.currentTimeMillis() < deadline);
+
+
}
};
}
@@ -138,22 +202,24 @@ public class TaskManagerTest {
if (taskManager != null) {
taskManager.tell(Kill.getInstance(), ActorRef.noSender());
}
- if (jobManager != null) {
- jobManager.tell(Kill.getInstance(), ActorRef.noSender());
- }
}
}};
}
@Test
public void testJobSubmissionAndCanceling() {
+
+ LOG.info( "--------------------------------------------------------------------\n" +
+ " Starting testJobSubmissionAndCanceling() \n" +
+ "--------------------------------------------------------------------");
+
new JavaTestKit(system){{
ActorRef jobManager = null;
ActorRef taskManager = null;
try {
jobManager = system.actorOf(Props.create(SimpleJobManager.class));
- taskManager = createTaskManager(jobManager);
+ taskManager = createTaskManager(jobManager, true);
final JobID jid1 = new JobID();
final JobID jid2 = new JobID();
@@ -274,6 +340,11 @@ public class TaskManagerTest {
@Test
public void testGateChannelEdgeMismatch() {
+
+ LOG.info( "--------------------------------------------------------------------\n" +
+ " Starting testGateChannelEdgeMismatch() \n" +
+ "--------------------------------------------------------------------");
+
new JavaTestKit(system){{
ActorRef jobManager = null;
@@ -281,7 +352,7 @@ public class TaskManagerTest {
try {
jobManager = system.actorOf(Props.create(SimpleJobManager.class));
- taskManager = createTaskManager(jobManager);
+ taskManager = createTaskManager(jobManager, true);
final ActorRef tm = taskManager;
final JobID jid = new JobID();
@@ -353,6 +424,11 @@ public class TaskManagerTest {
@Test
public void testRunJobWithForwardChannel() {
+
+ LOG.info( "--------------------------------------------------------------------\n" +
+ " Starting testRunJobWithForwardChannel() \n" +
+ "--------------------------------------------------------------------");
+
new JavaTestKit(system){{
ActorRef jobManager = null;
@@ -368,7 +444,7 @@ public class TaskManagerTest {
jobManager = system.actorOf(Props.create(new SimpleLookupJobManagerCreator()));
- taskManager = createTaskManager(jobManager);
+ taskManager = createTaskManager(jobManager, true);
final ActorRef tm = taskManager;
IntermediateResultPartitionID partitionId = new IntermediateResultPartitionID();
@@ -470,6 +546,10 @@ public class TaskManagerTest {
@Test
public void testCancellingDependentAndStateUpdateFails() {
+ LOG.info( "--------------------------------------------------------------------\n" +
+ " Starting testCancellingDependentAndStateUpdateFails() \n" +
+ "--------------------------------------------------------------------");
+
// this tests creates two tasks. the sender sends data, and fails to send the
// state update back to the job manager
// the second one blocks to be canceled
@@ -491,7 +571,7 @@ public class TaskManagerTest {
new SimpleLookupFailingUpdateJobManagerCreator(eid2)
)
);
- taskManager = createTaskManager(jobManager);
+ taskManager = createTaskManager(jobManager, true);
final ActorRef tm = taskManager;
IntermediateResultPartitionID partitionId = new IntermediateResultPartitionID();
@@ -676,7 +756,7 @@ public class TaskManagerTest {
}
}
- public static ActorRef createTaskManager(ActorRef jobManager) {
+ public static ActorRef createTaskManager(ActorRef jobManager, boolean waitForRegistration) {
ActorRef taskManager = null;
try {
Configuration cfg = new Configuration();
@@ -695,16 +775,18 @@ public class TaskManagerTest {
fail("Could not create test TaskManager: " + e.getMessage());
}
- Future<Object> response = Patterns.ask(taskManager,
- TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(), timeout);
-
- try {
- FiniteDuration d = new FiniteDuration(100, TimeUnit.SECONDS);
- Await.ready(response, d);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail("Exception while waiting for the task manager registration: " + e.getMessage());
+ if (waitForRegistration) {
+ Future<Object> response = Patterns.ask(taskManager,
+ TaskManagerMessages.getNotifyWhenRegisteredAtJobManagerMessage(), timeout);
+
+ try {
+ FiniteDuration d = new FiniteDuration(100, TimeUnit.SECONDS);
+ Await.ready(response, d);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail("Exception while waiting for the task manager registration: " + e.getMessage());
+ }
}
return taskManager;
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
index 3a8fcd8..4492372 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskTest.java
@@ -19,72 +19,277 @@
package org.apache.flink.runtime.taskmanager;
import akka.actor.ActorRef;
+import akka.actor.ActorSystem;
+import akka.actor.Kill;
+import akka.actor.Props;
+
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.akka.AkkaUtils;
import org.apache.flink.runtime.blob.BlobKey;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor;
import org.apache.flink.runtime.execution.ExecutionState;
-import org.apache.flink.runtime.execution.RuntimeEnvironment;
+import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
+import org.apache.flink.runtime.filecache.FileCache;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.network.MockNetworkEnvironment;
import org.apache.flink.api.common.JobID;
+import org.apache.flink.runtime.io.network.NetworkEnvironment;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier;
+import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.memorymanager.MemoryManager;
-import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.runtime.messages.TaskMessages;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
+import scala.concurrent.duration.FiniteDuration;
-import java.util.ArrayList;
import java.util.Collections;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doNothing;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+/**
+ * Tests for the Task, which make sure that correct state transitions happen,
+ * and failures are correctly handled.
+ *
+ * All tests here have a set of mock actors for TaskManager, JobManager, and
+ * execution listener, which simply put the messages in a queue to be picked
+ * up by the test and validated.
+ */
public class TaskTest {
+
+ private static ActorSystem actorSystem;
+
+ private static OneShotLatch awaitLatch;
+ private static OneShotLatch triggerLatch;
+
+ private ActorRef taskManagerMock;
+ private ActorRef jobManagerMock;
+ private ActorRef listenerActor;
+
+ private BlockingQueue<Object> taskManagerMessages;
+ private BlockingQueue<Object> jobManagerMessages;
+ private BlockingQueue<Object> listenerMessages;
+
+ // ------------------------------------------------------------------------
+ // Init & Shutdown
+ // ------------------------------------------------------------------------
+ @BeforeClass
+ public static void startActorSystem() {
+ actorSystem = AkkaUtils.createLocalActorSystem(new Configuration());
+ }
+
+ @AfterClass
+ public static void shutdown() {
+ actorSystem.shutdown();
+ actorSystem.awaitTermination();
+ }
+
+ @Before
+ public void createQueuesAndActors() {
+ taskManagerMessages = new LinkedBlockingQueue<Object>();
+ jobManagerMessages = new LinkedBlockingQueue<Object>();
+ listenerMessages = new LinkedBlockingQueue<Object>();
+ taskManagerMock = actorSystem.actorOf(Props.create(ForwardingActor.class, taskManagerMessages));
+ jobManagerMock = actorSystem.actorOf(Props.create(ForwardingActor.class, jobManagerMessages));
+ listenerActor = actorSystem.actorOf(Props.create(ForwardingActor.class, listenerMessages));
+
+ awaitLatch = new OneShotLatch();
+ triggerLatch = new OneShotLatch();
+ }
+
+ @After
+ public void clearActorsAndMessages() {
+ jobManagerMessages = null;
+ taskManagerMessages = null;
+ listenerMessages = null;
+ taskManagerMock.tell(Kill.getInstance(), ActorRef.noSender());
+ jobManagerMock.tell(Kill.getInstance(), ActorRef.noSender());
+ listenerActor.tell(Kill.getInstance(), ActorRef.noSender());
+ }
+
+ // ------------------------------------------------------------------------
+ // Tests
+ // ------------------------------------------------------------------------
+
@Test
- public void testTaskStates() {
+ public void testRegularExecution() {
try {
- final JobID jid = new JobID();
- final JobVertexID vid = new JobVertexID();
- final ExecutionAttemptID eid = new ExecutionAttemptID();
+ Task task = createTask(TestInvokableCorrect.class);
- final RuntimeEnvironment env = mock(RuntimeEnvironment.class);
+ // task should be new and perfect
+ assertEquals(ExecutionState.CREATED, task.getExecutionState());
+ assertFalse(task.isCanceledOrFailed());
+ assertNull(task.getFailureCause());
- Task task = spy(new Task(jid, vid, 2, 7, eid, "TestTask", ActorRef.noSender()));
- doNothing().when(task).unregisterTask();
- task.setEnvironment(env);
+ task.registerExecutionListener(listenerActor);
- assertEquals(ExecutionState.DEPLOYING, task.getExecutionState());
+ // go into the run method. we should switch to DEPLOYING, RUNNING, then
+ // FINISHED, and all should be good
+ task.run();
- // cancel
- task.cancelExecution();
- assertEquals(ExecutionState.CANCELED, task.getExecutionState());
+ // verify final state
+ assertEquals(ExecutionState.FINISHED, task.getExecutionState());
+ assertFalse(task.isCanceledOrFailed());
+ assertNull(task.getFailureCause());
- // cannot go into running or finished state
+ // verify listener messages
+ validateListenerMessage(ExecutionState.RUNNING, task, false);
+ validateListenerMessage(ExecutionState.FINISHED, task, false);
- assertFalse(task.startExecution());
- assertEquals(ExecutionState.CANCELED, task.getExecutionState());
+ // make sure that the TaskManager received an message to unregister the task
+ validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+ validateUnregisterTask(task.getExecutionId());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCancelRightAway() {
+ try {
+ Task task = createTask(TestInvokableCorrect.class);
+ task.cancelExecution();
+
+ assertEquals(ExecutionState.CANCELING, task.getExecutionState());
- assertFalse(task.markAsFinished());
+ task.run();
+
+ // verify final state
assertEquals(ExecutionState.CANCELED, task.getExecutionState());
+ validateUnregisterTask(task.getExecutionId());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testFailExternallyRightAway() {
+ try {
+ Task task = createTask(TestInvokableCorrect.class);
+ task.failExternally(new Exception("fail externally"));
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+
+ task.run();
+
+ // verify final state
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ validateUnregisterTask(task.getExecutionId());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testLibraryCacheRegistrationFailed() {
+ try {
+ Task task = createTask(TestInvokableCorrect.class, mock(LibraryCacheManager.class));
+
+ // task should be new and perfect
+ assertEquals(ExecutionState.CREATED, task.getExecutionState());
+ assertFalse(task.isCanceledOrFailed());
+ assertNull(task.getFailureCause());
+
+ task.registerExecutionListener(listenerActor);
+
+ // should fail
+ task.run();
+
+ // verify final state
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertNotNull(task.getFailureCause());
+ assertTrue(task.getFailureCause().getMessage().contains("classloader"));
+
+ // verify listener messages
+ validateListenerMessage(ExecutionState.FAILED, task, true);
+
+ // make sure that the TaskManager received an message to unregister the task
+ validateUnregisterTask(task.getExecutionId());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExecutionFailsInNetworkRegistration() {
+ try {
+ // mock a working library cache
+ LibraryCacheManager libCache = mock(LibraryCacheManager.class);
+ when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader());
+
+ // mock a network manager that rejects registration
+ ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
+ ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
+ NetworkEnvironment network = mock(NetworkEnvironment.class);
+ when(network.getPartitionManager()).thenReturn(partitionManager);
+ when(network.getPartitionConsumableNotifier()).thenReturn(consumableNotifier);
+ when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
+ doThrow(new RuntimeException("buffers")).when(network).registerTask(any(Task.class));
+
+ Task task = createTask(TestInvokableCorrect.class, libCache, network);
+
+ task.registerExecutionListener(listenerActor);
+
+ task.run();
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("buffers"));
- task.markFailed(new Exception("test"));
- assertTrue(ExecutionState.CANCELED == task.getExecutionState());
+ validateUnregisterTask(task.getExecutionId());
+ validateListenerMessage(ExecutionState.FAILED, task, true);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
- verify(task).unregisterTask();
+ @Test
+ public void testInvokableInstantiationFailed() {
+ try {
+ Task task = createTask(InvokableNonInstantiable.class);
+ task.registerExecutionListener(listenerActor);
+
+ task.run();
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("instantiate"));
+
+ validateUnregisterTask(task.getExecutionId());
+ validateListenerMessage(ExecutionState.FAILED, task, true);
}
catch (Exception e) {
e.printStackTrace();
@@ -93,48 +298,19 @@ public class TaskTest {
}
@Test
- public void testTaskStartFinish() {
+ public void testExecutionFailsInRegisterInputOutput() {
try {
- final JobID jid = new JobID();
- final JobVertexID vid = new JobVertexID();
- final ExecutionAttemptID eid = new ExecutionAttemptID();
-
- final Task task = spy(new Task(jid, vid, 2, 7, eid, "TestTask", ActorRef.noSender()));
- doNothing().when(task).unregisterTask();
-
- final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
-
- Thread operation = new Thread() {
- @Override
- public void run() {
- try {
- assertTrue(task.markAsFinished());
- }
- catch (Throwable t) {
- error.set(t);
- }
- }
- };
-
- final RuntimeEnvironment env = mock(RuntimeEnvironment.class);
- when(env.getExecutingThread()).thenReturn(operation);
-
- assertEquals(ExecutionState.DEPLOYING, task.getExecutionState());
-
- // start the execution
- task.setEnvironment(env);
- task.startExecution();
-
- // wait for the execution to be finished
- operation.join();
-
- if (error.get() != null) {
- ExceptionUtils.rethrow(error.get());
- }
-
- assertEquals(ExecutionState.FINISHED, task.getExecutionState());
+ Task task = createTask(InvokableWithExceptionInRegisterInOut.class);
+ task.registerExecutionListener(listenerActor);
- verify(task).unregisterTask();
+ task.run();
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("registerInputOutput"));
+
+ validateUnregisterTask(task.getExecutionId());
+ validateListenerMessage(ExecutionState.FAILED, task, true);
}
catch (Exception e) {
e.printStackTrace();
@@ -143,48 +319,22 @@ public class TaskTest {
}
@Test
- public void testTaskFailesInRunning() {
+ public void testExecutionFailsInInvoke() {
try {
- final JobID jid = new JobID();
- final JobVertexID vid = new JobVertexID();
- final ExecutionAttemptID eid = new ExecutionAttemptID();
-
- final Task task = spy(new Task(jid, vid, 2, 7, eid, "TestTask", ActorRef.noSender()));
- doNothing().when(task).unregisterTask();
-
- final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
-
- Thread operation = new Thread() {
- @Override
- public void run() {
- try {
- task.markFailed(new Exception("test exception message"));
- }
- catch (Throwable t) {
- error.set(t);
- }
- }
- };
-
- final RuntimeEnvironment env = mock(RuntimeEnvironment.class);
- when(env.getExecutingThread()).thenReturn(operation);
+ Task task = createTask(InvokableWithExceptionInInvoke.class);
+ task.registerExecutionListener(listenerActor);
- assertEquals(ExecutionState.DEPLOYING, task.getExecutionState());
-
- // start the execution
- task.setEnvironment(env);
- task.startExecution();
-
- // wait for the execution to be finished
- operation.join();
-
- if (error.get() != null) {
- ExceptionUtils.rethrow(error.get());
- }
-
- // make sure the final state is correct and the task manager knows the changes
+ task.run();
+
assertEquals(ExecutionState.FAILED, task.getExecutionState());
- verify(task).unregisterTask();
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("test"));
+
+ validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+ validateUnregisterTask(task.getExecutionId());
+
+ validateListenerMessage(ExecutionState.RUNNING, task, false);
+ validateListenerMessage(ExecutionState.FAILED, task, true);
}
catch (Exception e) {
e.printStackTrace();
@@ -193,59 +343,180 @@ public class TaskTest {
}
@Test
- public void testTaskCanceledInRunning() {
+ public void testCancelDuringRegisterInputOutput() {
try {
- final JobID jid = new JobID();
- final JobVertexID vid = new JobVertexID();
- final ExecutionAttemptID eid = new ExecutionAttemptID();
-
- final Task task = spy(new Task(jid, vid, 2, 7, eid, "TestTask", ActorRef.noSender()));
- doNothing().when(task).unregisterTask();
-
- final AtomicReference<Throwable> error = new AtomicReference<Throwable>();
-
- // latches to create a deterministic order of events
- final OneShotLatch toRunning = new OneShotLatch();
- final OneShotLatch afterCanceling = new OneShotLatch();
-
- Thread operation = new Thread() {
- @Override
- public void run() {
- try {
- toRunning.trigger();
- afterCanceling.await();
- assertFalse(task.markAsFinished());
- task.cancelingDone();
- }
- catch (Throwable t) {
- error.set(t);
- }
- }
- };
+ Task task = createTask(InvokableBlockingInRegisterInOut.class);
+ task.registerExecutionListener(listenerActor);
+
+ // run the task asynchronous
+ task.startTaskThread();
- final RuntimeEnvironment env = mock(RuntimeEnvironment.class);
- when(env.getExecutingThread()).thenReturn(operation);
+ // wait till the task is in regInOut
+ awaitLatch.await();
- assertEquals(ExecutionState.DEPLOYING, task.getExecutionState());
+ task.cancelExecution();
+ assertEquals(ExecutionState.CANCELING, task.getExecutionState());
+ triggerLatch.trigger();
- // start the execution
- task.setEnvironment(env);
- task.startExecution();
+ task.getExecutingThread().join();
+
+ assertEquals(ExecutionState.CANCELED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertNull(task.getFailureCause());
- toRunning.await();
+ validateUnregisterTask(task.getExecutionId());
+ validateListenerMessage(ExecutionState.CANCELING, task, false);
+ validateListenerMessage(ExecutionState.CANCELED, task, false);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testFailDuringRegisterInputOutput() {
+ try {
+ Task task = createTask(InvokableBlockingInRegisterInOut.class);
+ task.registerExecutionListener(listenerActor);
+
+ // run the task asynchronous
+ task.startTaskThread();
+
+ // wait till the task is in regInOut
+ awaitLatch.await();
+
+ task.failExternally(new Exception("test"));
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ triggerLatch.trigger();
+
+ task.getExecutingThread().join();
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("test"));
+
+ validateUnregisterTask(task.getExecutionId());
+ validateListenerMessage(ExecutionState.FAILED, task, true);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCancelDuringInvoke() {
+ try {
+ Task task = createTask(InvokableBlockingInInvoke.class);
+ task.registerExecutionListener(listenerActor);
+
+ // run the task asynchronous
+ task.startTaskThread();
+
+ // wait till the task is in invoke
+ awaitLatch.await();
+
task.cancelExecution();
- afterCanceling.trigger();
+ assertTrue(task.getExecutionState() == ExecutionState.CANCELING ||
+ task.getExecutionState() == ExecutionState.CANCELED);
+
+ task.getExecutingThread().join();
+
+ assertEquals(ExecutionState.CANCELED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertNull(task.getFailureCause());
+
+ validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+ validateUnregisterTask(task.getExecutionId());
- // wait for the execution to be finished
- operation.join();
+ validateListenerMessage(ExecutionState.RUNNING, task, false);
+ validateListenerMessage(ExecutionState.CANCELING, task, false);
+ validateListenerMessage(ExecutionState.CANCELED, task, false);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testFailExternallyDuringInvoke() {
+ try {
+ Task task = createTask(InvokableBlockingInInvoke.class);
+ task.registerExecutionListener(listenerActor);
+
+ // run the task asynchronous
+ task.startTaskThread();
+
+ // wait till the task is in regInOut
+ awaitLatch.await();
+
+ task.failExternally(new Exception("test"));
+ assertTrue(task.getExecutionState() == ExecutionState.FAILED);
+
+ task.getExecutingThread().join();
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("test"));
+
+ validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+ validateUnregisterTask(task.getExecutionId());
+
+ validateListenerMessage(ExecutionState.RUNNING, task, false);
+ validateListenerMessage(ExecutionState.FAILED, task, true);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCanceledAfterExecutionFailedInRegInOut() {
+ try {
+ Task task = createTask(InvokableWithExceptionInRegisterInOut.class);
+ task.registerExecutionListener(listenerActor);
+
+ task.run();
- if (error.get() != null) {
- ExceptionUtils.rethrow(error.get());
- }
+ // this should not overwrite the failure state
+ task.cancelExecution();
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("registerInputOutput"));
+
+ validateUnregisterTask(task.getExecutionId());
+ validateListenerMessage(ExecutionState.FAILED, task, true);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCanceledAfterExecutionFailedInInvoke() {
+ try {
+ Task task = createTask(InvokableWithExceptionInInvoke.class);
+ task.registerExecutionListener(listenerActor);
+
+ task.run();
+
+ // this should not overwrite the failure state
+ task.cancelExecution();
+
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("test"));
+
+ validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+ validateUnregisterTask(task.getExecutionId());
- // make sure the final state is correct and the task manager knows the changes
- assertEquals(ExecutionState.CANCELED, task.getExecutionState());
- verify(task).unregisterTask();
+ validateListenerMessage(ExecutionState.RUNNING, task, false);
+ validateListenerMessage(ExecutionState.FAILED, task, true);
}
catch (Exception e) {
e.printStackTrace();
@@ -254,80 +525,206 @@ public class TaskTest {
}
@Test
- public void testTaskWithEnvironment() {
+ public void testExecutionFailesAfterCanceling() {
try {
- final JobID jid = new JobID();
- final JobVertexID vid = new JobVertexID();
- final ExecutionAttemptID eid = new ExecutionAttemptID();
-
- TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, vid, eid, "TestTask", 2, 7,
- new Configuration(), new Configuration(), TestInvokableCorrect.class.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- new ArrayList<BlobKey>(), 0);
-
- Task task = spy(new Task(jid, vid, 2, 7, eid, "TestTask", ActorRef.noSender()));
- doNothing().when(task).unregisterTask();
+ Task task = createTask(InvokableWithExceptionOnTrigger.class);
+ task.registerExecutionListener(listenerActor);
+
+ // run the task asynchronous
+ task.startTaskThread();
+
+ // wait till the task is in invoke
+ awaitLatch.await();
+
+ task.cancelExecution();
+ assertEquals(ExecutionState.CANCELING, task.getExecutionState());
- RuntimeEnvironment env = new RuntimeEnvironment(mock(ActorRef.class), task, tdd, getClass().getClassLoader(),
- mock(MemoryManager.class), mock(IOManager.class), mock(InputSplitProvider.class),
- new BroadcastVariableManager(), MockNetworkEnvironment.getMock());
+ // this causes an exception
+ triggerLatch.trigger();
+
+ task.getExecutingThread().join();
- task.setEnvironment(env);
+ // we should still be in state canceled
+ assertEquals(ExecutionState.CANCELED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertNull(task.getFailureCause());
- assertEquals(ExecutionState.DEPLOYING, task.getExecutionState());
+ validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+ validateUnregisterTask(task.getExecutionId());
+
+ validateListenerMessage(ExecutionState.RUNNING, task, false);
+ validateListenerMessage(ExecutionState.CANCELING, task, false);
+ validateListenerMessage(ExecutionState.CANCELED, task, false);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testExecutionFailesAfterTaskMarkedFailed() {
+ try {
+ Task task = createTask(InvokableWithExceptionOnTrigger.class);
+ task.registerExecutionListener(listenerActor);
+
+ // run the task asynchronous
+ task.startTaskThread();
+
+ // wait till the task is in invoke
+ awaitLatch.await();
+
+ task.failExternally(new Exception("external"));
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+
+ // this causes an exception
+ triggerLatch.trigger();
+
+ task.getExecutingThread().join();
- task.startExecution();
- task.getEnvironment().getExecutingThread().join();
+ assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ assertTrue(task.isCanceledOrFailed());
+ assertTrue(task.getFailureCause().getMessage().contains("external"));
- assertEquals(ExecutionState.FINISHED, task.getExecutionState());
+ validateTaskManagerStateChange(ExecutionState.RUNNING, task, false);
+ validateUnregisterTask(task.getExecutionId());
- verify(task).unregisterTask();
+ validateListenerMessage(ExecutionState.RUNNING, task, false);
+ validateListenerMessage(ExecutionState.FAILED, task, true);
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
+
+ private Task createTask(Class<? extends AbstractInvokable> invokable) {
+ LibraryCacheManager libCache = mock(LibraryCacheManager.class);
+ when(libCache.getClassLoader(any(JobID.class))).thenReturn(getClass().getClassLoader());
+ return createTask(invokable, libCache);
+ }
+
+ private Task createTask(Class<? extends AbstractInvokable> invokable,
+ LibraryCacheManager libCache) {
+
+ ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
+ ResultPartitionConsumableNotifier consumableNotifier = mock(ResultPartitionConsumableNotifier.class);
+ NetworkEnvironment network = mock(NetworkEnvironment.class);
+ when(network.getPartitionManager()).thenReturn(partitionManager);
+ when(network.getPartitionConsumableNotifier()).thenReturn(consumableNotifier);
+ when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
+
+ return createTask(invokable, libCache, network);
+ }
- @Test
- public void testTaskWithEnvironmentAndException() {
+ private Task createTask(Class<? extends AbstractInvokable> invokable,
+ LibraryCacheManager libCache,
+ NetworkEnvironment networkEnvironment) {
+
+ TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(invokable);
+
+ return new Task(tdd,
+ mock(MemoryManager.class),
+ mock(IOManager.class),
+ networkEnvironment,
+ mock(BroadcastVariableManager.class),
+ taskManagerMock, jobManagerMock,
+ new FiniteDuration(60, TimeUnit.SECONDS),
+ libCache,
+ mock(FileCache.class));
+ }
+
+ private TaskDeploymentDescriptor createTaskDeploymentDescriptor(Class<? extends AbstractInvokable> invokable) {
+ return new TaskDeploymentDescriptor(
+ new JobID(), new JobVertexID(), new ExecutionAttemptID(),
+ "Test Task", 0, 1,
+ new Configuration(), new Configuration(),
+ invokable.getName(),
+ Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
+ Collections.<InputGateDeploymentDescriptor>emptyList(),
+ Collections.<BlobKey>emptyList(),
+ 0);
+ }
+
+ // ------------------------------------------------------------------------
+ // Validation Methods
+ // ------------------------------------------------------------------------
+
+ private void validateUnregisterTask(ExecutionAttemptID id) {
try {
- final JobID jid = new JobID();
- final JobVertexID vid = new JobVertexID();
- final ExecutionAttemptID eid = new ExecutionAttemptID();
+ // we may have to wait for a bit to give the actors time to receive the message
+ // and put it into the queue
+ Object rawMessage = taskManagerMessages.poll(10, TimeUnit.SECONDS);
- TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(jid, vid, eid, "TestTask", 2, 7,
- new Configuration(), new Configuration(), TestInvokableWithException.class.getName(),
- Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
- Collections.<InputGateDeploymentDescriptor>emptyList(),
- new ArrayList<BlobKey>(), 0);
+ assertNotNull("There is no additional TaskManager message", rawMessage);
+ assertTrue("TaskManager message is not 'UnregisterTask'", rawMessage instanceof TaskMessages.TaskInFinalState);
- Task task = spy(new Task(jid, vid, 2, 7, eid, "TestTask", ActorRef.noSender()));
- doNothing().when(task).unregisterTask();
-
- RuntimeEnvironment env = new RuntimeEnvironment(mock(ActorRef.class), task, tdd, getClass().getClassLoader(),
- mock(MemoryManager.class), mock(IOManager.class), mock(InputSplitProvider.class),
- new BroadcastVariableManager(), MockNetworkEnvironment.getMock());
+ TaskMessages.TaskInFinalState message = (TaskMessages.TaskInFinalState) rawMessage;
+ assertEquals(id, message.executionID());
+ }
+ catch (InterruptedException e) {
+ fail("interrupted");
+ }
+ }
- task.setEnvironment(env);
+ private void validateTaskManagerStateChange(ExecutionState state, Task task, boolean hasError) {
+ try {
+ // we may have to wait for a bit to give the actors time to receive the message
+ // and put it into the queue
+ Object rawMessage = taskManagerMessages.poll(10, TimeUnit.SECONDS);
+
+ assertNotNull("There is no additional TaskManager message", rawMessage);
+ assertTrue("TaskManager message is not 'UpdateTaskExecutionState'",
+ rawMessage instanceof TaskMessages.UpdateTaskExecutionState);
- assertEquals(ExecutionState.DEPLOYING, task.getExecutionState());
+ TaskMessages.UpdateTaskExecutionState message =
+ (TaskMessages.UpdateTaskExecutionState) rawMessage;
- task.startExecution();
- task.getEnvironment().getExecutingThread().join();
+ TaskExecutionState taskState = message.taskExecutionState();
+
+ assertEquals(task.getJobID(), taskState.getJobID());
+ assertEquals(task.getExecutionId(), taskState.getID());
+ assertEquals(state, taskState.getExecutionState());
+
+ if (hasError) {
+ assertNotNull(taskState.getError(getClass().getClassLoader()));
+ } else {
+ assertNull(taskState.getError(getClass().getClassLoader()));
+ }
+ }
+ catch (InterruptedException e) {
+ fail("interrupted");
+ }
+ }
+
+ private void validateListenerMessage(ExecutionState state, Task task, boolean hasError) {
+ try {
+ // we may have to wait for a bit to give the actors time to receive the message
+ // and put it into the queue
+ TaskMessages.UpdateTaskExecutionState message =
+ (TaskMessages.UpdateTaskExecutionState) listenerMessages.poll(10, TimeUnit.SECONDS);
+ assertNotNull("There is no additional listener message", message);
- assertEquals(ExecutionState.FAILED, task.getExecutionState());
+ TaskExecutionState taskState = message.taskExecutionState();
- verify(task).unregisterTask();
+ assertEquals(task.getJobID(), taskState.getJobID());
+ assertEquals(task.getExecutionId(), taskState.getID());
+ assertEquals(state, taskState.getExecutionState());
+
+ if (hasError) {
+ assertNotNull(taskState.getError(getClass().getClassLoader()));
+ } else {
+ assertNull(taskState.getError(getClass().getClassLoader()));
+ }
}
- catch (Exception e) {
- e.printStackTrace();
- fail(e.getMessage());
+ catch (InterruptedException e) {
+ fail("interrupted");
}
}
// --------------------------------------------------------------------------------------------
+ // Mock invokable code
+ // --------------------------------------------------------------------------------------------
public static final class TestInvokableCorrect extends AbstractInvokable {
@@ -336,16 +733,93 @@ public class TaskTest {
@Override
public void invoke() {}
+
+ @Override
+ public void cancel() throws Exception {
+ fail("This should not be called");
+ }
+ }
+
+ public static final class InvokableWithExceptionInRegisterInOut extends AbstractInvokable {
+
+ @Override
+ public void registerInputOutput() {
+ throw new RuntimeException("test");
+ }
+
+ @Override
+ public void invoke() {}
}
- public static final class TestInvokableWithException extends AbstractInvokable {
+ public static final class InvokableWithExceptionInInvoke extends AbstractInvokable {
+
+ @Override
+ public void registerInputOutput() {}
+
+ @Override
+ public void invoke() throws Exception {
+ throw new Exception("test");
+ }
+ }
+
+ public static final class InvokableWithExceptionOnTrigger extends AbstractInvokable {
+
+ @Override
+ public void registerInputOutput() {}
+
+ @Override
+ public void invoke() {
+ awaitLatch.trigger();
+
+ // make sure that the interrupt call does not
+ // grab us out of the lock early
+ while (true) {
+ try {
+ triggerLatch.await();
+ break;
+ }
+ catch (InterruptedException e) {
+ // fall through the loop
+ }
+ }
+
+ throw new RuntimeException("test");
+ }
+ }
+
+ public static abstract class InvokableNonInstantiable extends AbstractInvokable {}
+
+ public static final class InvokableBlockingInRegisterInOut extends AbstractInvokable {
+
+ @Override
+ public void registerInputOutput() {
+ awaitLatch.trigger();
+
+ try {
+ triggerLatch.await();
+ }
+ catch (InterruptedException e) {
+ throw new RuntimeException();
+ }
+ }
+
+ @Override
+ public void invoke() {}
+ }
+
+ public static final class InvokableBlockingInInvoke extends AbstractInvokable {
@Override
public void registerInputOutput() {}
@Override
public void invoke() throws Exception {
- throw new Exception("test exception");
+ awaitLatch.trigger();
+
+ // block forever
+ synchronized (this) {
+ wait();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/8e613014/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
index bb0c1f9..5318254 100644
--- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
+++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingTaskManager.scala
@@ -27,7 +27,7 @@ import org.apache.flink.runtime.io.disk.iomanager.IOManager
import org.apache.flink.runtime.io.network.NetworkEnvironment
import org.apache.flink.runtime.memorymanager.DefaultMemoryManager
import org.apache.flink.runtime.messages.Messages.Disconnect
-import org.apache.flink.runtime.messages.TaskMessages.{UpdateTaskExecutionState, UnregisterTask}
+import org.apache.flink.runtime.messages.TaskMessages.{TaskInFinalState, UpdateTaskExecutionState}
import org.apache.flink.runtime.taskmanager.{TaskManagerConfiguration, TaskManager}
import org.apache.flink.runtime.testingUtils.TestingJobManagerMessages.NotifyWhenJobRemoved
import org.apache.flink.runtime.testingUtils.TestingMessages.DisableDisconnect
@@ -95,8 +95,8 @@ class TestingTaskManager(config: TaskManagerConfiguration,
}
}
- case UnregisterTask(executionID) =>
- super.receiveWithLogMessages(UnregisterTask(executionID))
+ case TaskInFinalState(executionID) =>
+ super.receiveWithLogMessages(TaskInFinalState(executionID))
waitForRemoval.remove(executionID) match {
case Some(actors) => for(actor <- actors) actor ! true
case None =>
[2/8] flink git commit: [streaming] New Source and state
checkpointing interfaces that allow operations to interact with the state
checkpointing in a more precise manner.
Posted by se...@apache.org.
[streaming] New Source and state checkpointing interfaces that allow operations to interact with the state checkpointing in a more precise manner.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/acca10ea
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/acca10ea
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/acca10ea
Branch: refs/heads/master
Commit: acca10ea0a94cac56fe53258fa145c777ad4ea67
Parents: d259e69
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Apr 30 22:05:27 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon May 11 21:07:57 2015 +0200
----------------------------------------------------------------------
.../flink/api/common/functions/Function.java | 7 +-
.../api/checkpoint/CheckpointCommitter.java | 37 +++++++++
.../streaming/api/checkpoint/Checkpointed.java | 53 +++++++++++++
.../checkpoint/CheckpointedAsynchronously.java | 35 +++++++++
.../api/functions/source/StreamSource.java | 80 ++++++++++++++++++++
5 files changed, 209 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/acca10ea/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
index 64ebac1..632a0d2 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/Function.java
@@ -19,9 +19,10 @@
package org.apache.flink.api.common.functions;
/**
- * An base interface for all user-defined functions. This interface is empty in order
- * to enable functions that are SAM (single abstract method) interfaces, so that they
- * can be called as Java 8 lambdas
+ * The base interface for all user-defined functions.
+ *
+ * <p>This interface is empty in order to allow extending interfaces to
+ * be SAM (single abstract method) interfaces that can be implemented via Java 8 lambdas.</p>
*/
public interface Function {
}
http://git-wip-us.apache.org/repos/asf/flink/blob/acca10ea/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointCommitter.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointCommitter.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointCommitter.java
new file mode 100644
index 0000000..a95b540
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointCommitter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.checkpoint;
+
+/**
+ * This interface must be implemented by functions/operations that want to receive
+ * a commit notification once a checkpoint has been completely acknowledged by all
+ * participants.
+ */
+public interface CheckpointCommitter {
+
+ /**
+ * This method is called as a notification once a distributed checkpoint has been completed.
+ *
+ * Note that any exception during this method will not cause the checkpoint to
+ * fail any more.
+ *
+ * @param checkpointId The ID of the checkpoint that has been completed.
+ */
+ void commitCheckpoint(long checkpointId);
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/acca10ea/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
new file mode 100644
index 0000000..f491dd3
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/Checkpointed.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.checkpoint;
+
+import org.apache.flink.runtime.state.OperatorState;
+
+/**
+ * This method must be implemented by functions that have state that needs to be
+ * checkpointed. The functions get a call whenever a checkpoint should take place
+ * and return a snapshot of their state, which will be checkpointed.
+ *
+ * <p>This interface marks a function as <i>synchronously</i> checkpointed. While the
+ * state is written, the function is not called, so the function needs not return a
+ * copy of its state, but may return a reference to its state. Functions that can
+ * continue to work and mutate the state, even while the state snapshot is being accessed,
+ * can implement the {@link org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously}
+ * interface.</p>
+ */
+public interface Checkpointed {
+
+ /**
+ * Gets the current operator state as a checkpoint. The state must reflect all operations
+ * from all prior operations if this function.
+ *
+ * @param checkpointId The ID of the checkpoint.
+ * @param checkpointTimestamp The timestamp of the checkpoint, as derived by
+ * System.currentTimeMillis() on the JobManager.
+ *
+ * @return A snapshot of the operator state.
+ *
+ * @throws Exception Thrown if the creation of the state object failed. This causes the
+ * checkpoint to fail. The system may decide to fail the operation (and trigger
+ * recovery), or to discard this checkpoint attempt and to continue running
+ * and to try again with the next checkpoint attempt.
+ */
+ OperatorState<?> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/acca10ea/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
new file mode 100644
index 0000000..196f7ec
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/checkpoint/CheckpointedAsynchronously.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.checkpoint;
+
+/**
+ * This interface marks a function/operator as <i>asynchronously checkpointed</i>.
+ * Similar to the {@link Checkpointed} interface, the function must produce a
+ * snapshot of its state. However, the function must be able to continue working
+ * and mutating its state without mutating the returned state snapshot.
+ *
+ * <p>Asynchronous checkpoints are desirable, because they allow the data streams at the
+ * point of the checkpointed function/operator to continue running while the checkpoint
+ * is in progress.</p>
+ *
+ * <p>To be able to support asynchronous snapshots, the state returned by the
+ * {@link #snapshotState(long, long)} method is typically a copy or shadow copy
+ * of the actual state.</p>
+ */
+public interface CheckpointedAsynchronously extends Checkpointed {}
http://git-wip-us.apache.org/repos/asf/flink/blob/acca10ea/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/StreamSource.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/StreamSource.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/StreamSource.java
new file mode 100644
index 0000000..9452930
--- /dev/null
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/api/functions/source/StreamSource.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.functions.source;
+
+/**
+ * Base interface for all stream data sources in Flink. The contract of a stream source
+ * is similar to an iterator - it is consumed as in the following pseudo code:
+ *
+ * <pre>{@code
+ * StreamSource<T> source = ...;
+ * Collector<T> out = ...;
+ * while (!source.reachedEnd()) {
+ * out.collect(source.next());
+ * }
+ * }
+ * </pre>
+ *
+ * <b>Note about blocking behavior</b>
+ * <p>This implementations of the methods in the stream sources must have certain guarantees about
+ * blocking behavior. One of the two characteristics must be fulfilled.</p>
+ * <ul>
+ * <li>The methods must react to thread interrupt calls and break out of blocking calls with
+ * an {@link InterruptedException}.</li>
+ * <li>The method may ignore interrupt calls and/or swallow InterruptedExceptions, if it is guaranteed
+ * that the method returns quasi immediately irrespectively of the input. This is true for example
+ * for file streams, where the call is guaranteed to return after a very short I/O delay in
+ * the order of milliseconds.</li>
+ * </ul>
+ *
+ * @param <T> The type of the records produced by this source.
+ */
+public interface StreamSource<T> {
+
+ /**
+ * Checks whether the stream has reached its end.
+ *
+ * <p>This method must obey the contract about blocking behavior declared in the
+ * description of this class.</p>
+ *
+ * @return True, if the end of the stream has been reached, false if more data is available.
+ *
+ * @throws InterruptedException The calling thread may be interrupted to pull the function out of this
+ * method during checkpoints.
+ * @throws Exception Any other exception that is thrown causes the source to fail and results in failure of
+ * the streaming program, or triggers recovery, depending on the program setup.
+ */
+ boolean reachedEnd() throws Exception;
+
+
+ /**
+ * Produces the next record.
+ *
+ * <p>This method must obey the contract about blocking behavior declared in the
+ * description of this class.</p>
+ *
+ * @return The next record produced by this stream source.
+ *
+ * @throws InterruptedException The calling thread may be interrupted to pull the function out of this
+ * method during checkpoints.
+ * @throws Exception Any other exception that is thrown causes the source to fail and results in failure of
+ * the streaming program, or triggers recovery, depending on the program setup.
+ */
+ T next() throws Exception;
+}
[3/8] flink git commit: [FLINK-1968] [runtime] Clean up and improve
the distributed cache.
Posted by se...@apache.org.
[FLINK-1968] [runtime] Clean up and improve the distributed cache.
- Gives a proper exception when a non-cached file is accessed
- Forwards I/O exceptions that happen during file transfer, rather than only returning null when transfer failed
- Consistently keeps reference counts and copies only when needed
- Properly removes all files when shutdown
- Uses a shutdown hook to remove files when process is killed
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1c8d866a
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1c8d866a
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1c8d866a
Branch: refs/heads/master
Commit: 1c8d866a83065e3d1bc9707dab81117f24c9f678
Parents: acca10e
Author: Stephan Ewen <se...@apache.org>
Authored: Sun May 3 01:15:39 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon May 11 21:07:57 2015 +0200
----------------------------------------------------------------------
.../api/common/cache/DistributedCache.java | 78 ++--
.../util/AbstractRuntimeUDFContext.java | 27 +-
.../functions/util/RuntimeUDFContext.java | 4 +-
.../org/apache/flink/core/fs/FileSystem.java | 18 +-
.../flink/runtime/execution/Environment.java | 45 ++-
.../flink/runtime/filecache/FileCache.java | 358 ++++++++++++++-----
.../operators/chaining/ChainedDriver.java | 3 +-
.../util/DistributedRuntimeUDFContext.java | 4 +-
.../FileCacheDeleteValidationTest.java | 126 ++++---
.../operators/testutils/MockEnvironment.java | 20 +-
.../runtime/tasks/StreamingRuntimeContext.java | 2 +-
11 files changed, 461 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-core/src/main/java/org/apache/flink/api/common/cache/DistributedCache.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/cache/DistributedCache.java b/flink-core/src/main/java/org/apache/flink/api/common/cache/DistributedCache.java
index af8c716..a4261d0 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/cache/DistributedCache.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/cache/DistributedCache.java
@@ -21,11 +21,13 @@ package org.apache.flink.api.common.cache;
import java.io.File;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import java.util.concurrent.FutureTask;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
@@ -37,6 +39,7 @@ import org.apache.flink.core.fs.Path;
public class DistributedCache {
public static class DistributedCacheEntry {
+
public String filePath;
public Boolean isExecutable;
@@ -45,19 +48,45 @@ public class DistributedCache {
this.isExecutable=isExecutable;
}
}
-
- final static String CACHE_FILE_NUM = "DISTRIBUTED_CACHE_FILE_NUM";
-
- final static String CACHE_FILE_NAME = "DISTRIBUTED_CACHE_FILE_NAME_";
-
- final static String CACHE_FILE_PATH = "DISTRIBUTED_CACHE_FILE_PATH_";
- final static String CACHE_FILE_EXE = "DISTRIBUTED_CACHE_FILE_EXE_";
+ // ------------------------------------------------------------------------
- public final static String TMP_PREFIX = "tmp_";
+ private final Map<String, Future<Path>> cacheCopyTasks;
- private Map<String, FutureTask<Path>> cacheCopyTasks = new HashMap<String, FutureTask<Path>>();
+ public DistributedCache(Map<String, Future<Path>> cacheCopyTasks) {
+ this.cacheCopyTasks = cacheCopyTasks;
+ }
+
+ // ------------------------------------------------------------------------
+ public File getFile(String name) {
+ if (name == null) {
+ throw new NullPointerException("name must not be null");
+ }
+
+ Future<Path> future = cacheCopyTasks.get(name);
+ if (future == null) {
+ throw new IllegalArgumentException("File with name '" + name + "' is not available." +
+ " Did you forget to register the file?");
+ }
+
+ try {
+ Path tmp = future.get();
+ return new File(tmp.toString());
+ }
+ catch (ExecutionException e) {
+ throw new RuntimeException("An error occurred while copying the file.", e.getCause());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Error while getting the file registered under '" + name +
+ "' from the distributed cache", e);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Utilities to read/write cache files from/to the configuration
+ // ------------------------------------------------------------------------
+
public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
int num = conf.getInteger(CACHE_FILE_NUM,0) + 1;
conf.setInteger(CACHE_FILE_NUM, num);
@@ -67,29 +96,26 @@ public class DistributedCache {
}
public static Set<Entry<String, DistributedCacheEntry>> readFileInfoFromConfig(Configuration conf) {
- Map<String, DistributedCacheEntry> cacheFiles = new HashMap<String, DistributedCacheEntry>();
int num = conf.getInteger(CACHE_FILE_NUM, 0);
+ if (num == 0) {
+ return Collections.emptySet();
+ }
+
+ Map<String, DistributedCacheEntry> cacheFiles = new HashMap<String, DistributedCacheEntry>();
for (int i = 1; i <= num; i++) {
- String name = conf.getString(CACHE_FILE_NAME + i, "");
- String filePath = conf.getString(CACHE_FILE_PATH + i, "");
+ String name = conf.getString(CACHE_FILE_NAME + i, null);
+ String filePath = conf.getString(CACHE_FILE_PATH + i, null);
Boolean isExecutable = conf.getBoolean(CACHE_FILE_EXE + i, false);
cacheFiles.put(name, new DistributedCacheEntry(filePath, isExecutable));
}
return cacheFiles.entrySet();
}
- public void setCopyTasks(Map<String, FutureTask<Path>> cpTasks) {
- this.cacheCopyTasks = cpTasks;
- }
+ private static final String CACHE_FILE_NUM = "DISTRIBUTED_CACHE_FILE_NUM";
- public File getFile(String name) {
- Path tmp = null;
- //The FutureTask.get() method will block until the file is ready.
- try {
- tmp = cacheCopyTasks.get(name).get();
- } catch (Exception e) {
- throw new RuntimeException("Error while getting file from distributed cache", e);
- }
- return new File(tmp.toString());
- }
+ private static final String CACHE_FILE_NAME = "DISTRIBUTED_CACHE_FILE_NAME_";
+
+ private static final String CACHE_FILE_PATH = "DISTRIBUTED_CACHE_FILE_PATH_";
+
+ private static final String CACHE_FILE_EXE = "DISTRIBUTED_CACHE_FILE_EXE_";
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
index ff531c7..735fe8e 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/AbstractRuntimeUDFContext.java
@@ -19,9 +19,10 @@
package org.apache.flink.api.common.functions.util;
import java.io.Serializable;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.FutureTask;
+import java.util.concurrent.Future;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.accumulators.Accumulator;
@@ -51,20 +52,30 @@ public abstract class AbstractRuntimeUDFContext implements RuntimeContext {
private final HashMap<String, Accumulator<?, ?>> accumulators = new HashMap<String, Accumulator<?, ?>>();
- private final DistributedCache distributedCache = new DistributedCache();
+ private final DistributedCache distributedCache;
- public AbstractRuntimeUDFContext(String name, int numParallelSubtasks, int subtaskIndex, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig) {
+ public AbstractRuntimeUDFContext(String name,
+ int numParallelSubtasks, int subtaskIndex,
+ ClassLoader userCodeClassLoader,
+ ExecutionConfig executionConfig)
+ {
+ this(name, numParallelSubtasks, subtaskIndex, userCodeClassLoader, executionConfig,
+ Collections.<String, Future<Path>>emptyMap());
+ }
+
+ public AbstractRuntimeUDFContext(String name,
+ int numParallelSubtasks, int subtaskIndex,
+ ClassLoader userCodeClassLoader,
+ ExecutionConfig executionConfig,
+ Map<String, Future<Path>> cpTasks)
+ {
this.name = name;
this.numParallelSubtasks = numParallelSubtasks;
this.subtaskIndex = subtaskIndex;
this.userCodeClassLoader = userCodeClassLoader;
this.executionConfig = executionConfig;
- }
-
- public AbstractRuntimeUDFContext(String name, int numParallelSubtasks, int subtaskIndex, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, Map<String, FutureTask<Path>> cpTasks) {
- this(name, numParallelSubtasks, subtaskIndex, userCodeClassLoader, executionConfig);
- this.distributedCache.setCopyTasks(cpTasks);
+ this.distributedCache = new DistributedCache(cpTasks);
}
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
index b9c98cd..d116d00 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/functions/util/RuntimeUDFContext.java
@@ -21,7 +21,7 @@ package org.apache.flink.api.common.functions.util;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.FutureTask;
+import java.util.concurrent.Future;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
@@ -42,7 +42,7 @@ public class RuntimeUDFContext extends AbstractRuntimeUDFContext {
super(name, numParallelSubtasks, subtaskIndex, userCodeClassLoader, executionConfig);
}
- public RuntimeUDFContext(String name, int numParallelSubtasks, int subtaskIndex, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, Map<String, FutureTask<Path>> cpTasks) {
+ public RuntimeUDFContext(String name, int numParallelSubtasks, int subtaskIndex, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, Map<String, Future<Path>> cpTasks) {
super(name, numParallelSubtasks, subtaskIndex, userCodeClassLoader, executionConfig, cpTasks);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java b/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
index 0e38d8a..8b4cdba 100644
--- a/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
+++ b/flink-core/src/main/java/org/apache/flink/core/fs/FileSystem.java
@@ -168,17 +168,15 @@ public abstract class FileSystem {
* @throws IOException
* thrown if a reference to the file system instance could not be obtained
*/
- public static FileSystem getLocalFileSystem() throws IOException {
-
- URI localUri;
-
+ public static FileSystem getLocalFileSystem() {
+ // this should really never fail.
try {
- localUri = OperatingSystem.isWindows() ? new URI("file:/") : new URI("file:///");
- } catch (URISyntaxException e) {
- throw new IOException("Cannot create URI for local file system");
+ URI localUri = OperatingSystem.isWindows() ? new URI("file:/") : new URI("file:///");
+ return get(localUri);
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Cannot create URI for local file system");
}
-
- return get(localUri);
}
/**
@@ -193,7 +191,7 @@ public abstract class FileSystem {
* thrown if a reference to the file system instance could not be obtained
*/
public static FileSystem get(URI uri) throws IOException {
- FileSystem fs = null;
+ FileSystem fs;
synchronized (SYNCHRONIZATION_OBJECT) {
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
index 1bd4f7b..7ab3bc9 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/execution/Environment.java
@@ -18,58 +18,62 @@
package org.apache.flink.runtime.execution;
-import akka.actor.ActorRef;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter;
import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
-import org.apache.flink.runtime.jobgraph.JobGraph;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.jobgraph.tasks.InputSplitProvider;
import org.apache.flink.runtime.memorymanager.MemoryManager;
import java.util.Map;
-import java.util.concurrent.FutureTask;
+import java.util.concurrent.Future;
/**
- * The user code of every task runs inside an <code>Environment</code> object.
- * The environment provides important services to the task. It keeps track of
- * setting up the communication channels and provides access to input splits,
- * memory manager, etc.
+ * The Environment gives the code executed in a task access to the task's properties
+ * (such as name, parallelism), the configurations, the data stream readers and writers,
+ * as well as the various components that are provided by the TaskManager, such as
+ * memory manager, I/O manager, ...
*/
public interface Environment {
/**
- * Returns the ID of the job from the original job graph. It is used by the library cache manager to find the
- * required
- * libraries for executing the assigned Nephele task.
+ * Returns the ID of the job that the task belongs to.
*
* @return the ID of the job from the original job graph
*/
JobID getJobID();
/**
- * Gets the ID of the jobVertex that this task corresponds to.
+ * Gets the ID of the JobVertex for which this task executes a parallel subtask.
*
* @return The JobVertexID of this task.
*/
JobVertexID getJobVertexId();
/**
- * Returns the task configuration object which was attached to the original JobVertex.
+ * Gets the ID of the task execution attempt.
*
- * @return the task configuration object which was attached to the original JobVertex.
+ * @return The ID of the task execution attempt.
+ */
+ ExecutionAttemptID getExecutionId();
+
+ /**
+ * Returns the task-wide configuration object, originally attache to the job vertex.
+ *
+ * @return The task-wide configuration
*/
Configuration getTaskConfiguration();
/**
- * Returns the job configuration object which was attached to the original {@link JobGraph}.
+ * Returns the job-wide configuration object that was attached to the JobGraph.
*
- * @return the job configuration object which was attached to the original {@link JobGraph}
+ * @return The job-wide configuration
*/
Configuration getJobConfiguration();
@@ -132,7 +136,7 @@ public interface Environment {
*/
ClassLoader getUserClassLoader();
- Map<String, FutureTask<Path>> getCopyTask();
+ Map<String, Future<Path>> getDistributedCacheEntries();
BroadcastVariableManager getBroadcastVariableManager();
@@ -155,13 +159,4 @@ public interface Environment {
InputGate[] getAllInputGates();
-
- /**
- * Returns the proxy object for the accumulator protocol.
- */
- // THIS DOES NOT BELONG HERE, THIS TOTALLY BREAKS COMPONENTIZATION.
- // THE EXECUTED TASKS HAVE BEEN KEPT INDEPENDENT OF ANY RPC OR ACTOR
- // COMMUNICATION !!!
- ActorRef getJobManager();
-
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java b/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
index 3449100..fa5ad85 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/filecache/FileCache.java
@@ -22,99 +22,219 @@ import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
-import org.apache.flink.api.common.cache.DistributedCache;
+import org.apache.commons.io.FileUtils;
import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
+import org.apache.flink.api.java.tuple.Tuple4;
import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.fs.FSDataOutputStream;
import org.apache.flink.core.fs.FileStatus;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.core.fs.local.LocalFileSystem;
import org.apache.flink.api.common.JobID;
import org.apache.flink.runtime.util.ExecutorThreadFactory;
import org.apache.flink.runtime.util.IOUtils;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * The FileCache is used to create the local files for the registered cache files when a task is deployed.
+ * The FileCache is used to create the local files for the registered cache files when a task is deployed.
* The files will be removed when the task is unregistered after a 5 second delay.
* A given file x will be placed in "<system-tmp-dir>/tmp_<jobID>/".
*/
public class FileCache {
- private static final Logger LOG = LoggerFactory.getLogger(FileCache.class);
-
- private static final Object lock = new Object();
-
+ static final Logger LOG = LoggerFactory.getLogger(FileCache.class);
- private LocalFileSystem lfs = new LocalFileSystem();
+ /** cache-wide lock to ensure consistency. copies are not done under this lock */
+ private final Object lock = new Object();
+
+ private final Map<JobID, Map<String, Tuple4<Integer, File, Path, Future<Path>>>> entries;
+
+ private final ScheduledExecutorService executorService;
+
+ private final File[] storageDirectories;
- private Map<JobID, Map<String, Integer>> jobCounts = new HashMap<JobID, Map<String, Integer>>();
+ private final Thread shutdownHook;
- private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(10, ExecutorThreadFactory.INSTANCE);
+ private int nextDirectory;
+
+ // ------------------------------------------------------------------------
+
+ public FileCache(Configuration config) throws IOException {
+
+ String tempDirs = config.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
+ ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH);
+
+ String[] directories = tempDirs.split(",|" + File.pathSeparator);
+ String cacheDirName = "flink-dist-cache-" + UUID.randomUUID().toString();
+ storageDirectories = new File[directories.length];
+
+ for (int i = 0; i < directories.length; i++) {
+ storageDirectories[i] = new File(directories[i], cacheDirName);
+ String path = storageDirectories[i].getAbsolutePath();
+
+ if (storageDirectories[i].mkdirs()) {
+ LOG.info("User file cache uses directory " + path);
+ } else {
+ LOG.error("User file cache cannot create directory " + path);
+ // delete all other directories we created so far
+ for (int k = 0; k < i; k++) {
+ if (!storageDirectories[k].delete()) {
+ LOG.warn("User file cache cannot remove prior directory " +
+ storageDirectories[k].getAbsolutePath());
+ }
+ }
+ throw new IOException("File cache cannot create temp storage directory: " + path);
+ }
+ }
+
+ this.shutdownHook = createShutdownHook(this, LOG);
+
+ this.entries = new HashMap<JobID, Map<String, Tuple4<Integer, File, Path, Future<Path>>>>();
+ this.executorService = Executors.newScheduledThreadPool(10, ExecutorThreadFactory.INSTANCE);
+ }
+
+ /**
+ * Shuts down the file cache by cancelling all
+ */
+ public void shutdown() {
+ synchronized (lock) {
+ // first shutdown the thread pool
+ ScheduledExecutorService es = this.executorService;
+ if (es != null) {
+ es.shutdown();
+ try {
+ es.awaitTermination(5000L, TimeUnit.MILLISECONDS);
+ }
+ catch (InterruptedException e) {
+ // may happen
+ }
+ }
+
+ entries.clear();
+
+ // clean up the all storage directories
+ for (File dir : storageDirectories) {
+ try {
+ FileUtils.deleteDirectory(dir);
+ }
+ catch (IOException e) {
+ LOG.error("File cache could not properly clean up storage directory.");
+ }
+ }
+
+ // Remove shutdown hook to prevent resource leaks, unless this is invoked by the
+ // shutdown hook itself
+ if (shutdownHook != null && shutdownHook != Thread.currentThread()) {
+ try {
+ Runtime.getRuntime().removeShutdownHook(shutdownHook);
+ }
+ catch (IllegalStateException e) {
+ // race, JVM is in shutdown already, we can safely ignore this
+ }
+ catch (Throwable t) {
+ LOG.warn("Exception while unregistering file cache's cleanup shutdown hook.");
+ }
+ }
+ }
+ }
+
+ // ------------------------------------------------------------------------
/**
* If the file doesn't exists locally, it will copy the file to the temp directory.
- * @param name file identifier
- * @param entry entry containing all relevant information
- * @param jobID
- * @return copy task
+ *
+ * @param name The name under which the file is registered.
+ * @param entry The cache entry descriptor (path, executable flag)
+ * @param jobID The ID of the job for which the file is copied.
+ * @return The handle to the task that copies the file.
*/
- public FutureTask<Path> createTmpFile(String name, DistributedCacheEntry entry, JobID jobID) {
+ public Future<Path> createTmpFile(String name, DistributedCacheEntry entry, JobID jobID) {
synchronized (lock) {
- if (!jobCounts.containsKey(jobID)) {
- jobCounts.put(jobID, new HashMap<String, Integer>());
+ Map<String, Tuple4<Integer, File, Path, Future<Path>>> jobEntries = entries.get(jobID);
+ if (jobEntries == null) {
+ jobEntries = new HashMap<String, Tuple4<Integer, File, Path, Future<Path>>>();
+ entries.put(jobID, jobEntries);
}
- Map<String, Integer> count = jobCounts.get(jobID);
- if (count.containsKey(name)) {
- count.put(name, count.get(name) + 1);
- } else {
- count.put(name, 1);
+
+ // tuple is (ref-count, parent-temp-dir, cached-file-path, copy-process)
+ Tuple4<Integer, File, Path, Future<Path>> fileEntry = jobEntries.get(name);
+ if (fileEntry != null) {
+ // file is already in the cache. return a future that
+ // immediately returns the file
+ fileEntry.f0 = fileEntry.f0 + 1;
+
+ // return the future. may be that the copy is still in progress
+ return fileEntry.f3;
+ }
+ else {
+ // need to copy the file
+
+ // create the target path
+ File tempDirToUse = new File(storageDirectories[nextDirectory++], jobID.toString());
+ if (nextDirectory >= storageDirectories.length) {
+ nextDirectory = 0;
+ }
+
+ String sourceFile = entry.filePath;
+ int posOfSep = sourceFile.lastIndexOf("/");
+ if (posOfSep > 0) {
+ sourceFile = sourceFile.substring(posOfSep + 1);
+ }
+
+ Path target = new Path(tempDirToUse.getAbsolutePath() + "/" + sourceFile);
+
+ // kick off the copying
+ CopyProcess cp = new CopyProcess(entry, target);
+ FutureTask<Path> copyTask = new FutureTask<Path>(cp);
+ executorService.submit(copyTask);
+
+ // store our entry
+ jobEntries.put(name, new Tuple4<Integer, File, Path, Future<Path>>(1, tempDirToUse, target, copyTask));
+
+ return copyTask;
}
}
- CopyProcess cp = new CopyProcess(entry, jobID);
- FutureTask<Path> copyTask = new FutureTask<Path>(cp);
- executorService.submit(copyTask);
- return copyTask;
}
/**
* Deletes the local file after a 5 second delay.
- * @param name file identifier
- * @param entry entry containing all relevant information
- * @param jobID
+ *
+ * @param name The name under which the file is registered.
+ * @param jobID The ID of the job for which the file is copied.
*/
- public void deleteTmpFile(String name, DistributedCacheEntry entry, JobID jobID) {
- DeleteProcess dp = new DeleteProcess(name, entry, jobID);
+ public void deleteTmpFile(String name, JobID jobID) {
+ DeleteProcess dp = new DeleteProcess(lock, entries, name, jobID);
executorService.schedule(dp, 5000L, TimeUnit.MILLISECONDS);
}
-
- public Path getTempDir(JobID jobID, String childPath) {
- return new Path(GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
- ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH), DistributedCache.TMP_PREFIX + jobID.toString() + "/" + childPath);
- }
-
- public void shutdown() {
- ScheduledExecutorService es = this.executorService;
- if (es != null) {
- es.shutdown();
- try {
- es.awaitTermination(5000L, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- throw new RuntimeException("Error shutting down the file cache", e);
- }
+
+
+ boolean holdsStillReference(String name, JobID jobId) {
+ Map<String, Tuple4<Integer, File, Path, Future<Path>>> jobEntries = entries.get(jobId);
+ if (jobEntries != null) {
+ Tuple4<Integer, File, Path, Future<Path>> entry = jobEntries.get(name);
+ return entry != null && entry.f0 > 0;
+ }
+ else {
+ return false;
}
}
+ // ------------------------------------------------------------------------
+ // Utilities
+ // ------------------------------------------------------------------------
+
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException {
FileSystem sFS = sourcePath.getFileSystem();
FileSystem tFS = targetPath.getFileSystem();
@@ -137,80 +257,144 @@ public class FileCache {
FSDataOutputStream lfsOutput = tFS.create(targetPath, false);
FSDataInputStream fsInput = sFS.open(sourcePath);
IOUtils.copyBytes(fsInput, lfsOutput);
+ //noinspection ResultOfMethodCallIgnored
new File(targetPath.toString()).setExecutable(executable);
- } catch (IOException ioe) {
+ }
+ catch (IOException ioe) {
LOG.error("could not copy file to local file cache.", ioe);
}
}
}
}
+ private static Thread createShutdownHook(final FileCache cache, final Logger logger) {
+
+ Thread shutdownHook = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ cache.shutdown();
+ }
+ catch (Throwable t) {
+ logger.error("Error during shutdown of file cache via JVM shutdown hook: " + t.getMessage(), t);
+ }
+ }
+ });
+
+ try {
+ // Add JVM shutdown hook to call shutdown of service
+ Runtime.getRuntime().addShutdownHook(shutdownHook);
+ return shutdownHook;
+ }
+ catch (IllegalStateException e) {
+ // JVM is already shutting down. no need to do our work
+ return null;
+ }
+ catch (Throwable t) {
+ logger.error("Cannot register shutdown hook that cleanly terminates the file cache service.");
+ return null;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // background processes
+ // ------------------------------------------------------------------------
+
/**
* Asynchronous file copy process
*/
- private class CopyProcess implements Callable<Path> {
-
- private JobID jobID;
- private String filePath;
- private Boolean executable;
+ private static class CopyProcess implements Callable<Path> {
+
+ private final Path filePath;
+ private final Path cachedPath;
+ private boolean executable;
- public CopyProcess(DistributedCacheEntry e, JobID jobID) {
- this.filePath = e.filePath;
+ public CopyProcess(DistributedCacheEntry e, Path cachedPath) {
+ this.filePath = new Path(e.filePath);
this.executable = e.isExecutable;
- this.jobID = jobID;
+ this.cachedPath = cachedPath;
}
+
@Override
- public Path call() {
- Path tmp = getTempDir(jobID, filePath.substring(filePath.lastIndexOf("/") + 1));
- try {
- synchronized (lock) {
- copy(new Path(filePath), tmp, this.executable);
- }
- } catch (IOException e) {
- LOG.error("Could not copy file to local file cache.", e);
- }
- return tmp;
+ public Path call() throws IOException {
+ // let exceptions propagate. we can retrieve them later from
+ // the future and report them upon access to the result
+ copy(filePath, cachedPath, this.executable);
+ return cachedPath;
}
}
/**
* If no task is using this file after 5 seconds, clear it.
*/
- private class DeleteProcess implements Runnable {
-
- private String name;
- private JobID jobID;
- private String filePath;
+ private static class DeleteProcess implements Runnable {
+
+ private final Object lock;
+ private final Map<JobID, Map<String, Tuple4<Integer, File, Path, Future<Path>>>> entries;
+
+ private final String name;
+ private final JobID jobID;
- public DeleteProcess(String name, DistributedCacheEntry e, JobID jobID) {
+ public DeleteProcess(Object lock, Map<JobID, Map<String, Tuple4<Integer, File, Path, Future<Path>>>> entries,
+ String name, JobID jobID)
+ {
+ this.lock = lock;
+ this.entries = entries;
this.name = name;
this.jobID = jobID;
- this.filePath = e.filePath;
}
+
@Override
public void run() {
- Path tmp = getTempDir(jobID, filePath.substring(filePath.lastIndexOf("/") + 1));
try {
synchronized (lock) {
- Map<String, Integer> count = jobCounts.get(jobID);
- if (count.containsKey(name)) {
- count.put(name, count.get(name) - 1);
- if (count.get(name) == 0) {
- if (lfs.exists(tmp)) {
- lfs.delete(tmp, true);
+ Map<String, Tuple4<Integer, File, Path, Future<Path>>> jobEntries = entries.get(jobID);
+
+ if (jobEntries != null) {
+ Tuple4<Integer, File, Path, Future<Path>> entry = jobEntries.get(name);
+
+ if (entry != null) {
+ int count = entry.f0;
+ if (count > 1) {
+ // multiple references still
+ entry.f0 = count - 1;
}
- count.remove(name);
- if (count.isEmpty()) { //delete job directory
- tmp = getTempDir(jobID, "");
- if (lfs.exists(tmp)) {
- lfs.delete(tmp, true);
+ else {
+ // we remove the last reference
+ jobEntries.remove(name);
+ if (jobEntries.isEmpty()) {
+ entries.remove(jobID);
+ }
+
+ // abort the copy
+ entry.f3.cancel(true);
+
+ // remove the file
+ File file = new File(entry.f2.toString());
+ if (file.exists()) {
+ if (file.isDirectory()) {
+ FileUtils.deleteDirectory(file);
+ }
+ else if (!file.delete()) {
+ LOG.error("Could not delete locally cached file " + file.getAbsolutePath());
+ }
+ }
+
+ // remove the job wide temp directory, if it is now empty
+ File parent = entry.f1;
+ if (parent.isDirectory()) {
+ String[] children = parent.list();
+ if (children == null || children.length == 0) {
+ //noinspection ResultOfMethodCallIgnored
+ parent.delete();
+ }
}
- jobCounts.remove(jobID);
}
}
}
}
- } catch (IOException e) {
+ }
+ catch (IOException e) {
LOG.error("Could not delete file from local file cache.", e);
}
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
index c84d888..b4cfa27 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/chaining/ChainedDriver.java
@@ -62,7 +62,8 @@ public abstract class ChainedDriver<IT, OT> implements Collector<IT> {
} else {
Environment env = parent.getEnvironment();
this.udfContext = new DistributedRuntimeUDFContext(taskName, env.getNumberOfSubtasks(),
- env.getIndexInSubtaskGroup(), userCodeClassLoader, parent.getExecutionConfig(), env.getCopyTask());
+ env.getIndexInSubtaskGroup(), userCodeClassLoader, parent.getExecutionConfig(),
+ env.getDistributedCacheEntries());
}
this.executionConfig = executionConfig;
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
index e841066..f4cd354 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/util/DistributedRuntimeUDFContext.java
@@ -21,7 +21,7 @@ package org.apache.flink.runtime.operators.util;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.FutureTask;
+import java.util.concurrent.Future;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.functions.BroadcastVariableInitializer;
@@ -45,7 +45,7 @@ public class DistributedRuntimeUDFContext extends AbstractRuntimeUDFContext {
super(name, numParallelSubtasks, subtaskIndex, userCodeClassLoader, executionConfig);
}
- public DistributedRuntimeUDFContext(String name, int numParallelSubtasks, int subtaskIndex, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, Map<String, FutureTask<Path>> cpTasks) {
+ public DistributedRuntimeUDFContext(String name, int numParallelSubtasks, int subtaskIndex, ClassLoader userCodeClassLoader, ExecutionConfig executionConfig, Map<String, Future<Path>> cpTasks) {
super(name, numParallelSubtasks, subtaskIndex, userCodeClassLoader, executionConfig, cpTasks);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
index 8623f75..c369674 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/filecache/FileCacheDeleteValidationTest.java
@@ -16,17 +16,16 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.filecache;
import java.io.File;
-import java.io.IOException;
-
-import org.junit.Assert;
+import java.util.concurrent.Future;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.fs.Path;
import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
-import org.apache.flink.core.fs.local.LocalFileSystem;
import org.apache.flink.api.common.JobID;
+
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -34,16 +33,16 @@ import org.junit.Test;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
+import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
/**
* Test delete process of {@link FileCache}. The local cache file should not be deleted why another task comes in 5 seconds.
*/
public class FileCacheDeleteValidationTest {
- FileCache fileCache = new FileCache();
- LocalFileSystem lfs = new LocalFileSystem();
- File f;
-
- String testFileContent = "Goethe - Faust: Der Tragoedie erster Teil\n" + "Prolog im Himmel.\n"
+ private static final String testFileContent = "Goethe - Faust: Der Tragoedie erster Teil\n" + "Prolog im Himmel.\n"
+ "Der Herr. Die himmlischen Heerscharen. Nachher Mephistopheles. Die drei\n" + "Erzengel treten vor.\n"
+ "RAPHAEL: Die Sonne toent, nach alter Weise, In Brudersphaeren Wettgesang,\n"
+ "Und ihre vorgeschriebne Reise Vollendet sie mit Donnergang. Ihr Anblick\n"
@@ -58,60 +57,83 @@ public class FileCacheDeleteValidationTest {
+ "Da flammt ein blitzendes Verheeren Dem Pfade vor des Donnerschlags. Doch\n"
+ "deine Boten, Herr, verehren Das sanfte Wandeln deines Tags.\n";
+ private FileCache fileCache;
+ private File f;
+
@Before
- public void createTmpCacheFile() {
+ public void setup() {
+ try {
+ fileCache = new FileCache(new Configuration());
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail("Cannot create FileCache: " + e.getMessage());
+ }
+
f = new File(System.getProperty("java.io.tmpdir"), "cacheFile");
try {
Files.write(testFileContent, f, Charsets.UTF_8);
- } catch (IOException e) {
- throw new RuntimeException("Error initializing the test", e);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ fail("Error initializing the test: " + e.getMessage());
}
}
- @Test
- public void testFileReuseForNextTask() {
- JobID jobID = new JobID();
- String filePath = f.toURI().toString();
- fileCache.createTmpFile("test_file", new DistributedCacheEntry(filePath, false), jobID);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- throw new RuntimeException("Interrupted error", e);
- }
- fileCache.deleteTmpFile("test_file", new DistributedCacheEntry(filePath, false), jobID);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- throw new RuntimeException("Interrupted error", e);
- }
- //new task comes after 1 second
+ @After
+ public void shutdown() {
try {
- Assert.assertTrue("Local cache file should not be deleted when another task comes in 5 seconds!", lfs.exists(fileCache.getTempDir(jobID, "cacheFile")));
- } catch (IOException e) {
- throw new RuntimeException("Interrupted error", e);
+ fileCache.shutdown();
}
- fileCache.createTmpFile("test_file", new DistributedCacheEntry(filePath, false), jobID);
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- throw new RuntimeException("Interrupted error", e);
+ catch (Exception e) {
+ e.printStackTrace();
+ fail("FileCache shutdown failed: " + e.getMessage());
}
- fileCache.deleteTmpFile("test_file", new DistributedCacheEntry(filePath, false), jobID);
+ }
+
+ @Test
+ public void testFileReuseForNextTask() {
try {
- Thread.sleep(7000);
- } catch (InterruptedException e) {
- throw new RuntimeException("Interrupted error", e);
+ final JobID jobID = new JobID();
+ final String fileName = "test_file";
+
+ final String filePath = f.toURI().toString();
+
+ // copy / create the file
+ Future<Path> copyResult = fileCache.createTmpFile(fileName, new DistributedCacheEntry(filePath, false), jobID);
+ copyResult.get();
+
+ // get another reference to the file
+ Future<Path> copyResult2 = fileCache.createTmpFile(fileName, new DistributedCacheEntry(filePath, false), jobID);
+
+ // this should be available immediately
+ assertTrue(copyResult2.isDone());
+
+ // delete the file
+ fileCache.deleteTmpFile(fileName, jobID);
+ // file should not yet be deleted
+ assertTrue(fileCache.holdsStillReference(fileName, jobID));
+
+ // delete the second reference
+ fileCache.deleteTmpFile(fileName, jobID);
+ // file should still not be deleted, but remain for a bit
+ assertTrue(fileCache.holdsStillReference(fileName, jobID));
+
+ fileCache.createTmpFile(fileName, new DistributedCacheEntry(filePath, false), jobID);
+ fileCache.deleteTmpFile(fileName, jobID);
+
+ // after a while, the file should disappear
+ long deadline = System.currentTimeMillis() + 20000;
+ do {
+ Thread.sleep(5500);
+ }
+ while (fileCache.holdsStillReference(fileName, jobID) && System.currentTimeMillis() < deadline);
+
+ assertFalse(fileCache.holdsStillReference(fileName, jobID));
}
- //no task comes in 7 seconds
- try {
- Assert.assertTrue("Local cache file should be deleted when no task comes in 5 seconds!", !lfs.exists(fileCache.getTempDir(jobID, "cacheFile")));
- } catch (IOException e) {
- throw new RuntimeException("Interrupted error", e);
+ catch (Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
}
}
-
- @After
- public void shutdown() {
- fileCache.shutdown();
- }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
index 319a37c..16cd66e 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
@@ -16,16 +16,15 @@
* limitations under the License.
*/
-
package org.apache.flink.runtime.operators.testutils;
-import akka.actor.ActorRef;
import org.apache.flink.api.common.accumulators.Accumulator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.fs.Path;
import org.apache.flink.core.memory.MemorySegment;
import org.apache.flink.runtime.broadcast.BroadcastVariableManager;
import org.apache.flink.runtime.execution.Environment;
+import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
import org.apache.flink.runtime.io.network.partition.consumer.IteratorWrappingTestSingleInputGate;
@@ -46,10 +45,11 @@ import org.apache.flink.util.MutableObjectIterator;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.FutureTask;
+import java.util.concurrent.Future;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
@@ -211,18 +211,13 @@ public class MockEnvironment implements Environment {
}
@Override
- public ActorRef getJobManager() {
- throw new UnsupportedOperationException("getAccumulatorProtocolProxy() is not supported by MockEnvironment");
- }
-
- @Override
public ClassLoader getUserClassLoader() {
return getClass().getClassLoader();
}
@Override
- public Map<String, FutureTask<Path>> getCopyTask() {
- return null;
+ public Map<String, Future<Path>> getDistributedCacheEntries() {
+ return Collections.emptyMap();
}
@Override
@@ -253,6 +248,11 @@ public class MockEnvironment implements Environment {
}
@Override
+ public ExecutionAttemptID getExecutionId() {
+ return new ExecutionAttemptID(0L, 0L);
+ }
+
+ @Override
public BroadcastVariableManager getBroadcastVariableManager() {
return this.bcVarManager;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/1c8d866a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
index eaf8a1d..8ae2ced 100644
--- a/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
+++ b/flink-staging/flink-streaming/flink-streaming-core/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamingRuntimeContext.java
@@ -42,7 +42,7 @@ public class StreamingRuntimeContext extends RuntimeUDFContext {
public StreamingRuntimeContext(String name, Environment env, ClassLoader userCodeClassLoader,
ExecutionConfig executionConfig, Map<String, OperatorState<?>> operatorStates) {
super(name, env.getNumberOfSubtasks(), env.getIndexInSubtaskGroup(), userCodeClassLoader,
- executionConfig, env.getCopyTask());
+ executionConfig, env.getDistributedCacheEntries());
this.env = env;
this.operatorStates = operatorStates;
}