You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2015/02/25 11:12:50 UTC
[2/6] flink git commit: [FLINK-1607] [tests] Fixes
SimpleRecoveryITCase by rebalancing the data sources so that all mappers
receive at least one element
[FLINK-1607] [tests] Fixes SimpleRecoveryITCase by rebalancing the data sources so that all mappers receive at least one element
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/80c2a0c7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/80c2a0c7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/80c2a0c7
Branch: refs/heads/master
Commit: 80c2a0c773583f4a37e86612cb600f49f2bc36cb
Parents: d18401c
Author: Till Rohrmann <tr...@apache.org>
Authored: Fri Feb 20 17:02:04 2015 +0100
Committer: Ufuk Celebi <uc...@apache.org>
Committed: Wed Feb 25 10:57:28 2015 +0100
----------------------------------------------------------------------
.../org/apache/flink/configuration/ConfigConstants.java | 2 +-
.../apache/flink/runtime/deployment/PartitionInfo.java | 7 +++++++
.../io/network/partition/IntermediateResultPartition.java | 10 ++++++++++
.../partition/IntermediateResultPartitionManager.java | 6 +++++-
.../apache/flink/runtime/taskmanager/TaskManager.scala | 1 -
.../apache/flink/test/recovery/SimpleRecoveryITCase.java | 6 ++++--
tools/log4j-travis.properties | 2 +-
7 files changed, 28 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index e5da77f..767648a 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -563,7 +563,7 @@ public final class ConfigConstants {
public static int DEFAULT_AKKA_DISPATCHER_THROUGHPUT = 15;
- public static boolean DEFAULT_AKKA_LOG_LIFECYCLE_EVENTS = true;
+ public static boolean DEFAULT_AKKA_LOG_LIFECYCLE_EVENTS = false;
public static String DEFAULT_AKKA_FRAMESIZE = "10485760b";
http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
index 333340a..2a0e8b1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
@@ -157,4 +157,11 @@ public class PartitionInfo implements IOReadableWritable, Serializable {
return partitions;
}
+
+ @Override
+ public String toString() {
+ return String.format("PartitionInfo(PartitionID: %s, ProducerID: %s, " +
+ "ProducerLocation: %s, ProducerAddress: %s)", partitionId, producerExecutionId,
+ producerLocation, producerAddress);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
index 80bd38d..f9253b6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
@@ -186,6 +186,9 @@ public class IntermediateResultPartition implements BufferPoolOwner {
}
public void releaseAllResources() throws IOException {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Release all resources of {}.", this);
+ }
synchronized (queues) {
if (!isReleased) {
try {
@@ -313,4 +316,11 @@ public class IntermediateResultPartition implements BufferPoolOwner {
return new IntermediateResultPartition(environment, partitionIndex, jobId, executionId, partitionId, partitionType,
partitionQueues, networkEnvironment);
}
+
+ @Override
+ public String toString() {
+ return String.format("IntermediateResultPartition(JobID: %s, ExecutionID: %s, " +
+ "PartitionID: %s, PartitionType: %s, [num queues: %d, (isFinished: %b)",
+ jobId, producerExecutionId, partitionId, partitionType, queues.length, isFinished);
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
index 55a741b..46c690e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
@@ -41,11 +41,15 @@ public class IntermediateResultPartitionManager implements IntermediateResultPar
private static final Logger LOG = LoggerFactory.getLogger(IntermediateResultPartitionManager.class);
- public final Table<ExecutionAttemptID, IntermediateResultPartitionID, IntermediateResultPartition> partitions = HashBasedTable.create();
+ private final Table<ExecutionAttemptID, IntermediateResultPartitionID,
+ IntermediateResultPartition> partitions = HashBasedTable.create();
private boolean isShutdown;
public void registerIntermediateResultPartition(IntermediateResultPartition partition) throws IOException {
+ if(LOG.isDebugEnabled()){
+ LOG.debug("Register intermediate result partition {}.", partition);
+ }
synchronized (partitions) {
if (isShutdown) {
throw new IOException("Intermediate result partition manager has already been shut down.");
http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 74c607e..54b628a 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -381,7 +381,6 @@ class TaskManager(val connectionInfo: InstanceConnectionInfo,
startRegisteringTask = System.currentTimeMillis()
}
- manager.registerTask(jobID, executionID, tdd.getRequiredJarFiles)
// triggers the download of all missing jar files from the job manager
manager.registerTask(jobID, executionID, tdd.getRequiredJarFiles)
http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
index 504fec7..1591d67 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
@@ -30,7 +30,6 @@ import org.apache.flink.runtime.client.JobExecutionException;
import org.apache.flink.test.util.ForkableFlinkMiniCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import java.util.ArrayList;
@@ -38,7 +37,6 @@ import java.util.List;
import static org.junit.Assert.*;
-@Ignore("Not working properly")
public class SimpleRecoveryITCase {
@@ -81,6 +79,7 @@ public class SimpleRecoveryITCase {
env.setNumberOfExecutionRetries(0);
env.generateSequence(1, 10)
+ .rebalance()
.map(new FailingMapper1<Long>())
.reduce(new ReduceFunction<Long>() {
@Override
@@ -109,6 +108,7 @@ public class SimpleRecoveryITCase {
env.setNumberOfExecutionRetries(0);
env.generateSequence(1, 10)
+ .rebalance()
.map(new FailingMapper1<Long>())
.reduce(new ReduceFunction<Long>() {
@Override
@@ -154,6 +154,7 @@ public class SimpleRecoveryITCase {
env.setNumberOfExecutionRetries(1);
env.generateSequence(1, 10)
+ .rebalance()
.map(new FailingMapper2<Long>())
.reduce(new ReduceFunction<Long>() {
@Override
@@ -197,6 +198,7 @@ public class SimpleRecoveryITCase {
env.setNumberOfExecutionRetries(3);
env.generateSequence(1, 10)
+ .rebalance()
.map(new FailingMapper3<Long>())
.reduce(new ReduceFunction<Long>() {
@Override
http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/tools/log4j-travis.properties
----------------------------------------------------------------------
diff --git a/tools/log4j-travis.properties b/tools/log4j-travis.properties
index f69a673..d92fcba 100644
--- a/tools/log4j-travis.properties
+++ b/tools/log4j-travis.properties
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-log4j.rootLogger=INFO, file
+log4j.rootLogger=DEBUG, file
# -----------------------------------------------------------------------------
# Console (use 'console')