You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2015/02/25 11:12:50 UTC

[2/6] flink git commit: [FLINK-1607] [tests] Fixes SimpleRecoveryITCase by rebalancing the data sources so that all mappers receive at least one element

[FLINK-1607] [tests] Fixes SimpleRecoveryITCase by rebalancing the data sources so that all mappers receive at least one element


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/80c2a0c7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/80c2a0c7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/80c2a0c7

Branch: refs/heads/master
Commit: 80c2a0c773583f4a37e86612cb600f49f2bc36cb
Parents: d18401c
Author: Till Rohrmann <tr...@apache.org>
Authored: Fri Feb 20 17:02:04 2015 +0100
Committer: Ufuk Celebi <uc...@apache.org>
Committed: Wed Feb 25 10:57:28 2015 +0100

----------------------------------------------------------------------
 .../org/apache/flink/configuration/ConfigConstants.java   |  2 +-
 .../apache/flink/runtime/deployment/PartitionInfo.java    |  7 +++++++
 .../io/network/partition/IntermediateResultPartition.java | 10 ++++++++++
 .../partition/IntermediateResultPartitionManager.java     |  6 +++++-
 .../apache/flink/runtime/taskmanager/TaskManager.scala    |  1 -
 .../apache/flink/test/recovery/SimpleRecoveryITCase.java  |  6 ++++--
 tools/log4j-travis.properties                             |  2 +-
 7 files changed, 28 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
index e5da77f..767648a 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/ConfigConstants.java
@@ -563,7 +563,7 @@ public final class ConfigConstants {
 
 	public static int DEFAULT_AKKA_DISPATCHER_THROUGHPUT = 15;
 
-	public static boolean DEFAULT_AKKA_LOG_LIFECYCLE_EVENTS = true;
+	public static boolean DEFAULT_AKKA_LOG_LIFECYCLE_EVENTS = false;
 
 	public static String DEFAULT_AKKA_FRAMESIZE = "10485760b";
 

http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
index 333340a..2a0e8b1 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/deployment/PartitionInfo.java
@@ -157,4 +157,11 @@ public class PartitionInfo implements IOReadableWritable, Serializable {
 
 		return partitions;
 	}
+
+	@Override
+	public String toString() {
+		return String.format("PartitionInfo(PartitionID: %s, ProducerID: %s, " +
+				"ProducerLocation: %s, ProducerAddress: %s)", partitionId, producerExecutionId,
+				producerLocation, producerAddress);
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
index 80bd38d..f9253b6 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartition.java
@@ -186,6 +186,9 @@ public class IntermediateResultPartition implements BufferPoolOwner {
 	}
 
 	public void releaseAllResources() throws IOException {
+		if(LOG.isDebugEnabled()) {
+			LOG.debug("Release all resources of {}.", this);
+		}
 		synchronized (queues) {
 			if (!isReleased) {
 				try {
@@ -313,4 +316,11 @@ public class IntermediateResultPartition implements BufferPoolOwner {
 		return new IntermediateResultPartition(environment, partitionIndex, jobId, executionId, partitionId, partitionType,
 				partitionQueues, networkEnvironment);
 	}
+
+	@Override
+	public String toString() {
+		return String.format("IntermediateResultPartition(JobID: %s, ExecutionID: %s, " +
+				"PartitionID: %s, PartitionType: %s, [num queues: %d, (isFinished: %b)",
+				jobId, producerExecutionId, partitionId, partitionType, queues.length, isFinished);
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
index 55a741b..46c690e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/IntermediateResultPartitionManager.java
@@ -41,11 +41,15 @@ public class IntermediateResultPartitionManager implements IntermediateResultPar
 
 	private static final Logger LOG = LoggerFactory.getLogger(IntermediateResultPartitionManager.class);
 
-	public final Table<ExecutionAttemptID, IntermediateResultPartitionID, IntermediateResultPartition> partitions = HashBasedTable.create();
+	private final Table<ExecutionAttemptID, IntermediateResultPartitionID,
+			IntermediateResultPartition> partitions = HashBasedTable.create();
 
 	private boolean isShutdown;
 
 	public void registerIntermediateResultPartition(IntermediateResultPartition partition) throws IOException {
+		if(LOG.isDebugEnabled()){
+			LOG.debug("Register intermediate result partition {}.", partition);
+		}
 		synchronized (partitions) {
 			if (isShutdown) {
 				throw new IOException("Intermediate result partition manager has already been shut down.");

http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
index 74c607e..54b628a 100644
--- a/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
+++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/taskmanager/TaskManager.scala
@@ -381,7 +381,6 @@ class TaskManager(val connectionInfo: InstanceConnectionInfo,
             startRegisteringTask = System.currentTimeMillis()
           }
 
-          manager.registerTask(jobID, executionID, tdd.getRequiredJarFiles)
           // triggers the download of all missing jar files from the job manager
           manager.registerTask(jobID, executionID, tdd.getRequiredJarFiles)
 

http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
index 504fec7..1591d67 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/SimpleRecoveryITCase.java
@@ -30,7 +30,6 @@ import org.apache.flink.runtime.client.JobExecutionException;
 import org.apache.flink.test.util.ForkableFlinkMiniCluster;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -38,7 +37,6 @@ import java.util.List;
 
 import static org.junit.Assert.*;
 
-@Ignore("Not working properly")
 public class SimpleRecoveryITCase {
 
 
@@ -81,6 +79,7 @@ public class SimpleRecoveryITCase {
 				env.setNumberOfExecutionRetries(0);
 
 				env.generateSequence(1, 10)
+						.rebalance()
 						.map(new FailingMapper1<Long>())
 						.reduce(new ReduceFunction<Long>() {
 							@Override
@@ -109,6 +108,7 @@ public class SimpleRecoveryITCase {
 				env.setNumberOfExecutionRetries(0);
 
 				env.generateSequence(1, 10)
+						.rebalance()
 						.map(new FailingMapper1<Long>())
 						.reduce(new ReduceFunction<Long>() {
 							@Override
@@ -154,6 +154,7 @@ public class SimpleRecoveryITCase {
 			env.setNumberOfExecutionRetries(1);
 
 			env.generateSequence(1, 10)
+					.rebalance()
 					.map(new FailingMapper2<Long>())
 					.reduce(new ReduceFunction<Long>() {
 						@Override
@@ -197,6 +198,7 @@ public class SimpleRecoveryITCase {
 			env.setNumberOfExecutionRetries(3);
 
 			env.generateSequence(1, 10)
+					.rebalance()
 					.map(new FailingMapper3<Long>())
 					.reduce(new ReduceFunction<Long>() {
 						@Override

http://git-wip-us.apache.org/repos/asf/flink/blob/80c2a0c7/tools/log4j-travis.properties
----------------------------------------------------------------------
diff --git a/tools/log4j-travis.properties b/tools/log4j-travis.properties
index f69a673..d92fcba 100644
--- a/tools/log4j-travis.properties
+++ b/tools/log4j-travis.properties
@@ -16,7 +16,7 @@
 # limitations under the License.
 ################################################################################
 
-log4j.rootLogger=INFO, file
+log4j.rootLogger=DEBUG, file
 
 # -----------------------------------------------------------------------------
 # Console (use 'console')