You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2019/06/29 07:21:23 UTC
[flink] branch master updated (f172129 -> 34ce59b)
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from f172129 [FLINK-12989][hive]: Generate HiveTableSink from from a Hive table
new 739c496 [FLINK-12882][network] Remove ExecutionAttemptID argument from ResultPartitionFactory#create
new 6144834 [hotfix][network,tests] Use the simple constructor of ResultPartitionID in tests
new 34ce59b [hotfix][runtime] add checkNotNull for parameters in ResultPartitionID constructor
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/flink/runtime/io/network/NettyShuffleEnvironment.java | 5 +----
.../runtime/io/network/partition/ResultPartitionFactory.java | 4 +---
.../flink/runtime/io/network/partition/ResultPartitionID.java | 8 ++++++--
.../runtime/io/network/netty/NettyMessageSerializationTest.java | 8 +++-----
.../runtime/io/network/partition/ResultPartitionFactoryTest.java | 3 +--
.../io/network/partition/consumer/SingleInputGateTest.java | 4 ++--
6 files changed, 14 insertions(+), 18 deletions(-)
[flink] 03/03: [hotfix][runtime] add checkNotNull for parameters in
ResultPartitionID constructor
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 34ce59bf3915f992d192bd10cc4c8bda285a44dd
Author: Zhijiang <wa...@aliyun.com>
AuthorDate: Thu Jun 27 13:10:59 2019 +0800
[hotfix][runtime] add checkNotNull for parameters in ResultPartitionID constructor
---
.../flink/runtime/io/network/partition/ResultPartitionID.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java
index 317d17f..fd6e68b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java
@@ -25,6 +25,8 @@ import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
import java.io.Serializable;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
/**
* Runtime identifier of a produced {@link IntermediateResultPartition}.
*
@@ -46,8 +48,8 @@ public final class ResultPartitionID implements Serializable {
}
public ResultPartitionID(IntermediateResultPartitionID partitionId, ExecutionAttemptID producerId) {
- this.partitionId = partitionId;
- this.producerId = producerId;
+ this.partitionId = checkNotNull(partitionId);
+ this.producerId = checkNotNull(producerId);
}
public IntermediateResultPartitionID getPartitionId() {
[flink] 01/03: [FLINK-12882][network] Remove ExecutionAttemptID
argument from ResultPartitionFactory#create
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 739c496d2d5a9d12cc7ee7b021fed4a9cd0499ed
Author: Zhijiang <wa...@aliyun.com>
AuthorDate: Thu Jun 27 11:56:06 2019 +0800
[FLINK-12882][network] Remove ExecutionAttemptID argument from ResultPartitionFactory#create
The ResultPartitionID could be got directly from ResultPartitionDeploymentDescriptor, so it is no need to pass
ExecutionAttemptID to construct new ResultPartitionID during creating ResultPartition in factory.
---
.../org/apache/flink/runtime/io/network/NettyShuffleEnvironment.java | 5 +----
.../flink/runtime/io/network/partition/ResultPartitionFactory.java | 4 +---
.../runtime/io/network/partition/ResultPartitionFactoryTest.java | 3 +--
3 files changed, 3 insertions(+), 9 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironment.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironment.java
index 1565ed8..5171d75 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironment.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/NettyShuffleEnvironment.java
@@ -187,10 +187,7 @@ public class NettyShuffleEnvironment implements ShuffleEnvironment<ResultPartiti
ResultPartition[] resultPartitions = new ResultPartition[resultPartitionDeploymentDescriptors.size()];
int counter = 0;
for (ResultPartitionDeploymentDescriptor rpdd : resultPartitionDeploymentDescriptors) {
- resultPartitions[counter++] = resultPartitionFactory.create(
- ownerContext.getOwnerName(),
- ownerContext.getExecutionAttemptID(),
- rpdd);
+ resultPartitions[counter++] = resultPartitionFactory.create(ownerContext.getOwnerName(), rpdd);
}
registerOutputMetrics(config.isNetworkDetailedMetrics(), ownerContext.getOutputGroup(), resultPartitions);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
index 9479653..6fc5cfc 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactory.java
@@ -20,7 +20,6 @@ package org.apache.flink.runtime.io.network.partition;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
import org.apache.flink.runtime.io.network.NettyShuffleEnvironment;
import org.apache.flink.runtime.io.network.buffer.BufferPool;
@@ -73,12 +72,11 @@ public class ResultPartitionFactory {
public ResultPartition create(
@Nonnull String taskNameWithSubtaskAndId,
- @Nonnull ExecutionAttemptID executionAttemptID,
@Nonnull ResultPartitionDeploymentDescriptor desc) {
return create(
taskNameWithSubtaskAndId,
- new ResultPartitionID(desc.getPartitionId(), executionAttemptID),
+ desc.getShuffleDescriptor().getResultPartitionID(),
desc.getPartitionType(),
desc.getNumberOfSubpartitions(),
desc.getMaxParallelism(),
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java
index 8e95f8a..0c6848b 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionFactoryTest.java
@@ -18,7 +18,6 @@
package org.apache.flink.runtime.io.network.partition;
import org.apache.flink.runtime.deployment.ResultPartitionDeploymentDescriptor;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.disk.iomanager.NoOpIOManager;
import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool;
import org.apache.flink.runtime.jobgraph.IntermediateDataSetID;
@@ -74,6 +73,6 @@ public class ResultPartitionFactoryTest extends TestLogger {
releaseType
);
- return factory.create("test", new ExecutionAttemptID(), descriptor);
+ return factory.create("test", descriptor);
}
}
[flink] 02/03: [hotfix][network,
tests] Use the simple constructor of ResultPartitionID in tests
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 614483450129069072b0da6bde4e993fdd63fd15
Author: Zhijiang <wa...@aliyun.com>
AuthorDate: Thu Jun 27 12:03:05 2019 +0800
[hotfix][network,tests] Use the simple constructor of ResultPartitionID in tests
---
.../flink/runtime/io/network/partition/ResultPartitionID.java | 2 ++
.../runtime/io/network/netty/NettyMessageSerializationTest.java | 8 +++-----
.../io/network/partition/consumer/SingleInputGateTest.java | 4 ++--
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java
index cee79a0..317d17f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/ResultPartitionID.java
@@ -18,6 +18,7 @@
package org.apache.flink.runtime.io.network.partition;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.executiongraph.IntermediateResultPartition;
import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
@@ -39,6 +40,7 @@ public final class ResultPartitionID implements Serializable {
private final ExecutionAttemptID producerId;
+ @VisibleForTesting
public ResultPartitionID() {
this(new IntermediateResultPartitionID(), new ExecutionAttemptID());
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageSerializationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageSerializationTest.java
index a8c473f..9d16dac 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageSerializationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageSerializationTest.java
@@ -20,13 +20,11 @@ package org.apache.flink.runtime.io.network.netty;
import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.runtime.event.task.IntegerTaskEvent;
-import org.apache.flink.runtime.executiongraph.ExecutionAttemptID;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler;
import org.apache.flink.runtime.io.network.buffer.NetworkBuffer;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.io.network.partition.consumer.InputChannelID;
-import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID;
import org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf;
import org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel;
@@ -97,7 +95,7 @@ public class NettyMessageSerializationTest {
}
{
- NettyMessage.PartitionRequest expected = new NettyMessage.PartitionRequest(new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), random.nextInt(), new InputChannelID(), random.nextInt());
+ NettyMessage.PartitionRequest expected = new NettyMessage.PartitionRequest(new ResultPartitionID(), random.nextInt(), new InputChannelID(), random.nextInt());
NettyMessage.PartitionRequest actual = encodeAndDecode(expected);
assertEquals(expected.partitionId, actual.partitionId);
@@ -107,7 +105,7 @@ public class NettyMessageSerializationTest {
}
{
- NettyMessage.TaskEventRequest expected = new NettyMessage.TaskEventRequest(new IntegerTaskEvent(random.nextInt()), new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), new InputChannelID());
+ NettyMessage.TaskEventRequest expected = new NettyMessage.TaskEventRequest(new IntegerTaskEvent(random.nextInt()), new ResultPartitionID(), new InputChannelID());
NettyMessage.TaskEventRequest actual = encodeAndDecode(expected);
assertEquals(expected.event, actual.event);
@@ -130,7 +128,7 @@ public class NettyMessageSerializationTest {
}
{
- NettyMessage.AddCredit expected = new NettyMessage.AddCredit(new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), random.nextInt(Integer.MAX_VALUE) + 1, new InputChannelID());
+ NettyMessage.AddCredit expected = new NettyMessage.AddCredit(new ResultPartitionID(), random.nextInt(Integer.MAX_VALUE) + 1, new InputChannelID());
NettyMessage.AddCredit actual = encodeAndDecode(expected);
assertEquals(expected.partitionId, actual.partitionId);
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
index 39ec9e2..d737949 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java
@@ -196,7 +196,7 @@ public class SingleInputGateTest extends InputGateTestBase {
inputGate.setBufferPool(bufferPool);
// Local
- ResultPartitionID localPartitionId = new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID());
+ ResultPartitionID localPartitionId = new ResultPartitionID();
InputChannelBuilder.newBuilder()
.setPartitionId(localPartitionId)
@@ -205,7 +205,7 @@ public class SingleInputGateTest extends InputGateTestBase {
.buildLocalAndSetToGate(inputGate);
// Unknown
- ResultPartitionID unknownPartitionId = new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID());
+ ResultPartitionID unknownPartitionId = new ResultPartitionID();
InputChannelBuilder.newBuilder()
.setChannelIndex(1)