You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2016/10/01 10:10:56 UTC
[1/2] flink git commit: [FLINK-4618] [kafka-connector] Incremented
the commited offset by one to avoid duplicate read message.
Repository: flink
Updated Branches:
refs/heads/master 92f4539af -> 9dbd1e3f7
[FLINK-4618] [kafka-connector] Incremented the commited offset by one to avoid duplicate read message.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ae219a8c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ae219a8c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ae219a8c
Branch: refs/heads/master
Commit: ae219a8c48210cc261ec0a6c7f5aaf261e3d2fd7
Parents: 92f4539
Author: Max Kuklinski <ma...@live.de>
Authored: Fri Sep 30 23:03:30 2016 +0200
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Sat Oct 1 09:45:29 2016 +0800
----------------------------------------------------------------------
.../streaming/connectors/kafka/internal/Kafka09Fetcher.java | 6 +++++-
.../flink/streaming/connectors/kafka/Kafka09FetcherTest.java | 4 ++--
2 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ae219a8c/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java b/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
index 1da2259..7e4177e 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
@@ -289,7 +289,11 @@ public class Kafka09Fetcher<T> extends AbstractFetcher<T, TopicPartition> implem
Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>(partitions.length);
for (KafkaTopicPartitionState<TopicPartition> partition : partitions) {
- Long offset = offsets.get(partition.getKafkaTopicPartition());
+ /*
+ * Increment offset by one, otherwise last record will be read again. This does not affect checkpoints/saved state.
+ * The offset is only read from Kafka/ZK on a fresh startup of a job, not restart or failure. See https://issues.apache.org/jira/browse/FLINK-4618
+ */
+ Long offset = offsets.get(partition.getKafkaTopicPartition()) + 1;
if (offset != null) {
offsetsToCommit.put(partition.getKafkaPartitionHandle(), new OffsetAndMetadata(offset));
partition.setCommittedOffset(offset);
http://git-wip-us.apache.org/repos/asf/flink/blob/ae219a8c/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09FetcherTest.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09FetcherTest.java b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09FetcherTest.java
index 4fd6c9f..5a638b2 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09FetcherTest.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/src/test/java/org/apache/flink/streaming/connectors/kafka/Kafka09FetcherTest.java
@@ -266,7 +266,7 @@ public class Kafka09FetcherTest {
TopicPartition partition = entry.getKey();
if (partition.topic().equals("test")) {
assertEquals(42, partition.partition());
- assertEquals(11L, entry.getValue().offset());
+ assertEquals(12L, entry.getValue().offset());
}
else if (partition.topic().equals("another")) {
assertEquals(99, partition.partition());
@@ -283,7 +283,7 @@ public class Kafka09FetcherTest {
TopicPartition partition = entry.getKey();
if (partition.topic().equals("test")) {
assertEquals(42, partition.partition());
- assertEquals(19L, entry.getValue().offset());
+ assertEquals(20L, entry.getValue().offset());
}
else if (partition.topic().equals("another")) {
assertEquals(99, partition.partition());
[2/2] flink git commit: [FLINK-4618] [kafka-connector] Minor
improvements to comment and variable naming
Posted by tz...@apache.org.
[FLINK-4618] [kafka-connector] Minor improvements to comment and variable naming
This closes #2579
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9dbd1e3f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9dbd1e3f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9dbd1e3f
Branch: refs/heads/master
Commit: 9dbd1e3f79cc2195b4becf00d11a8cca8c3a18b1
Parents: ae219a8
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Sat Oct 1 17:20:19 2016 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Sat Oct 1 17:31:11 2016 +0800
----------------------------------------------------------------------
.../connectors/kafka/internal/Kafka09Fetcher.java | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9dbd1e3f/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java b/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
index 7e4177e..3c2cca3 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.9/src/main/java/org/apache/flink/streaming/connectors/kafka/internal/Kafka09Fetcher.java
@@ -289,14 +289,12 @@ public class Kafka09Fetcher<T> extends AbstractFetcher<T, TopicPartition> implem
Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = new HashMap<>(partitions.length);
for (KafkaTopicPartitionState<TopicPartition> partition : partitions) {
- /*
- * Increment offset by one, otherwise last record will be read again. This does not affect checkpoints/saved state.
- * The offset is only read from Kafka/ZK on a fresh startup of a job, not restart or failure. See https://issues.apache.org/jira/browse/FLINK-4618
- */
- Long offset = offsets.get(partition.getKafkaTopicPartition()) + 1;
- if (offset != null) {
- offsetsToCommit.put(partition.getKafkaPartitionHandle(), new OffsetAndMetadata(offset));
- partition.setCommittedOffset(offset);
+ // committed offsets through the KafkaConsumer need to be 1 more than the last processed offset.
+ // This does not affect Flink's checkpoints/saved state.
+ Long offsetToCommit = offsets.get(partition.getKafkaTopicPartition()) + 1;
+ if (offsetToCommit != null) {
+ offsetsToCommit.put(partition.getKafkaPartitionHandle(), new OffsetAndMetadata(offsetToCommit));
+ partition.setCommittedOffset(offsetToCommit);
}
}