You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by jg...@apache.org on 2017/04/27 21:29:33 UTC
[1/3] kafka git commit: KAFKA-4818; Exactly once transactional clients
Repository: kafka
Updated Branches:
refs/heads/trunk bb663d04f -> a82f194b2
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
index 0d19aa0..7de378d 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java
@@ -21,7 +21,6 @@ import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.MockClient;
import org.apache.kafka.clients.NodeApiVersions;
import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.kafka.clients.producer.TransactionState;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.Node;
@@ -377,8 +376,8 @@ public class SenderTest {
@Test
public void testInitPidRequest() throws Exception {
final long producerId = 343434L;
- TransactionState transactionState = new TransactionState(new MockTime());
- setupWithTransactionState(transactionState);
+ TransactionManager transactionManager = new TransactionManager();
+ setupWithTransactionState(transactionManager);
client.setNode(new Node(1, "localhost", 33343));
client.prepareResponse(new MockClient.RequestMatcher() {
@Override
@@ -387,17 +386,17 @@ public class SenderTest {
}
}, new InitPidResponse(Errors.NONE, producerId, (short) 0));
sender.run(time.milliseconds());
- assertTrue(transactionState.hasPid());
- assertEquals(producerId, transactionState.pidAndEpoch().producerId);
- assertEquals((short) 0, transactionState.pidAndEpoch().epoch);
+ assertTrue(transactionManager.hasPid());
+ assertEquals(producerId, transactionManager.pidAndEpoch().producerId);
+ assertEquals((short) 0, transactionManager.pidAndEpoch().epoch);
}
@Test
public void testSequenceNumberIncrement() throws InterruptedException {
final long producerId = 343434L;
- TransactionState transactionState = new TransactionState(new MockTime());
- transactionState.setPidAndEpoch(producerId, (short) 0);
- setupWithTransactionState(transactionState);
+ TransactionManager transactionManager = new TransactionManager();
+ transactionManager.setPidAndEpoch(producerId, (short) 0);
+ setupWithTransactionState(transactionManager);
client.setNode(new Node(1, "localhost", 33343));
int maxRetries = 10;
@@ -413,7 +412,7 @@ public class SenderTest {
time,
REQUEST_TIMEOUT,
50,
- transactionState,
+ transactionManager,
apiVersions
);
@@ -442,15 +441,15 @@ public class SenderTest {
sender.run(time.milliseconds()); // receive response
assertTrue(responseFuture.isDone());
- assertEquals((long) transactionState.sequenceNumber(tp0), 1L);
+ assertEquals((long) transactionManager.sequenceNumber(tp0), 1L);
}
@Test
public void testAbortRetryWhenPidChanges() throws InterruptedException {
final long producerId = 343434L;
- TransactionState transactionState = new TransactionState(new MockTime());
- transactionState.setPidAndEpoch(producerId, (short) 0);
- setupWithTransactionState(transactionState);
+ TransactionManager transactionManager = new TransactionManager();
+ transactionManager.setPidAndEpoch(producerId, (short) 0);
+ setupWithTransactionState(transactionManager);
client.setNode(new Node(1, "localhost", 33343));
int maxRetries = 10;
@@ -466,7 +465,7 @@ public class SenderTest {
time,
REQUEST_TIMEOUT,
50,
- transactionState,
+ transactionManager,
apiVersions
);
@@ -481,7 +480,7 @@ public class SenderTest {
assertEquals(0, client.inFlightRequestCount());
assertFalse("Client ready status should be false", client.isReady(node, 0L));
- transactionState.setPidAndEpoch(producerId + 1, (short) 0);
+ transactionManager.setPidAndEpoch(producerId + 1, (short) 0);
sender.run(time.milliseconds()); // receive error
sender.run(time.milliseconds()); // reconnect
sender.run(time.milliseconds()); // nothing to do, since the pid has changed. We should check the metrics for errors.
@@ -491,15 +490,15 @@ public class SenderTest {
assertTrue("Expected non-zero value for record send errors", recordErrors.value() > 0);
assertTrue(responseFuture.isDone());
- assertEquals((long) transactionState.sequenceNumber(tp0), 0L);
+ assertEquals((long) transactionManager.sequenceNumber(tp0), 0L);
}
@Test
public void testResetWhenOutOfOrderSequenceReceived() throws InterruptedException {
final long producerId = 343434L;
- TransactionState transactionState = new TransactionState(new MockTime());
- transactionState.setPidAndEpoch(producerId, (short) 0);
- setupWithTransactionState(transactionState);
+ TransactionManager transactionManager = new TransactionManager();
+ transactionManager.setPidAndEpoch(producerId, (short) 0);
+ setupWithTransactionState(transactionManager);
client.setNode(new Node(1, "localhost", 33343));
int maxRetries = 10;
@@ -515,7 +514,7 @@ public class SenderTest {
time,
REQUEST_TIMEOUT,
50,
- transactionState,
+ transactionManager,
apiVersions
);
@@ -529,7 +528,7 @@ public class SenderTest {
sender.run(time.milliseconds());
assertTrue(responseFuture.isDone());
- assertFalse("Expected transaction state to be reset upon receiving an OutOfOrderSequenceException", transactionState.hasPid());
+ assertFalse("Expected transaction state to be reset upon receiving an OutOfOrderSequenceException", transactionManager.hasPid());
}
private void completedWithError(Future<RecordMetadata> future, Errors error) throws Exception {
@@ -548,12 +547,12 @@ public class SenderTest {
return new ProduceResponse(partResp, throttleTimeMs);
}
- private void setupWithTransactionState(TransactionState transactionState) {
+ private void setupWithTransactionState(TransactionManager transactionManager) {
Map<String, String> metricTags = new LinkedHashMap<>();
metricTags.put("client-id", CLIENT_ID);
MetricConfig metricConfig = new MetricConfig().tags(metricTags);
this.metrics = new Metrics(metricConfig, time);
- this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionState);
+ this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionManager);
this.sender = new Sender(this.client,
this.metadata,
this.accumulator,
@@ -565,7 +564,7 @@ public class SenderTest {
this.time,
REQUEST_TIMEOUT,
50,
- transactionState,
+ transactionManager,
apiVersions);
this.metadata.update(this.cluster, Collections.<String>emptySet(), time.milliseconds());
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
new file mode 100644
index 0000000..7c5b2b5
--- /dev/null
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java
@@ -0,0 +1,588 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.producer.internals;
+
+import org.apache.kafka.clients.ApiVersions;
+import org.apache.kafka.clients.Metadata;
+import org.apache.kafka.clients.MockClient;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.internals.ClusterResourceListeners;
+import org.apache.kafka.common.metrics.MetricConfig;
+import org.apache.kafka.common.metrics.Metrics;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.CompressionType;
+import org.apache.kafka.common.record.MemoryRecords;
+import org.apache.kafka.common.record.MutableRecordBatch;
+import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.AddOffsetsToTxnRequest;
+import org.apache.kafka.common.requests.AddOffsetsToTxnResponse;
+import org.apache.kafka.common.requests.AddPartitionsToTxnRequest;
+import org.apache.kafka.common.requests.AddPartitionsToTxnResponse;
+import org.apache.kafka.common.requests.EndTxnRequest;
+import org.apache.kafka.common.requests.EndTxnResponse;
+import org.apache.kafka.common.requests.FindCoordinatorRequest;
+import org.apache.kafka.common.requests.FindCoordinatorResponse;
+import org.apache.kafka.common.requests.InitPidRequest;
+import org.apache.kafka.common.requests.InitPidResponse;
+import org.apache.kafka.common.requests.ProduceRequest;
+import org.apache.kafka.common.requests.ProduceResponse;
+import org.apache.kafka.common.requests.TransactionResult;
+import org.apache.kafka.common.requests.TxnOffsetCommitRequest;
+import org.apache.kafka.common.requests.TxnOffsetCommitResponse;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.test.TestUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TransactionManagerTest {
+ private static final int MAX_REQUEST_SIZE = 1024 * 1024;
+ private static final short ACKS_ALL = -1;
+ private static final int MAX_RETRIES = 0;
+ private static final String CLIENT_ID = "clientId";
+ private static final int MAX_BLOCK_TIMEOUT = 1000;
+ private static final int REQUEST_TIMEOUT = 1000;
+ private final String transactionalId = "foobar";
+ private final int transactionTimeoutMs = 1121;
+
+ private TopicPartition tp0 = new TopicPartition("test", 0);
+ private TopicPartition tp1 = new TopicPartition("test", 1);
+ private MockTime time = new MockTime();
+ private MockClient client = new MockClient(time);
+
+ private Metadata metadata = new Metadata(0, Long.MAX_VALUE, true, new ClusterResourceListeners());
+ private ApiVersions apiVersions = new ApiVersions();
+ private Cluster cluster = TestUtils.singletonCluster("test", 2);
+ private RecordAccumulator accumulator = null;
+ private Sender sender = null;
+ private TransactionManager transactionManager = null;
+ private Node brokerNode = null;
+
+ @Before
+ public void setup() {
+ Map<String, String> metricTags = new LinkedHashMap<>();
+ metricTags.put("client-id", CLIENT_ID);
+ int batchSize = 16 * 1024;
+ MetricConfig metricConfig = new MetricConfig().tags(metricTags);
+ this.brokerNode = new Node(0, "localhost", 2211);
+ this.transactionManager = new TransactionManager(transactionalId, transactionTimeoutMs);
+ Metrics metrics = new Metrics(metricConfig, time);
+ this.accumulator = new RecordAccumulator(batchSize, 1024 * 1024, CompressionType.NONE, 0L, 0L, metrics, time, apiVersions, transactionManager);
+ this.sender = new Sender(this.client,
+ this.metadata,
+ this.accumulator,
+ true,
+ MAX_REQUEST_SIZE,
+ ACKS_ALL,
+ MAX_RETRIES,
+ metrics,
+ this.time,
+ REQUEST_TIMEOUT,
+ 50,
+ transactionManager,
+ apiVersions);
+ this.metadata.update(this.cluster, Collections.<String>emptySet(), time.milliseconds());
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testInvalidSequenceIncrement() {
+ TransactionManager transactionManager = new TransactionManager();
+ transactionManager.incrementSequenceNumber(tp0, 3333);
+ }
+
+ @Test
+ public void testDefaultSequenceNumber() {
+ TransactionManager transactionManager = new TransactionManager();
+ assertEquals((int) transactionManager.sequenceNumber(tp0), 0);
+ transactionManager.incrementSequenceNumber(tp0, 3);
+ assertEquals((int) transactionManager.sequenceNumber(tp0), 3);
+ }
+
+
+ @Test
+ public void testProducerIdReset() {
+ TransactionManager transactionManager = new TransactionManager();
+ assertEquals((int) transactionManager.sequenceNumber(tp0), 0);
+ transactionManager.incrementSequenceNumber(tp0, 3);
+ assertEquals((int) transactionManager.sequenceNumber(tp0), 3);
+ transactionManager.resetProducerId();
+ assertEquals((int) transactionManager.sequenceNumber(tp0), 0);
+ }
+
+ @Test
+ public void testBasicTransaction() throws InterruptedException {
+ client.setNode(brokerNode);
+ // This is called from the initTransactions method in the producer as the first order of business.
+ // It finds the coordinator and then gets a PID.
+ final long pid = 13131L;
+ final short epoch = 1;
+ transactionManager.initializeTransactions();
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+
+ sender.run(time.milliseconds()); // find coordinator
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+
+ prepareInitPidResponse(Errors.NONE, false, pid, epoch);
+
+ sender.run(time.milliseconds()); // get pid.
+
+ assertTrue(transactionManager.hasPid());
+ transactionManager.beginTransaction();
+ transactionManager.maybeAddPartitionToTransaction(tp0);
+
+ Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
+ "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future;
+
+ assertFalse(responseFuture.isDone());
+ prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
+
+ prepareProduceResponse(Errors.NONE, pid, epoch);
+ assertFalse(transactionManager.transactionContainsPartition(tp0));
+ sender.run(time.milliseconds()); // send addPartitions.
+ // Check that only addPartitions was sent.
+ assertTrue(transactionManager.transactionContainsPartition(tp0));
+ assertFalse(responseFuture.isDone());
+
+ sender.run(time.milliseconds()); // send produce request.
+ assertTrue(responseFuture.isDone());
+
+ Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
+ offsets.put(tp1, new OffsetAndMetadata(1));
+ final String consumerGroupId = "myconsumergroup";
+ FutureTransactionalResult addOffsetsResult = transactionManager.sendOffsetsToTransaction(offsets, consumerGroupId);
+
+ assertFalse(transactionManager.hasPendingOffsetCommits());
+
+ client.prepareResponse(new MockClient.RequestMatcher() {
+ @Override
+ public boolean matches(AbstractRequest body) {
+ AddOffsetsToTxnRequest addOffsetsToTxnRequest = (AddOffsetsToTxnRequest) body;
+ assertEquals(consumerGroupId, addOffsetsToTxnRequest.consumerGroupId());
+ assertEquals(transactionalId, addOffsetsToTxnRequest.transactionalId());
+ assertEquals(pid, addOffsetsToTxnRequest.producerId());
+ assertEquals(epoch, addOffsetsToTxnRequest.producerEpoch());
+ return true;
+ }
+ }, new AddOffsetsToTxnResponse(Errors.NONE));
+
+ sender.run(time.milliseconds()); // Send AddOffsetsRequest
+ assertTrue(transactionManager.hasPendingOffsetCommits()); // We should now have created and queued the offset commit request.
+ assertFalse(addOffsetsResult.isDone());
+
+ Map<TopicPartition, Errors> txnOffsetCommitResponse = new HashMap<>();
+ txnOffsetCommitResponse.put(tp1, Errors.NONE);
+
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.GROUP, consumerGroupId);
+
+ client.prepareResponse(new MockClient.RequestMatcher() {
+ @Override
+ public boolean matches(AbstractRequest body) {
+ TxnOffsetCommitRequest txnOffsetCommitRequest = (TxnOffsetCommitRequest) body;
+ assertEquals(consumerGroupId, txnOffsetCommitRequest.consumerGroupId());
+ assertEquals(pid, txnOffsetCommitRequest.producerId());
+ assertEquals(epoch, txnOffsetCommitRequest.producerEpoch());
+ return true;
+ }
+ }, new TxnOffsetCommitResponse(txnOffsetCommitResponse));
+
+ assertEquals(null, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP));
+ sender.run(time.milliseconds()); // try to send TxnOffsetCommitRequest, but find we don't have a group coordinator.
+ sender.run(time.milliseconds()); // send find coordinator for group request
+ assertNotNull(transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.GROUP));
+ assertTrue(transactionManager.hasPendingOffsetCommits());
+
+ sender.run(time.milliseconds()); // send TxnOffsetCommitRequest commit.
+
+ assertFalse(transactionManager.hasPendingOffsetCommits());
+ assertTrue(addOffsetsResult.isDone()); // We should only be done after both RPCs complete.
+
+ transactionManager.beginCommittingTransaction();
+ prepareEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch);
+ sender.run(time.milliseconds()); // commit.
+
+ assertFalse(transactionManager.isInTransaction());
+ assertFalse(transactionManager.isCompletingTransaction());
+ assertFalse(transactionManager.transactionContainsPartition(tp0));
+ }
+
+ @Test
+ public void testDisconnectAndRetry() {
+ client.setNode(brokerNode);
+ // This is called from the initTransactions method in the producer as the first order of business.
+ // It finds the coordinator and then gets a PID.
+ transactionManager.initializeTransactions();
+ prepareFindCoordinatorResponse(Errors.NONE, true, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ sender.run(time.milliseconds()); // find coordinator, connection lost.
+
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ sender.run(time.milliseconds()); // find coordinator
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+ }
+
+ @Test
+ public void testCoordinatorLost() {
+ client.setNode(brokerNode);
+ // This is called from the initTransactions method in the producer as the first order of business.
+ // It finds the coordinator and then gets a PID.
+ final long pid = 13131L;
+ final short epoch = 1;
+ FutureTransactionalResult initPidResult = transactionManager.initializeTransactions();
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ sender.run(time.milliseconds()); // find coordinator
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+
+ prepareInitPidResponse(Errors.NOT_COORDINATOR, false, pid, epoch);
+ sender.run(time.milliseconds()); // send pid, get not coordinator. Should resend the FindCoordinator and InitPid requests
+
+ assertEquals(null, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+ assertFalse(initPidResult.isDone());
+ assertFalse(transactionManager.hasPid());
+
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ sender.run(time.milliseconds());
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+ assertFalse(initPidResult.isDone());
+ prepareInitPidResponse(Errors.NONE, false, pid, epoch);
+ sender.run(time.milliseconds()); // get pid and epoch
+
+ assertTrue(initPidResult.isDone()); // The future should only return after the second round of retries succeed.
+ assertTrue(transactionManager.hasPid());
+ assertEquals(pid, transactionManager.pidAndEpoch().producerId);
+ assertEquals(epoch, transactionManager.pidAndEpoch().epoch);
+ }
+
+ @Test
+ public void testFlushPendingPartitionsOnCommit() throws InterruptedException {
+ client.setNode(brokerNode);
+ // This is called from the initTransactions method in the producer as the first order of business.
+ // It finds the coordinator and then gets a PID.
+ final long pid = 13131L;
+ final short epoch = 1;
+ transactionManager.initializeTransactions();
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+
+ sender.run(time.milliseconds()); // find coordinator
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+
+ prepareInitPidResponse(Errors.NONE, false, pid, epoch);
+
+ sender.run(time.milliseconds()); // get pid.
+
+ assertTrue(transactionManager.hasPid());
+
+ transactionManager.beginTransaction();
+ transactionManager.maybeAddPartitionToTransaction(tp0);
+
+ Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
+ "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future;
+
+ assertFalse(responseFuture.isDone());
+
+ FutureTransactionalResult commitResult = transactionManager.beginCommittingTransaction();
+
+ // we have an append, an add partitions request, and now also an endtxn.
+ // The order should be:
+ // 1. Add Partitions
+ // 2. Produce
+ // 3. EndTxn.
+ assertFalse(transactionManager.transactionContainsPartition(tp0));
+ prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
+
+ sender.run(time.milliseconds()); // AddPartitions.
+ assertTrue(transactionManager.transactionContainsPartition(tp0));
+ assertFalse(responseFuture.isDone());
+ assertFalse(commitResult.isDone());
+
+ prepareProduceResponse(Errors.NONE, pid, epoch);
+ sender.run(time.milliseconds()); // Produce.
+ assertTrue(responseFuture.isDone());
+
+ prepareEndTxnResponse(Errors.NONE, TransactionResult.COMMIT, pid, epoch);
+ assertFalse(commitResult.isDone());
+ assertTrue(transactionManager.isInTransaction());
+ assertTrue(transactionManager.isCompletingTransaction());
+
+ sender.run(time.milliseconds());
+ assertTrue(commitResult.isDone());
+ assertFalse(transactionManager.isInTransaction());
+ }
+
+ @Test
+ public void testMultipleAddPartitionsPerForOneProduce() throws InterruptedException {
+ client.setNode(brokerNode);
+ // This is called from the initTransactions method in the producer as the first order of business.
+ // It finds the coordinator and then gets a PID.
+ final long pid = 13131L;
+ final short epoch = 1;
+ transactionManager.initializeTransactions();
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+
+ sender.run(time.milliseconds()); // find coordinator
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+
+ prepareInitPidResponse(Errors.NONE, false, pid, epoch);
+
+ sender.run(time.milliseconds()); // get pid.
+
+ assertTrue(transactionManager.hasPid());
+ transactionManager.beginTransaction();
+ // User does one producer.sed
+ transactionManager.maybeAddPartitionToTransaction(tp0);
+
+ Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
+ "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future;
+
+ assertFalse(responseFuture.isDone());
+ prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
+
+ assertFalse(transactionManager.transactionContainsPartition(tp0));
+
+ // Sender flushes one add partitions. The produce goes next.
+ sender.run(time.milliseconds()); // send addPartitions.
+ // Check that only addPartitions was sent.
+ assertTrue(transactionManager.transactionContainsPartition(tp0));
+
+ // In the mean time, the user does a second produce to a different partition
+ transactionManager.maybeAddPartitionToTransaction(tp1);
+ Future<RecordMetadata> secondResponseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
+ "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future;
+
+ prepareAddPartitionsToTxnResponse(Errors.NONE, tp1, epoch, pid);
+ prepareProduceResponse(Errors.NONE, pid, epoch);
+
+ assertFalse(transactionManager.transactionContainsPartition(tp1));
+
+ assertFalse(responseFuture.isDone());
+ assertFalse(secondResponseFuture.isDone());
+
+ // The second add partitionsh should go out here.
+ sender.run(time.milliseconds()); // send second add partitions request
+ assertTrue(transactionManager.transactionContainsPartition(tp1));
+
+ assertFalse(responseFuture.isDone());
+ assertFalse(secondResponseFuture.isDone());
+
+ // Finally we get to the produce.
+ sender.run(time.milliseconds()); // send produce request
+
+ assertTrue(responseFuture.isDone());
+ assertTrue(secondResponseFuture.isDone());
+ }
+
+ @Test(expected = ExecutionException.class)
+ public void testProducerFencedException() throws InterruptedException, ExecutionException {
+ client.setNode(brokerNode);
+ // This is called from the initTransactions method in the producer as the first order of business.
+ // It finds the coordinator and then gets a PID.
+ final long pid = 13131L;
+ final short epoch = 1;
+ transactionManager.initializeTransactions();
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+
+ sender.run(time.milliseconds()); // find coordinator
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+
+ prepareInitPidResponse(Errors.NONE, false, pid, epoch);
+
+ sender.run(time.milliseconds()); // get pid.
+
+ assertTrue(transactionManager.hasPid());
+ transactionManager.beginTransaction();
+ transactionManager.maybeAddPartitionToTransaction(tp0);
+
+ Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
+ "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future;
+
+ assertFalse(responseFuture.isDone());
+ prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
+ prepareProduceResponse(Errors.INVALID_PRODUCER_EPOCH, pid, epoch);
+ sender.run(time.milliseconds()); // Add partitions.
+
+ sender.run(time.milliseconds()); // send produce.
+
+ responseFuture.get();
+ }
+
+ @Test
+ public void testDisallowCommitOnProduceFailure() throws InterruptedException {
+ client.setNode(brokerNode);
+ // This is called from the initTransactions method in the producer as the first order of business.
+ // It finds the coordinator and then gets a PID.
+ final long pid = 13131L;
+ final short epoch = 1;
+ transactionManager.initializeTransactions();
+ prepareFindCoordinatorResponse(Errors.NONE, false, FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+
+ sender.run(time.milliseconds()); // find coordinator
+ assertEquals(brokerNode, transactionManager.coordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION));
+
+ prepareInitPidResponse(Errors.NONE, false, pid, epoch);
+
+ sender.run(time.milliseconds()); // get pid.
+
+ assertTrue(transactionManager.hasPid());
+ transactionManager.beginTransaction();
+ transactionManager.maybeAddPartitionToTransaction(tp0);
+
+ Future<RecordMetadata> responseFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(),
+ "value".getBytes(), new MockCallback(transactionManager), MAX_BLOCK_TIMEOUT).future;
+
+ FutureTransactionalResult commitResult = transactionManager.beginCommittingTransaction();
+ assertFalse(responseFuture.isDone());
+ prepareAddPartitionsToTxnResponse(Errors.NONE, tp0, epoch, pid);
+ prepareProduceResponse(Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, pid, epoch);
+
+ sender.run(time.milliseconds()); // Send AddPartitionsRequest
+ assertFalse(commitResult.isDone());
+
+ sender.run(time.milliseconds()); // Send Produce Request, returns OutOfOrderSequenceException.
+ sender.run(time.milliseconds()); // try to commit.
+ assertTrue(commitResult.isDone()); // commit should be cancelled with exception without being sent.
+
+ try {
+ commitResult.get();
+ fail(); // the get() must throw an exception.
+ } catch (RuntimeException e) {
+ assertTrue(e instanceof KafkaException);
+ }
+
+ // Commit is not allowed, so let's abort and try again.
+ FutureTransactionalResult abortResult = transactionManager.beginAbortingTransaction();
+ prepareEndTxnResponse(Errors.NONE, TransactionResult.ABORT, pid, epoch);
+ sender.run(time.milliseconds()); // Send abort request. It is valid to transition from ERROR to ABORT
+
+ assertTrue(abortResult.isDone());
+ assertTrue(abortResult.get().isSuccessful());
+ assertTrue(transactionManager.isReadyForTransaction()); // make sure we are ready for a transaction now.
+ }
+
+ private static class MockCallback implements Callback {
+ private final TransactionManager transactionManager;
+ public MockCallback(TransactionManager transactionManager) {
+ this.transactionManager = transactionManager;
+ }
+ @Override
+ public void onCompletion(RecordMetadata metadata, Exception exception) {
+ if (exception != null && transactionManager != null) {
+ transactionManager.maybeSetError(exception);
+ }
+ }
+ }
+
+ private void prepareFindCoordinatorResponse(Errors error, boolean shouldDisconnect,
+ final FindCoordinatorRequest.CoordinatorType coordinatorType,
+ final String coordinatorKey) {
+ client.prepareResponse(new MockClient.RequestMatcher() {
+ @Override
+ public boolean matches(AbstractRequest body) {
+ FindCoordinatorRequest findCoordinatorRequest = (FindCoordinatorRequest) body;
+ assertEquals(findCoordinatorRequest.coordinatorType(), coordinatorType);
+ assertEquals(findCoordinatorRequest.coordinatorKey(), coordinatorKey);
+ return true;
+ }
+ }, new FindCoordinatorResponse(error, brokerNode), shouldDisconnect);
+ }
+
+ private void prepareInitPidResponse(Errors error, boolean shouldDisconnect, long pid, short epoch) {
+ client.prepareResponse(new MockClient.RequestMatcher() {
+ @Override
+ public boolean matches(AbstractRequest body) {
+ InitPidRequest initPidRequest = (InitPidRequest) body;
+ assertEquals(initPidRequest.transactionalId(), transactionalId);
+ assertEquals(initPidRequest.transactionTimeoutMs(), transactionTimeoutMs);
+ return true;
+ }
+ }, new InitPidResponse(error, pid, epoch), shouldDisconnect);
+ }
+
+ private void prepareProduceResponse(Errors error, final long pid, final short epoch) {
+ client.prepareResponse(new MockClient.RequestMatcher() {
+ @Override
+ public boolean matches(AbstractRequest body) {
+ ProduceRequest produceRequest = (ProduceRequest) body;
+ MemoryRecords records = produceRequest.partitionRecordsOrFail().get(tp0);
+ assertNotNull(records);
+ Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
+ assertTrue(batchIterator.hasNext());
+ MutableRecordBatch batch = batchIterator.next();
+ assertFalse(batchIterator.hasNext());
+ assertTrue(batch.isTransactional());
+ assertEquals(pid, batch.producerId());
+ assertEquals(epoch, batch.producerEpoch());
+ assertEquals(transactionalId, produceRequest.transactionalId());
+ return true;
+ }
+ }, produceResponse(tp0, 0, error, 0));
+
+ }
+
+ private void prepareAddPartitionsToTxnResponse(Errors error, final TopicPartition topicPartition, final short epoch, final long pid) {
+ client.prepareResponse(new MockClient.RequestMatcher() {
+ @Override
+ public boolean matches(AbstractRequest body) {
+ AddPartitionsToTxnRequest addPartitionsToTxnRequest = (AddPartitionsToTxnRequest) body;
+ assertEquals(pid, addPartitionsToTxnRequest.producerId());
+ assertEquals(epoch, addPartitionsToTxnRequest.producerEpoch());
+ assertEquals(Arrays.asList(topicPartition), addPartitionsToTxnRequest.partitions());
+ assertEquals(transactionalId, addPartitionsToTxnRequest.transactionalId());
+ return true;
+ }
+ }, new AddPartitionsToTxnResponse(error));
+ }
+
+ private void prepareEndTxnResponse(Errors error, final TransactionResult result, final long pid, final short epoch) {
+ client.prepareResponse(new MockClient.RequestMatcher() {
+ @Override
+ public boolean matches(AbstractRequest body) {
+ EndTxnRequest endTxnRequest = (EndTxnRequest) body;
+ assertEquals(transactionalId, endTxnRequest.transactionalId());
+ assertEquals(pid, endTxnRequest.producerId());
+ assertEquals(epoch, endTxnRequest.producerEpoch());
+ assertEquals(result, endTxnRequest.command());
+ return true;
+ }
+ }, new EndTxnResponse(error));
+ }
+
+ private ProduceResponse produceResponse(TopicPartition tp, long offset, Errors error, int throttleTimeMs) {
+ ProduceResponse.PartitionResponse resp = new ProduceResponse.PartitionResponse(error, offset, RecordBatch.NO_TIMESTAMP);
+ Map<TopicPartition, ProduceResponse.PartitionResponse> partResp = Collections.singletonMap(tp, resp);
+ return new ProduceResponse(partResp, throttleTimeMs);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java
deleted file mode 100644
index a8a1716..0000000
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionStateTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.producer.internals;
-
-
-import org.apache.kafka.clients.producer.TransactionState;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.utils.MockTime;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class TransactionStateTest {
-
- private TopicPartition topicPartition;
-
- @Before
- public void setUp() {
- topicPartition = new TopicPartition("topic-0", 0);
- }
-
- @Test(expected = IllegalStateException.class)
- public void testInvalidSequenceIncrement() {
- TransactionState transactionState = new TransactionState(new MockTime());
- transactionState.incrementSequenceNumber(topicPartition, 3333);
- }
-
- @Test
- public void testDefaultSequenceNumber() {
- TransactionState transactionState = new TransactionState(new MockTime());
- assertEquals((int) transactionState.sequenceNumber(topicPartition), 0);
- transactionState.incrementSequenceNumber(topicPartition, 3);
- assertEquals((int) transactionState.sequenceNumber(topicPartition), 3);
- }
-
-
- @Test
- public void testProducerIdReset() {
- TransactionState transactionState = new TransactionState(new MockTime());
- assertEquals((int) transactionState.sequenceNumber(topicPartition), 0);
- transactionState.incrementSequenceNumber(topicPartition, 3);
- assertEquals((int) transactionState.sequenceNumber(topicPartition), 3);
- transactionState.resetProducerId();
- assertEquals((int) transactionState.sequenceNumber(topicPartition), 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
index 3cc8c20..330879f 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/MemoryRecordsBuilderTest.java
@@ -111,8 +111,9 @@ public class MemoryRecordsBuilderTest {
short epoch = 15;
int sequence = 2342;
- new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME,
+ MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME,
0L, 0L, pid, epoch, sequence, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity());
+ builder.close();
}
@Test(expected = IllegalArgumentException.class)
@@ -124,8 +125,9 @@ public class MemoryRecordsBuilderTest {
short epoch = RecordBatch.NO_PRODUCER_EPOCH;
int sequence = 2342;
- new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME,
+ MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME,
0L, 0L, pid, epoch, sequence, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity());
+ builder.close();
}
@Test(expected = IllegalArgumentException.class)
@@ -137,8 +139,9 @@ public class MemoryRecordsBuilderTest {
short epoch = 15;
int sequence = RecordBatch.NO_SEQUENCE;
- new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME,
+ MemoryRecordsBuilder builder = new MemoryRecordsBuilder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, compressionType, TimestampType.CREATE_TIME,
0L, 0L, pid, epoch, sequence, true, RecordBatch.NO_PARTITION_LEADER_EPOCH, buffer.capacity());
+ builder.close();
}
@Test
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/core/src/main/scala/kafka/log/LogValidator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/log/LogValidator.scala b/core/src/main/scala/kafka/log/LogValidator.scala
index 0616d41..c1777d5 100644
--- a/core/src/main/scala/kafka/log/LogValidator.scala
+++ b/core/src/main/scala/kafka/log/LogValidator.scala
@@ -107,7 +107,7 @@ private[kafka] object LogValidator extends Logging {
val newBuffer = ByteBuffer.allocate(sizeInBytesAfterConversion)
val builder = MemoryRecords.builder(newBuffer, toMagicValue, CompressionType.NONE, timestampType,
- offsetCounter.value, now, pid, epoch, sequence, partitionLeaderEpoch)
+ offsetCounter.value, now, pid, epoch, sequence, false, partitionLeaderEpoch)
for (batch <- records.batches.asScala) {
validateBatch(batch)
@@ -281,7 +281,7 @@ private[kafka] object LogValidator extends Logging {
val estimatedSize = AbstractRecords.estimateSizeInBytes(magic, offsetCounter.value, compressionType, validatedRecords.asJava)
val buffer = ByteBuffer.allocate(estimatedSize)
val builder = MemoryRecords.builder(buffer, magic, compressionType, timestampType, offsetCounter.value,
- logAppendTime, producerId, epoch, baseSequence, partitionLeaderEpoch)
+ logAppendTime, producerId, epoch, baseSequence, false, partitionLeaderEpoch)
validatedRecords.foreach { record =>
builder.appendWithOffset(offsetCounter.getAndIncrement(), record)
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/core/src/test/scala/unit/kafka/log/LogTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala
index 0b1d299..b5e376e 100755
--- a/core/src/test/scala/unit/kafka/log/LogTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogTest.scala
@@ -434,22 +434,22 @@ class LogTest {
val buffer = ByteBuffer.allocate(512)
- var builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 0L, time.milliseconds(), 1L, epoch, 0, 0)
+ var builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 0L, time.milliseconds(), 1L, epoch, 0, false, 0)
builder.append(new SimpleRecord("key".getBytes, "value".getBytes))
builder.close()
// Append a record with other pids.
- builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 1L, time.milliseconds(), 2L, epoch, 0, 0)
+ builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 1L, time.milliseconds(), 2L, epoch, 0, false, 0)
builder.append(new SimpleRecord("key".getBytes, "value".getBytes))
builder.close()
// Append a record with other pids.
- builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 2L, time.milliseconds(), 3L, epoch, 0, 0)
+ builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 2L, time.milliseconds(), 3L, epoch, 0, false, 0)
builder.append(new SimpleRecord("key".getBytes, "value".getBytes))
builder.close()
// Append a record with other pids.
- builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 3L, time.milliseconds(), 4L, epoch, 0, 0)
+ builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE, TimestampType.LOG_APPEND_TIME, 3L, time.milliseconds(), 4L, epoch, 0, false, 0)
builder.append(new SimpleRecord("key".getBytes, "value".getBytes))
builder.close()
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
index eefb35e..73e48af 100644
--- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
+++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala
@@ -26,7 +26,7 @@ import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord}
import org.apache.kafka.common.TopicPartition
import org.apache.kafka.common.protocol.{ApiKeys, Errors}
import org.apache.kafka.common.record.Record
-import org.apache.kafka.common.requests.{FetchRequest, FetchResponse}
+import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, IsolationLevel}
import org.apache.kafka.common.serialization.StringSerializer
import org.junit.Assert._
import org.junit.Test
@@ -156,8 +156,8 @@ class FetchRequestTest extends BaseRequestTest {
val (topicPartition, leaderId) = createTopics(numTopics = 1, numPartitions = 1).head
producer.send(new ProducerRecord(topicPartition.topic, topicPartition.partition,
"key", new String(new Array[Byte](maxPartitionBytes + 1)))).get
- val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0,
- createPartitionMap(maxPartitionBytes, Seq(topicPartition))).build(2)
+ val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(maxPartitionBytes,
+ Seq(topicPartition))).build(2)
val fetchResponse = sendFetchRequest(leaderId, fetchRequest, version = 2)
val partitionData = fetchResponse.responseData.get(topicPartition)
assertEquals(Errors.NONE, partitionData.error)
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/gradle/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/gradle/findbugs-exclude.xml b/gradle/findbugs-exclude.xml
index 4fc729e..eeea7fc 100644
--- a/gradle/findbugs-exclude.xml
+++ b/gradle/findbugs-exclude.xml
@@ -291,6 +291,15 @@ For a detailed description of findbugs bug categories, see http://findbugs.sourc
</Match>
<Match>
+ <!-- Ignore spurious warning about imbalanced synchronization on a cached Exception. Some accesses of the
+ variable must occur in synchronized blocks, while some need not, because of the nature of the producer.
+ This seems to throw the static checker off. -->
+ <Package name="org.apache.kafka.clients.producer" />
+ <Source name="TransactionState.java" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+
+ <Match>
<!-- Ignore a spurious warning about the types used in
AdminClient#deleteRecordsBefore -->
<Package name="kafka.admin"/>
[3/3] kafka git commit: KAFKA-4818; Exactly once transactional clients
Posted by jg...@apache.org.
KAFKA-4818; Exactly once transactional clients
Author: Apurva Mehta <ap...@confluent.io>
Reviewers: Guozhang Wang <wa...@gmail.com>, Ismael Juma <is...@juma.me.uk>, Jason Gustafson <ja...@confluent.io>
Closes #2840 from apurvam/exactly-once-transactional-clients
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/a82f194b
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/a82f194b
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/a82f194b
Branch: refs/heads/trunk
Commit: a82f194b21a6af2f52e36e55e2c6adcdba942c08
Parents: bb663d0
Author: Apurva Mehta <ap...@confluent.io>
Authored: Thu Apr 27 14:11:17 2017 -0700
Committer: Jason Gustafson <ja...@confluent.io>
Committed: Thu Apr 27 14:11:36 2017 -0700
----------------------------------------------------------------------
checkstyle/import-control.xml | 1 +
checkstyle/suppressions.xml | 44 +-
.../kafka/clients/consumer/ConsumerConfig.java | 20 +
.../kafka/clients/consumer/KafkaConsumer.java | 8 +-
.../clients/consumer/internals/Fetcher.java | 94 +-
.../kafka/clients/producer/KafkaProducer.java | 169 +++-
.../kafka/clients/producer/MockProducer.java | 23 +
.../apache/kafka/clients/producer/Producer.java | 30 +-
.../kafka/clients/producer/ProducerConfig.java | 25 +-
.../clients/producer/TransactionState.java | 135 ---
.../internals/FutureTransactionalResult.java | 64 ++
.../producer/internals/ProducerBatch.java | 7 +-
.../producer/internals/RecordAccumulator.java | 32 +-
.../clients/producer/internals/Sender.java | 134 ++-
.../producer/internals/TransactionManager.java | 866 +++++++++++++++++++
.../internals/TransactionalRequestResult.java | 64 ++
.../apache/kafka/common/config/ConfigDef.java | 10 +
.../kafka/common/record/MemoryRecords.java | 32 +-
.../common/record/MemoryRecordsBuilder.java | 31 +-
.../requests/AddOffsetsToTxnResponse.java | 1 +
.../requests/AddPartitionsToTxnResponse.java | 1 +
.../kafka/common/requests/EndTxnResponse.java | 1 +
.../kafka/common/requests/FetchRequest.java | 21 +-
.../kafka/common/requests/ProduceRequest.java | 22 +-
.../clients/consumer/KafkaConsumerTest.java | 7 +-
.../clients/consumer/internals/FetcherTest.java | 313 ++++++-
.../internals/RecordAccumulatorTest.java | 3 +-
.../clients/producer/internals/SenderTest.java | 49 +-
.../internals/TransactionManagerTest.java | 588 +++++++++++++
.../internals/TransactionStateTest.java | 61 --
.../common/record/MemoryRecordsBuilderTest.java | 9 +-
.../src/main/scala/kafka/log/LogValidator.scala | 4 +-
.../src/test/scala/unit/kafka/log/LogTest.scala | 8 +-
.../unit/kafka/server/FetchRequestTest.scala | 6 +-
gradle/findbugs-exclude.xml | 9 +
35 files changed, 2515 insertions(+), 377 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/checkstyle/import-control.xml
----------------------------------------------------------------------
diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml
index 3475062..a6de9a7 100644
--- a/checkstyle/import-control.xml
+++ b/checkstyle/import-control.xml
@@ -124,6 +124,7 @@
</subpackage>
<subpackage name="producer">
+ <allow pkg="org.apache.kafka.clients.consumer" />
<allow pkg="org.apache.kafka.clients.producer" />
</subpackage>
</subpackage>
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/checkstyle/suppressions.xml
----------------------------------------------------------------------
diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml
index 042be6b..eae8dde 100644
--- a/checkstyle/suppressions.xml
+++ b/checkstyle/suppressions.xml
@@ -8,7 +8,7 @@
<!-- Clients -->
<suppress checks="ClassFanOutComplexity"
- files="(Fetcher|ConsumerCoordinator|KafkaConsumer|KafkaProducer|SaslServerAuthenticator|Utils).java"/>
+ files="(Fetcher|ConsumerCoordinator|KafkaConsumer|KafkaProducer|SaslServerAuthenticator|Utils|TransactionManagerTest).java"/>
<suppress checks="ClassFanOutComplexity"
files=".*/protocol/Errors.java"/>
<suppress checks="ClassFanOutComplexity"
@@ -35,7 +35,7 @@
files="DefaultRecordBatch.java"/>
<suppress checks="ClassDataAbstractionCoupling"
- files="(KafkaConsumer|ConsumerCoordinator|Fetcher|KafkaProducer|AbstractRequest|AbstractResponse).java"/>
+ files="(KafkaConsumer|ConsumerCoordinator|Fetcher|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager).java"/>
<suppress checks="ClassDataAbstractionCoupling"
files=".*/protocol/Errors.java"/>
@@ -43,51 +43,17 @@
files="(Utils|KafkaLZ4BlockOutputStream).java"/>
<suppress checks="CyclomaticComplexity"
- files="ConsumerCoordinator.java"/>
- <suppress checks="CyclomaticComplexity"
- files="Fetcher.java"/>
- <suppress checks="CyclomaticComplexity"
- files="KafkaProducer.java"/>
- <suppress checks="CyclomaticComplexity"
- files="BufferPool.java"/>
- <suppress checks="CyclomaticComplexity"
- files="RecordAccumulator.java"/>
- <suppress checks="CyclomaticComplexity"
- files="ConfigDef.java"/>
- <suppress checks="CyclomaticComplexity"
- files="Selector.java"/>
- <suppress checks="CyclomaticComplexity"
- files="SslTransportLayer.java"/>
- <suppress checks="CyclomaticComplexity"
- files="KerberosLogin.java"/>
- <suppress checks="CyclomaticComplexity"
- files="AbstractRequest.java"/>
- <suppress checks="CyclomaticComplexity"
- files="AbstractResponse.java"/>
+ files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|SsLTransportLayer|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslTransportLayer).java"/>
<suppress checks="JavaNCSS"
files="KerberosLogin.java"/>
<suppress checks="NPathComplexity"
- files="BufferPool.java"/>
- <suppress checks="NPathComplexity"
- files="MetricName.java"/>
- <suppress checks="NPathComplexity"
- files="Node.java"/>
- <suppress checks="NPathComplexity"
- files="ConfigDef.java"/>
- <suppress checks="NPathComplexity"
- files="SslTransportLayer.java"/>
- <suppress checks="NPathComplexity"
- files="MetadataResponse.java"/>
- <suppress checks="NPathComplexity"
- files="KerberosLogin.java"/>
- <suppress checks="NPathComplexity"
- files="SslTransportLayer.java"/>
+ files="(BufferPool|MetricName|Node|ConfigDef|SslTransportLayer|MetadataResponse|KerberosLogin|SslTransportLayer|Sender).java"/>
<!-- clients tests -->
<suppress checks="ClassDataAbstractionCoupling"
- files="(Sender|Fetcher|KafkaConsumer|Metrics|ConsumerCoordinator|RequestResponse)Test.java"/>
+ files="(Sender|Fetcher|KafkaConsumer|Metrics|ConsumerCoordinator|RequestResponse|TransactionManager)Test.java"/>
<suppress checks="ClassFanOutComplexity"
files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher)Test.java"/>
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
index abb8255..0c0c29d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java
@@ -22,10 +22,12 @@ import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigDef.Importance;
import org.apache.kafka.common.config.ConfigDef.Type;
import org.apache.kafka.common.metrics.Sensor;
+import org.apache.kafka.common.requests.IsolationLevel;
import org.apache.kafka.common.serialization.Deserializer;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
@@ -234,6 +236,18 @@ public class ConsumerConfig extends AbstractConfig {
*/
static final String LEAVE_GROUP_ON_CLOSE_CONFIG = "internal.leave.group.on.close";
+ /** <code>isolation.level</code> */
+ public static final String ISOLATION_LEVEL_CONFIG = "isolation.level";
+ public static final String ISOLATION_LEVEL_DOC = "<p>Controls how to read messages written transactionally. If set to <code>read_committed</code>, consumer.poll() will only return" +
+ " transactional messages which have been committed. If set to <code>read_uncommitted</code>' (the default), consumer.poll() will return all messages, even transactional messages" +
+ " which have been aborted. Non-transactional messages will be returned unconditionally in either mode.</p> <p>Messages will always be returned in offset order. Hence, in " +
+ " <code>read_committed</code> mode, consumer.poll() will only return messages up to the last stable offset (LSO), which is the one less than the offset of the first open transaction." +
+ " In particular any messages appearing after messages belonging to ongoing transactions will be withheld until the relevant transaction has been completed. As a result, <code>read_committed</code>" +
+ " consumers will not be able to read up to the high watermark when there are in flight transactions.</p><p> Further, when in <code>read_committed</mode> the seekToEnd method will" +
+ " return the LSO";
+
+ public static final String DEFAULT_ISOLATION_LEVEL = IsolationLevel.READ_UNCOMMITTED.toString().toLowerCase(Locale.ROOT);
+
static {
CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG,
Type.LIST,
@@ -405,6 +419,12 @@ public class ConsumerConfig extends AbstractConfig {
Type.BOOLEAN,
true,
Importance.LOW)
+ .define(ISOLATION_LEVEL_CONFIG,
+ Type.STRING,
+ DEFAULT_ISOLATION_LEVEL,
+ in(IsolationLevel.READ_COMMITTED.toString().toLowerCase(Locale.ROOT), IsolationLevel.READ_UNCOMMITTED.toString().toLowerCase(Locale.ROOT)),
+ Importance.MEDIUM,
+ ISOLATION_LEVEL_DOC)
// security support
.define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG,
Type.STRING,
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 9c703b5..da5b7fb 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -43,6 +43,7 @@ import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.metrics.MetricsReporter;
import org.apache.kafka.common.network.ChannelBuilder;
import org.apache.kafka.common.network.Selector;
+import org.apache.kafka.common.requests.IsolationLevel;
import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.utils.AppInfoParser;
@@ -660,6 +661,10 @@ public class KafkaConsumer<K, V> implements Consumer<K, V> {
this.metadata.update(Cluster.bootstrap(addresses), Collections.<String>emptySet(), 0);
String metricGrpPrefix = "consumer";
ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config);
+
+ IsolationLevel isolationLevel = IsolationLevel.valueOf(
+ config.getString(ConsumerConfig.ISOLATION_LEVEL_CONFIG).toUpperCase(Locale.ROOT));
+
NetworkClient netClient = new NetworkClient(
new Selector(config.getLong(ConsumerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder),
this.metadata,
@@ -710,7 +715,8 @@ public class KafkaConsumer<K, V> implements Consumer<K, V> {
metrics,
metricGrpPrefix,
this.time,
- this.retryBackoffMs);
+ this.retryBackoffMs,
+ isolationLevel);
config.logUnused();
AppInfoParser.registerAppInfo(JMX_PREFIX, clientId);
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
index f421dfb..947214f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java
@@ -29,6 +29,7 @@ import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.CorruptRecordException;
import org.apache.kafka.common.errors.InvalidMetadataException;
import org.apache.kafka.common.errors.InvalidTopicException;
import org.apache.kafka.common.errors.RecordTooLargeException;
@@ -44,12 +45,14 @@ import org.apache.kafka.common.metrics.stats.Max;
import org.apache.kafka.common.metrics.stats.Rate;
import org.apache.kafka.common.metrics.stats.Value;
import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.record.ControlRecordType;
import org.apache.kafka.common.record.InvalidRecordException;
import org.apache.kafka.common.record.Record;
import org.apache.kafka.common.record.RecordBatch;
import org.apache.kafka.common.record.TimestampType;
import org.apache.kafka.common.requests.FetchRequest;
import org.apache.kafka.common.requests.FetchResponse;
+import org.apache.kafka.common.requests.IsolationLevel;
import org.apache.kafka.common.requests.ListOffsetRequest;
import org.apache.kafka.common.requests.ListOffsetResponse;
import org.apache.kafka.common.requests.MetadataRequest;
@@ -66,16 +69,20 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.PriorityQueue;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicInteger;
+import static java.util.Collections.emptyList;
+
/**
* This class manage the fetching process with the brokers.
*/
@@ -98,6 +105,8 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
private final ConcurrentLinkedQueue<CompletedFetch> completedFetches;
private final Deserializer<K> keyDeserializer;
private final Deserializer<V> valueDeserializer;
+ private final IsolationLevel isolationLevel;
+
private PartitionRecords nextInLineRecords = null;
private ExceptionMetadata nextInLineExceptionMetadata = null;
@@ -115,7 +124,8 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
Metrics metrics,
String metricGrpPrefix,
Time time,
- long retryBackoffMs) {
+ long retryBackoffMs,
+ IsolationLevel isolationLevel) {
this.time = time;
this.client = client;
this.metadata = metadata;
@@ -131,6 +141,7 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
this.completedFetches = new ConcurrentLinkedQueue<>();
this.sensors = new FetchManagerMetrics(metrics, metricGrpPrefix);
this.retryBackoffMs = retryBackoffMs;
+ this.isolationLevel = isolationLevel;
subscriptions.addListener(this);
}
@@ -370,7 +381,7 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
if (strategy == OffsetResetStrategy.EARLIEST)
timestamp = ListOffsetRequest.EARLIEST_TIMESTAMP;
else if (strategy == OffsetResetStrategy.LATEST)
- timestamp = ListOffsetRequest.LATEST_TIMESTAMP;
+ timestamp = endTimestamp();
else
throw new NoOffsetForPartitionException(partition);
return timestamp;
@@ -457,7 +468,11 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
}
public Map<TopicPartition, Long> endOffsets(Collection<TopicPartition> partitions, long timeout) {
- return beginningOrEndOffset(partitions, ListOffsetRequest.LATEST_TIMESTAMP, timeout);
+ return beginningOrEndOffset(partitions, endTimestamp(), timeout);
+ }
+
+ private long endTimestamp() {
+ return ListOffsetRequest.LATEST_TIMESTAMP;
}
private Map<TopicPartition, Long> beginningOrEndOffset(Collection<TopicPartition> partitions,
@@ -570,7 +585,7 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
}
partitionRecords.drain();
- return Collections.emptyList();
+ return emptyList();
}
/**
@@ -771,7 +786,7 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
Map<Node, FetchRequest.Builder> requests = new HashMap<>();
for (Map.Entry<Node, LinkedHashMap<TopicPartition, FetchRequest.PartitionData>> entry : fetchable.entrySet()) {
Node node = entry.getKey();
- FetchRequest.Builder fetch = FetchRequest.Builder.forConsumer(this.maxWaitMs, this.minBytes, entry.getValue()).
+ FetchRequest.Builder fetch = FetchRequest.Builder.forConsumer(this.maxWaitMs, this.minBytes, entry.getValue(), isolationLevel).
setMaxBytes(this.maxBytes);
requests.put(node, fetch);
}
@@ -906,6 +921,8 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
private final TopicPartition partition;
private final CompletedFetch completedFetch;
private final Iterator<? extends RecordBatch> batches;
+ private final Set<Long> abortedProducerIds;
+ private final PriorityQueue<FetchResponse.AbortedTransaction> abortedTransactions;
private int recordsRead;
private int bytesRead;
@@ -921,6 +938,8 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
this.completedFetch = completedFetch;
this.batches = batches;
this.nextFetchOffset = completedFetch.fetchedOffset;
+ this.abortedProducerIds = new HashSet<>();
+ this.abortedTransactions = abortedTransactions(completedFetch.partitionData);
}
private void drain() {
@@ -960,8 +979,10 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
}
private void maybeCloseRecordStream() {
- if (records != null)
+ if (records != null) {
records.close();
+ records = null;
+ }
}
private Record nextFetchedRecord() {
@@ -974,7 +995,14 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
return null;
}
currentBatch = batches.next();
+
maybeEnsureValid(currentBatch);
+
+ if (isolationLevel == IsolationLevel.READ_COMMITTED && isBatchAborted(currentBatch)) {
+ nextFetchOffset = currentBatch.lastOffset() + 1;
+ continue;
+ }
+
records = currentBatch.streamingIterator();
}
@@ -1008,6 +1036,60 @@ public class Fetcher<K, V> implements SubscriptionState.Listener, Closeable {
}
return records;
}
+
+ private boolean isBatchAborted(RecordBatch batch) {
+ /* When in READ_COMMITTED mode, we need to do the following for each incoming entry:
+ * 0. Check whether the pid is in the 'abortedProducerIds' set && the entry does not include an abort marker.
+ * If so, skip the entry.
+ * 1. If the pid is in aborted pids and the entry contains an abort marker, remove the pid from
+ * aborted pids and skip the entry.
+ * 2. Check lowest offset entry in the abort index. If the PID of the current entry matches the
+ * pid of the abort index entry, and the incoming offset is no smaller than the abort index offset,
+ * this means that the entry has been aborted. Add the pid to the aborted pids set, and remove
+ * the entry from the abort index.
+ */
+ FetchResponse.AbortedTransaction nextAbortedTransaction = abortedTransactions.peek();
+ if (abortedProducerIds.contains(batch.producerId())
+ || (nextAbortedTransaction != null && nextAbortedTransaction.producerId == batch.producerId() && nextAbortedTransaction.firstOffset <= batch.baseOffset())) {
+ if (abortedProducerIds.contains(batch.producerId()) && containsAbortMarker(batch)) {
+ abortedProducerIds.remove(batch.producerId());
+ } else if (nextAbortedTransaction != null && nextAbortedTransaction.producerId == batch.producerId() && nextAbortedTransaction.firstOffset <= batch.baseOffset()) {
+ abortedProducerIds.add(batch.producerId());
+ abortedTransactions.poll();
+ }
+ log.trace("Skipping aborted record batch with producerId {} and base offset {}, partition: {}", batch.producerId(), batch.baseOffset(), partition);
+ return true;
+ }
+ return false;
+ }
+
+ private PriorityQueue<FetchResponse.AbortedTransaction> abortedTransactions(FetchResponse.PartitionData partition) {
+ PriorityQueue<FetchResponse.AbortedTransaction> abortedTransactions = null;
+ if (partition.abortedTransactions != null && !partition.abortedTransactions.isEmpty()) {
+ abortedTransactions = new PriorityQueue<>(
+ partition.abortedTransactions.size(),
+ new Comparator<FetchResponse.AbortedTransaction>() {
+ @Override
+ public int compare(FetchResponse.AbortedTransaction o1, FetchResponse.AbortedTransaction o2) {
+ return Long.compare(o1.firstOffset, o2.firstOffset);
+ }
+ }
+ );
+ abortedTransactions.addAll(partition.abortedTransactions);
+ } else {
+ abortedTransactions = new PriorityQueue<>();
+ }
+ return abortedTransactions;
+ }
+
+ private boolean containsAbortMarker(RecordBatch batch) {
+ Iterator<Record> batchIterator = batch.iterator();
+ Record firstRecord = batchIterator.hasNext() ? batchIterator.next() : null;
+ boolean containsAbortMarker = firstRecord != null && firstRecord.isControlRecord() && ControlRecordType.ABORT == ControlRecordType.parse(firstRecord.key());
+ if (containsAbortMarker && batchIterator.hasNext())
+ throw new CorruptRecordException("A record batch containing a control message contained more than one record. partition: " + partition + ", offset: " + batch.baseOffset());
+ return containsAbortMarker;
+ }
}
private static class ExceptionMetadata {
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
index 8fe99dd..3745aba 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/KafkaProducer.java
@@ -20,9 +20,12 @@ import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.ClientUtils;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.NetworkClient;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.producer.internals.FutureTransactionalResult;
import org.apache.kafka.clients.producer.internals.ProducerInterceptors;
import org.apache.kafka.clients.producer.internals.RecordAccumulator;
import org.apache.kafka.clients.producer.internals.Sender;
+import org.apache.kafka.clients.producer.internals.TransactionManager;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.Metric;
@@ -32,6 +35,7 @@ import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.config.ConfigException;
import org.apache.kafka.common.errors.ApiException;
import org.apache.kafka.common.errors.InterruptException;
+import org.apache.kafka.common.errors.ProducerFencedException;
import org.apache.kafka.common.errors.RecordTooLargeException;
import org.apache.kafka.common.errors.SerializationException;
import org.apache.kafka.common.errors.TimeoutException;
@@ -159,7 +163,7 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
private final int requestTimeoutMs;
private final ProducerInterceptors<K, V> interceptors;
private final ApiVersions apiVersions;
- private final TransactionState transactionState;
+ private final TransactionManager transactionManager;
/**
* A producer is instantiated by providing a set of key-value pairs as configuration. Valid configuration strings
@@ -263,10 +267,10 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
this.maxBlockTimeMs = configureMaxBlockTime(config, userProvidedConfigs);
this.requestTimeoutMs = configureRequestTimeout(config, userProvidedConfigs);
- this.transactionState = configureTransactionState(config, time);
- int retries = configureRetries(config, transactionState != null);
- int maxInflightRequests = configureInflightRequests(config, transactionState != null);
- short acks = configureAcks(config, transactionState != null);
+ this.transactionManager = configureTransactionState(config);
+ int retries = configureRetries(config, transactionManager != null);
+ int maxInflightRequests = configureInflightRequests(config, transactionManager != null);
+ short acks = configureAcks(config, transactionManager != null);
this.apiVersions = new ApiVersions();
this.accumulator = new RecordAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG),
@@ -277,7 +281,7 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
metrics,
time,
apiVersions,
- transactionState);
+ transactionManager);
List<InetSocketAddress> addresses = ClientUtils.parseAndValidateAddresses(config.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG));
this.metadata.update(Cluster.bootstrap(addresses), Collections.<String>emptySet(), time.milliseconds());
ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config);
@@ -305,7 +309,7 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
Time.SYSTEM,
this.requestTimeoutMs,
config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG),
- this.transactionState,
+ this.transactionManager,
apiVersions);
String ioThreadName = "kafka-producer-network-thread" + (clientId.length() > 0 ? " | " + clientId : "");
this.ioThread = new KafkaThread(ioThreadName, this.sender, true);
@@ -363,13 +367,37 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
}
}
- private static TransactionState configureTransactionState(ProducerConfig config, Time time) {
+ private static TransactionManager configureTransactionState(ProducerConfig config) {
+
+ TransactionManager transactionManager = null;
+
+ boolean userConfiguredIdempotence = false;
+ if (config.originals().containsKey(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG))
+ userConfiguredIdempotence = true;
+
+ boolean userConfiguredTransactions = false;
+ if (config.originals().containsKey(ProducerConfig.TRANSACTIONAL_ID_CONFIG))
+ userConfiguredTransactions = true;
+
boolean idempotenceEnabled = config.getBoolean(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG);
+
+ if (!idempotenceEnabled && userConfiguredIdempotence && userConfiguredTransactions)
+ throw new ConfigException("Cannot set a " + ProducerConfig.TRANSACTIONAL_ID_CONFIG + " without also enabling idempotence.");
+
+ if (userConfiguredTransactions)
+ idempotenceEnabled = true;
+
if (idempotenceEnabled) {
- return new TransactionState(time);
- } else {
- return null;
+ String transactionalId = config.getString(ProducerConfig.TRANSACTIONAL_ID_CONFIG);
+ int transactionTimeoutMs = config.getInt(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG);
+ transactionManager = new TransactionManager(transactionalId, transactionTimeoutMs);
+ if (transactionManager.isTransactional())
+ log.info("Instantiated a transactional producer.");
+ else
+ log.info("Instantiated an idempotent producer.");
}
+
+ return transactionManager;
}
private static int configureRetries(ProducerConfig config, boolean idempotenceEnabled) {
@@ -431,6 +459,87 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
}
/**
+ * Needs to be called before any other methods when the transactional.id is set in the configuration.
+ *
+ * This method does the following:
+ * 1. Ensures any transactions initiated by previous instances of the producer
+ * are completed. If the previous instance had failed with a transaction in
+ * progress, it will be aborted. If the last transaction had begun completion,
+ * but not yet finished, this method awaits its completion.
+ * 2. Gets the internal producer id and epoch, used in all future transactional
+ * messages issued by the producer.
+ *
+ * @throws IllegalStateException if the TransactionalId for the producer is not set
+ * in the configuration.
+ */
+ public void initTransactions() {
+ if (transactionManager == null)
+ throw new IllegalStateException("Cannot call initTransactions without setting a transactional id.");
+ transactionManager.initializeTransactions().get();
+ }
+
+ /**
+ * Should be called before the start of each new transaction.
+ *
+ * @throws ProducerFencedException if another producer is with the same
+ * transactional.id is active.
+ */
+ public void beginTransaction() throws ProducerFencedException {
+ // Set the transactional bit in the producer.
+ if (transactionManager == null)
+ throw new IllegalStateException("Cannot use transactional methods without enabling transactions");
+ transactionManager.beginTransaction();
+ }
+
+ /**
+ * Sends a list of consumed offsets to the consumer group coordinator, and also marks
+ * those offsets as part of the current transaction. These offsets will be considered
+ * consumed only if the transaction is committed successfully.
+ *
+ * This method should be used when you need to batch consumed and produced messages
+ * together, typically in a consume-transform-produce pattern.
+ *
+ * @throws ProducerFencedException if another producer with the same
+ * transactional.id is active.
+ */
+ public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets,
+ String consumerGroupId) throws ProducerFencedException {
+ if (transactionManager == null)
+ throw new IllegalStateException("Cannot send offsets to transaction since transactions are not enabled.");
+ FutureTransactionalResult result = transactionManager.sendOffsetsToTransaction(offsets, consumerGroupId);
+ sender.wakeup();
+ result.get();
+ }
+
+ /**
+ * Commits the ongoing transaction.
+ *
+ * @throws ProducerFencedException if another producer with the same
+ * transactional.id is active.
+ */
+ public void commitTransaction() throws ProducerFencedException {
+ if (transactionManager == null)
+ throw new IllegalStateException("Cannot commit transaction since transactions are not enabled");
+ FutureTransactionalResult result = transactionManager.beginCommittingTransaction();
+ sender.wakeup();
+ result.get();
+ }
+
+ /**
+ * Aborts the ongoing transaction.
+ *
+ * @throws ProducerFencedException if another producer with the same
+ * transactional.id is active.
+ */
+ public void abortTransaction() throws ProducerFencedException {
+ if (transactionManager == null)
+ throw new IllegalStateException("Cannot abort transaction since transactions are not enabled.");
+ FutureTransactionalResult result = transactionManager.beginAbortingTransaction();
+ sender.wakeup();
+ result.get();
+ }
+
+ /**
* Asynchronously send a record to a topic. Equivalent to <code>send(record, null)</code>.
* See {@link #send(ProducerRecord, Callback)} for details.
*/
@@ -523,6 +632,7 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
* Implementation of asynchronously send a record to a topic.
*/
private Future<RecordMetadata> doSend(ProducerRecord<K, V> record, Callback callback) {
+ ensureProperTransactionalState();
TopicPartition tp = null;
try {
// first make sure the metadata for the topic is available
@@ -554,7 +664,11 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
long timestamp = record.timestamp() == null ? time.milliseconds() : record.timestamp();
log.trace("Sending record {} with callback {} to topic {} partition {}", record, callback, record.topic(), partition);
// producer callback will make sure to call both 'callback' and interceptor callback
- Callback interceptCallback = this.interceptors == null ? callback : new InterceptorCallback<>(callback, this.interceptors, tp);
+ Callback interceptCallback = new InterceptorCallback<>(callback, this.interceptors, tp, transactionManager);
+
+ if (transactionManager != null)
+ transactionManager.maybeAddPartitionToTransaction(tp);
+
RecordAccumulator.RecordAppendResult result = accumulator.append(tp, timestamp, serializedKey,
serializedValue, interceptCallback, remainingWaitMs);
if (result.batchIsFull || result.newBatchCreated) {
@@ -597,6 +711,30 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
}
}
+ private void ensureProperTransactionalState() {
+ if (transactionManager == null)
+ return;
+
+ if (transactionManager.isTransactional() && !transactionManager.hasPid())
+ throw new IllegalStateException("Cannot perform a 'send' before completing a call to initTransactions when transactions are enabled.");
+
+ if (transactionManager.isFenced())
+ throw new ProducerFencedException("The current producer has been fenced off by a another producer using the same transactional id.");
+
+ if (transactionManager.isInTransaction()) {
+ if (transactionManager.isInErrorState()) {
+ String errorMessage = "Cannot perform a transactional send because at least one previous transactional request has failed with errors.";
+ Exception lastError = transactionManager.lastError();
+ if (lastError != null)
+ throw new KafkaException(errorMessage, lastError);
+ else
+ throw new KafkaException(errorMessage);
+ }
+ if (transactionManager.isCompletingTransaction())
+ throw new IllegalStateException("Cannot call send while a commit or abort is in progress.");
+ }
+ }
+
/**
* Wait for cluster metadata including partitions for the given topic to be available.
* @param topic The topic we want metadata for
@@ -890,12 +1028,14 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
private final Callback userCallback;
private final ProducerInterceptors<K, V> interceptors;
private final TopicPartition tp;
+ private final TransactionManager transactionManager;
public InterceptorCallback(Callback userCallback, ProducerInterceptors<K, V> interceptors,
- TopicPartition tp) {
+ TopicPartition tp, TransactionManager transactionManager) {
this.userCallback = userCallback;
this.interceptors = interceptors;
this.tp = tp;
+ this.transactionManager = transactionManager;
}
public void onCompletion(RecordMetadata metadata, Exception exception) {
@@ -909,6 +1049,9 @@ public class KafkaProducer<K, V> implements Producer<K, V> {
}
if (this.userCallback != null)
this.userCallback.onCompletion(metadata, exception);
+
+ if (exception != null && transactionManager != null)
+ transactionManager.maybeSetError(exception);
}
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
index ab09997..80ea372 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/MockProducer.java
@@ -16,6 +16,7 @@
*/
package org.apache.kafka.clients.producer;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
import org.apache.kafka.clients.producer.internals.FutureRecordMetadata;
import org.apache.kafka.clients.producer.internals.ProduceRequestResult;
@@ -24,6 +25,7 @@ import org.apache.kafka.common.Metric;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.ProducerFencedException;
import org.apache.kafka.common.record.RecordBatch;
import org.apache.kafka.common.serialization.Serializer;
@@ -95,6 +97,27 @@ public class MockProducer<K, V> implements Producer<K, V> {
this(Cluster.empty(), autoComplete, partitioner, keySerializer, valueSerializer);
}
+ public void initTransactions() {
+
+ }
+
+ public void beginTransaction() throws ProducerFencedException {
+
+ }
+
+ public void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets,
+ String consumerGroupId) throws ProducerFencedException {
+
+ }
+
+ public void commitTransaction() throws ProducerFencedException {
+
+ }
+
+ public void abortTransaction() throws ProducerFencedException {
+
+ }
+
/**
* Adds the record to the list of sent records. The {@link RecordMetadata} returned will be immediately satisfied.
*
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java
index 4da8681..1e77633 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/Producer.java
@@ -19,6 +19,9 @@ package org.apache.kafka.clients.producer;
import org.apache.kafka.common.Metric;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.PartitionInfo;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.ProducerFencedException;
import java.io.Closeable;
import java.util.List;
@@ -26,7 +29,6 @@ import java.util.Map;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-
/**
* The interface for the {@link KafkaProducer}
* @see KafkaProducer
@@ -35,6 +37,32 @@ import java.util.concurrent.TimeUnit;
public interface Producer<K, V> extends Closeable {
/**
+ * See {@link KafkaProducer#initTransactions()}
+ */
+ void initTransactions();
+
+ /**
+ * See {@link KafkaProducer#beginTransaction()}
+ */
+ void beginTransaction() throws ProducerFencedException;
+
+ /**
+ * See {@link KafkaProducer#sendOffsetsToTransaction(Map, String)}
+ */
+ void sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets,
+ String consumerGroupId) throws ProducerFencedException;
+
+ /**
+ * See {@link KafkaProducer#commitTransaction()}
+ */
+ void commitTransaction() throws ProducerFencedException;
+
+ /**
+ * See {@link KafkaProducer#abortTransaction()}
+ */
+ void abortTransaction() throws ProducerFencedException;
+
+ /**
* Send the given record asynchronously and return a future which will eventually contain the response information.
*
* @param record The record to send
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
index d6e03d2..4bceb95 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/ProducerConfig.java
@@ -232,6 +232,18 @@ public class ProducerConfig extends AbstractConfig {
+ "<code>" + RETRIES_CONFIG + "</code> cannot be zero. Additionally " + ACKS_CONFIG + " must be set to 'all'. If these values "
+ "are left at their defaults, we will override the default to be suitable. "
+ "If the values are set to something incompatible with the idempotent producer, a ConfigException will be thrown.";
+
+ /** <code> transaction.timeout.ms </code> */
+ public static final String TRANSACTION_TIMEOUT_CONFIG = "transaction.timeout.ms";
+ public static final String TRANSACTION_TIMEOUT_DOC = "The maximum amount of time in ms that the transaction coordinator will wait for a transaction status update from the producer before proactively aborting the ongoing transaction." +
+ "If this value is larger than the max.transaction.timeout.ms setting in the broker, the request will fail with a `InvalidTransactionTimeout` error.";
+
+ /** <code> transactional.id </code> */
+ public static final String TRANSACTIONAL_ID_CONFIG = "transactional.id";
+ public static final String TRANSACTIONAL_ID_DOC = "The TransactionalId to use for transactional delivery. This enables reliability semantics which span multiple producer sessions since it allows the client to guarantee that transactions using the same TransactionalId have been completed prior to starting any new transactions. If no TransactionalId is provided, then the producer is limited to idempotent delivery. " +
+ "Note that enable.idempotence must be enabled if a TransactionalId is configured. " +
+ "The default is empty, which means transactions cannot be used.";
+
static {
CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, Importance.HIGH, CommonClientConfigs.BOOTSTRAP_SERVERS_DOC)
.define(BUFFER_MEMORY_CONFIG, Type.LONG, 32 * 1024 * 1024L, atLeast(0L), Importance.HIGH, BUFFER_MEMORY_DOC)
@@ -325,7 +337,18 @@ public class ProducerConfig extends AbstractConfig {
Type.BOOLEAN,
false,
Importance.LOW,
- ENABLE_IDEMPOTENCE_DOC);
+ ENABLE_IDEMPOTENCE_DOC)
+ .define(TRANSACTION_TIMEOUT_CONFIG,
+ Type.INT,
+ 60000,
+ Importance.LOW,
+ TRANSACTION_TIMEOUT_DOC)
+ .define(TRANSACTIONAL_ID_CONFIG,
+ Type.STRING,
+ null,
+ new ConfigDef.NonEmptyString(),
+ Importance.LOW,
+ TRANSACTIONAL_ID_DOC);
}
public static Map<String, Object> addSerializerToConfig(Map<String, Object> configs,
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java b/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java
deleted file mode 100644
index fa30b3f..0000000
--- a/clients/src/main/java/org/apache/kafka/clients/producer/TransactionState.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kafka.clients.producer;
-
-import org.apache.kafka.common.TopicPartition;
-import org.apache.kafka.common.utils.Time;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_EPOCH;
-import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_ID;
-
-/**
- * A class which maintains state for transactions. Also keeps the state necessary to ensure idempotent production.
- */
-public class TransactionState {
- private volatile PidAndEpoch pidAndEpoch;
- private final Map<TopicPartition, Integer> sequenceNumbers;
- private final Time time;
-
- public static class PidAndEpoch {
- public final long producerId;
- public final short epoch;
-
- PidAndEpoch(long producerId, short epoch) {
- this.producerId = producerId;
- this.epoch = epoch;
- }
-
- public boolean isValid() {
- return NO_PRODUCER_ID < producerId;
- }
- }
-
- public TransactionState(Time time) {
- this.pidAndEpoch = new PidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH);
- this.sequenceNumbers = new HashMap<>();
- this.time = time;
- }
-
- public boolean hasPid() {
- return pidAndEpoch.isValid();
- }
-
- /**
- * A blocking call to get the pid and epoch for the producer. If the PID and epoch has not been set, this method
- * will block for at most maxWaitTimeMs. It is expected that this method be called from application thread
- * contexts (ie. through Producer.send). The PID it self will be retrieved in the background thread.
- * @param maxWaitTimeMs The maximum time to block.
- * @return a PidAndEpoch object. Callers must call the 'isValid' method fo the returned object to ensure that a
- * valid Pid and epoch is actually returned.
- */
- public synchronized PidAndEpoch awaitPidAndEpoch(long maxWaitTimeMs) throws InterruptedException {
- long start = time.milliseconds();
- long elapsed = 0;
- while (!hasPid() && elapsed < maxWaitTimeMs) {
- wait(maxWaitTimeMs);
- elapsed = time.milliseconds() - start;
- }
- return pidAndEpoch;
- }
-
- /**
- * Get the current pid and epoch without blocking. Callers must use {@link PidAndEpoch#isValid()} to
- * verify that the result is valid.
- *
- * @return the current PidAndEpoch.
- */
- public PidAndEpoch pidAndEpoch() {
- return pidAndEpoch;
- }
-
- /**
- * Set the pid and epoch atomically. This method will signal any callers blocked on the `pidAndEpoch` method
- * once the pid is set. This method will be called on the background thread when the broker responds with the pid.
- */
- public synchronized void setPidAndEpoch(long pid, short epoch) {
- this.pidAndEpoch = new PidAndEpoch(pid, epoch);
- if (this.pidAndEpoch.isValid())
- notifyAll();
- }
-
- /**
- * This method is used when the producer needs to reset it's internal state because of an irrecoverable exception
- * from the broker.
- *
- * We need to reset the producer id and associated state when we have sent a batch to the broker, but we either get
- * a non-retriable exception or we run out of retries, or the batch expired in the producer queue after it was already
- * sent to the broker.
- *
- * In all of these cases, we don't know whether batch was actually committed on the broker, and hence whether the
- * sequence number was actually updated. If we don't reset the producer state, we risk the chance that all future
- * messages will return an OutOfOrderSequenceException.
- */
- public synchronized void resetProducerId() {
- setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH);
- this.sequenceNumbers.clear();
- }
-
- /**
- * Returns the next sequence number to be written to the given TopicPartition.
- */
- public synchronized Integer sequenceNumber(TopicPartition topicPartition) {
- Integer currentSequenceNumber = sequenceNumbers.get(topicPartition);
- if (currentSequenceNumber == null) {
- currentSequenceNumber = 0;
- sequenceNumbers.put(topicPartition, currentSequenceNumber);
- }
- return currentSequenceNumber;
- }
-
- public synchronized void incrementSequenceNumber(TopicPartition topicPartition, int increment) {
- Integer currentSequenceNumber = sequenceNumbers.get(topicPartition);
- if (currentSequenceNumber == null)
- throw new IllegalStateException("Attempt to increment sequence number for a partition with no current sequence.");
-
- currentSequenceNumber += increment;
- sequenceNumbers.put(topicPartition, currentSequenceNumber);
- }
-}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java
new file mode 100644
index 0000000..d05bc6a
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/FutureTransactionalResult.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.producer.internals;
+
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public final class FutureTransactionalResult implements Future<TransactionalRequestResult> {
+
+ private final TransactionalRequestResult result;
+
+ public FutureTransactionalResult(TransactionalRequestResult result) {
+ this.result = result;
+ }
+
+ @Override
+ public boolean isDone() {
+ return this.result.isCompleted();
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return false;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return false;
+ }
+
+ @Override
+ public TransactionalRequestResult get() {
+ this.result.await();
+ if (!result.isSuccessful()) {
+ throw result.error();
+ }
+ return result;
+ }
+
+ @Override
+ public TransactionalRequestResult get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
+ boolean occurred = this.result.await(timeout, unit);
+ if (!occurred) {
+ throw new TimeoutException("Could not complete transactional operation within " + TimeUnit.MILLISECONDS.convert(timeout, unit) + "ms.");
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java
index bacf0a2..eba3078 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/ProducerBatch.java
@@ -18,7 +18,6 @@ package org.apache.kafka.clients.producer.internals;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.kafka.clients.producer.TransactionState;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.TimeoutException;
import org.apache.kafka.common.record.AbstractRecords;
@@ -231,7 +230,7 @@ public final class ProducerBatch {
return recordsBuilder.isFull();
}
- public void setProducerState(TransactionState.PidAndEpoch pidAndEpoch, int baseSequence) {
+ public void setProducerState(TransactionManager.PidAndEpoch pidAndEpoch, int baseSequence) {
recordsBuilder.setProducerState(pidAndEpoch.producerId, pidAndEpoch.epoch, baseSequence);
}
@@ -273,4 +272,8 @@ public final class ProducerBatch {
public long producerId() {
return recordsBuilder.producerId();
}
+
+ public short producerEpoch() {
+ return recordsBuilder.producerEpoch();
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
index be03142..aa4d9d3 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/RecordAccumulator.java
@@ -18,7 +18,6 @@ package org.apache.kafka.clients.producer.internals;
import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.producer.Callback;
-import org.apache.kafka.clients.producer.TransactionState;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.Node;
@@ -82,7 +81,7 @@ public final class RecordAccumulator {
// The following variables are only accessed by the sender thread, so we don't need to protect them.
private final Set<TopicPartition> muted;
private int drainIndex;
- private final TransactionState transactionState;
+ private final TransactionManager transactionManager;
/**
* Create a new record accumulator
@@ -98,7 +97,7 @@ public final class RecordAccumulator {
* @param metrics The metrics
* @param time The time instance to use
* @param apiVersions Request API versions for current connected brokers
- * @param transactionState The shared transaction state object which tracks Pids, epochs, and sequence numbers per
+ * @param transactionManager The shared transaction state object which tracks Pids, epochs, and sequence numbers per
* partition.
*/
public RecordAccumulator(int batchSize,
@@ -109,7 +108,7 @@ public final class RecordAccumulator {
Metrics metrics,
Time time,
ApiVersions apiVersions,
- TransactionState transactionState) {
+ TransactionManager transactionManager) {
this.drainIndex = 0;
this.closed = false;
this.flushesInProgress = new AtomicInteger(0);
@@ -125,7 +124,7 @@ public final class RecordAccumulator {
this.muted = new HashSet<>();
this.time = time;
this.apiVersions = apiVersions;
- this.transactionState = transactionState;
+ this.transactionManager = transactionManager;
registerMetrics(metrics, metricGrpName);
}
@@ -229,11 +228,14 @@ public final class RecordAccumulator {
}
private MemoryRecordsBuilder recordsBuilder(ByteBuffer buffer, byte maxUsableMagic) {
- if (transactionState != null && maxUsableMagic < RecordBatch.MAGIC_VALUE_V2) {
+ if (transactionManager != null && maxUsableMagic < RecordBatch.MAGIC_VALUE_V2) {
throw new UnsupportedVersionException("Attempting to use idempotence with a broker which does not " +
"support the required message format (v2). The broker must be version 0.11 or later.");
}
- return MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L);
+ boolean isTransactional = false;
+ if (transactionManager != null)
+ isTransactional = transactionManager.isInTransaction();
+ return MemoryRecords.builder(buffer, maxUsableMagic, compression, TimestampType.CREATE_TIME, 0L, isTransactional);
}
/**
@@ -437,9 +439,9 @@ public final class RecordAccumulator {
// request
break;
} else {
- TransactionState.PidAndEpoch pidAndEpoch = null;
- if (transactionState != null) {
- pidAndEpoch = transactionState.pidAndEpoch();
+ TransactionManager.PidAndEpoch pidAndEpoch = null;
+ if (transactionManager != null) {
+ pidAndEpoch = transactionManager.pidAndEpoch();
if (!pidAndEpoch.isValid())
// we cannot send the batch until we have refreshed the PID
break;
@@ -452,7 +454,7 @@ public final class RecordAccumulator {
// the previous attempt may actually have been accepted, and if we change
// the pid and sequence here, this attempt will also be accepted, causing
// a duplicate.
- int sequenceNumber = transactionState.sequenceNumber(batch.topicPartition);
+ int sequenceNumber = transactionManager.sequenceNumber(batch.topicPartition);
log.debug("Dest: {} : producerId: {}, epoch: {}, Assigning sequence for {}: {}",
node, pidAndEpoch.producerId, pidAndEpoch.epoch,
batch.topicPartition, sequenceNumber);
@@ -542,6 +544,14 @@ public final class RecordAccumulator {
}
}
+ public boolean hasUnflushedBatches() {
+ for (Map.Entry<TopicPartition, Deque<ProducerBatch>> entry : this.batches().entrySet()) {
+ if (!entry.getValue().isEmpty())
+ return true;
+ }
+ return !this.incomplete.incomplete.isEmpty();
+ }
+
/**
* This function is only called when sender is closed forcefully. It will fail all the
* incomplete batches and return.
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
index ab92522..698f4de 100644
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java
@@ -22,8 +22,8 @@ import org.apache.kafka.clients.ClientResponse;
import org.apache.kafka.clients.KafkaClient;
import org.apache.kafka.clients.Metadata;
import org.apache.kafka.clients.RequestCompletionHandler;
-import org.apache.kafka.clients.producer.TransactionState;
import org.apache.kafka.common.Cluster;
+import org.apache.kafka.common.KafkaException;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.TopicPartition;
@@ -109,7 +109,7 @@ public class Sender implements Runnable {
private final ApiVersions apiVersions;
/* all the state related to transactions, in particular the PID, epoch, and sequence numbers */
- private final TransactionState transactionState;
+ private final TransactionManager transactionManager;
public Sender(KafkaClient client,
Metadata metadata,
@@ -122,7 +122,7 @@ public class Sender implements Runnable {
Time time,
int requestTimeout,
long retryBackoffMs,
- TransactionState transactionState,
+ TransactionManager transactionManager,
ApiVersions apiVersions) {
this.client = client;
this.accumulator = accumulator;
@@ -137,7 +137,7 @@ public class Sender implements Runnable {
this.requestTimeout = requestTimeout;
this.retryBackoffMs = retryBackoffMs;
this.apiVersions = apiVersions;
- this.transactionState = transactionState;
+ this.transactionManager = transactionManager;
}
/**
@@ -187,8 +187,16 @@ public class Sender implements Runnable {
* @param now The current POSIX time in milliseconds
*/
void run(long now) {
- Cluster cluster = metadata.fetch();
+ long pollTimeout = 0;
+ if (!maybeSendTransactionalRequest(now))
+ pollTimeout = sendProducerData(now);
+
+ this.client.poll(pollTimeout, now);
+ }
+
+ private long sendProducerData(long now) {
+ Cluster cluster = metadata.fetch();
maybeWaitForPid();
// get the list of partitions with data ready to send
@@ -233,15 +241,15 @@ public class Sender implements Runnable {
// for expired batches. see the documentation of @TransactionState.resetProducerId to understand why
// we need to reset the producer id here.
for (ProducerBatch expiredBatch : expiredBatches) {
- if (transactionState != null && expiredBatch.inRetry()) {
+ if (transactionManager != null && expiredBatch.inRetry()) {
needsTransactionStateReset = true;
}
this.sensors.recordErrors(expiredBatch.topicPartition.topic(), expiredBatch.recordCount);
}
if (needsTransactionStateReset) {
- transactionState.resetProducerId();
- return;
+ transactionManager.resetProducerId();
+ return 0;
}
sensors.updateProduceRequestMetrics(batches);
@@ -253,15 +261,79 @@ public class Sender implements Runnable {
long pollTimeout = Math.min(result.nextReadyCheckDelayMs, notReadyTimeout);
if (!result.readyNodes.isEmpty()) {
log.trace("Nodes with data ready to send: {}", result.readyNodes);
+ // if some partitions are already ready to be sent, the select time would be 0;
+ // otherwise if some partition already has some data accumulated but not ready yet,
+ // the select time will be the time difference between now and its linger expiry time;
+ // otherwise the select time will be the time difference between now and the metadata expiry time;
pollTimeout = 0;
}
sendProduceRequests(batches, now);
- // if some partitions are already ready to be sent, the select time would be 0;
- // otherwise if some partition already has some data accumulated but not ready yet,
- // the select time will be the time difference between now and its linger expiry time;
- // otherwise the select time will be the time difference between now and the metadata expiry time;
- this.client.poll(pollTimeout, now);
+ return pollTimeout;
+
+ }
+
+ private boolean maybeSendTransactionalRequest(long now) {
+ if (transactionManager != null && transactionManager.hasInflightTransactionalRequest())
+ return true;
+
+ if (transactionManager == null || !transactionManager.hasPendingTransactionalRequests())
+ return false;
+
+ TransactionManager.TransactionalRequest nextRequest = transactionManager.nextTransactionalRequest();
+
+ if (nextRequest.isEndTxnRequest() && transactionManager.isCompletingTransaction() && accumulator.hasUnflushedBatches()) {
+ if (!accumulator.flushInProgress())
+ accumulator.beginFlush();
+ transactionManager.reenqueue(nextRequest);
+ return false;
+ }
+
+ if (nextRequest.isEndTxnRequest() && transactionManager.isInErrorState()) {
+ nextRequest.maybeTerminateWithError(new KafkaException("Cannot commit transaction when there are " +
+ "request errors. Please check your logs for the details of the errors encountered."));
+ return false;
+ }
+
+ Node targetNode = null;
+
+ while (targetNode == null) {
+ try {
+ if (nextRequest.needsCoordinator()) {
+ targetNode = transactionManager.coordinator(nextRequest.coordinatorType());
+ if (targetNode == null) {
+ transactionManager.needsCoordinator(nextRequest);
+ break;
+ }
+ if (!NetworkClientUtils.awaitReady(client, targetNode, time, requestTimeout)) {
+ transactionManager.needsCoordinator(nextRequest);
+ targetNode = null;
+ break;
+ }
+ } else {
+ targetNode = awaitLeastLoadedNodeReady(requestTimeout);
+ }
+ if (targetNode != null) {
+ if (nextRequest.isRetry()) {
+ time.sleep(retryBackoffMs);
+ }
+ ClientRequest clientRequest = client.newClientRequest(targetNode.idString(), nextRequest.requestBuilder(),
+ now, true, nextRequest.responseHandler());
+ transactionManager.setInFlightRequestCorrelationId(clientRequest.correlationId());
+ client.send(clientRequest, now);
+ return true;
+ }
+ } catch (IOException e) {
+ log.warn("Got an exception when trying to find a node to send a transactional request to. Going to back off and retry", e);
+ }
+ time.sleep(retryBackoffMs);
+ metadata.requestUpdate();
+ }
+
+ if (targetNode == null)
+ transactionManager.needsRetry(nextRequest);
+
+ return true;
}
/**
@@ -299,17 +371,19 @@ public class Sender implements Runnable {
}
private void maybeWaitForPid() {
- if (transactionState == null)
+ // If this is a transactional producer, the PID will be received when recovering transactions in the
+ // initTransactions() method of the producer.
+ if (transactionManager == null || transactionManager.isTransactional())
return;
- while (!transactionState.hasPid()) {
+ while (!transactionManager.hasPid()) {
try {
Node node = awaitLeastLoadedNodeReady(requestTimeout);
if (node != null) {
ClientResponse response = sendAndAwaitInitPidRequest(node);
if (response.hasResponse() && (response.responseBody() instanceof InitPidResponse)) {
InitPidResponse initPidResponse = (InitPidResponse) response.responseBody();
- transactionState.setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch());
+ transactionManager.setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch());
} else {
log.error("Received an unexpected response type for an InitPidRequest from {}. " +
"We will back off and try again.", node);
@@ -381,17 +455,17 @@ public class Sender implements Runnable {
batch.topicPartition,
this.retries - batch.attempts() - 1,
error);
- if (transactionState == null) {
+ if (transactionManager == null) {
reenqueueBatch(batch, now);
- } else if (transactionState.pidAndEpoch().producerId == batch.producerId()) {
+ } else if (transactionManager.pidAndEpoch().producerId == batch.producerId() && transactionManager.pidAndEpoch().epoch == batch.producerEpoch()) {
// If idempotence is enabled only retry the request if the current PID is the same as the pid of the batch.
log.debug("Retrying batch to topic-partition {}. Sequence number : {}", batch.topicPartition,
- transactionState.sequenceNumber(batch.topicPartition));
+ transactionManager.sequenceNumber(batch.topicPartition));
reenqueueBatch(batch, now);
} else {
failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " +
"batch but the producer id changed from " + batch.producerId() + " to " +
- transactionState.pidAndEpoch().producerId + " in the mean time. This batch will be dropped."));
+ transactionManager.pidAndEpoch().producerId + " in the mean time. This batch will be dropped."));
this.sensors.recordErrors(batch.topicPartition.topic(), batch.recordCount);
}
} else {
@@ -400,7 +474,7 @@ public class Sender implements Runnable {
exception = new TopicAuthorizationException(batch.topicPartition.topic());
else
exception = error.exception();
- if (error == Errors.OUT_OF_ORDER_SEQUENCE_NUMBER && batch.producerId() == transactionState.pidAndEpoch().producerId)
+ if (error == Errors.OUT_OF_ORDER_SEQUENCE_NUMBER && batch.producerId() == transactionManager.pidAndEpoch().producerId)
log.error("The broker received an out of order sequence number for correlation id {}, topic-partition " +
"{} at offset {}. This indicates data loss on the broker, and should be investigated.",
correlationId, batch.topicPartition, response.baseOffset);
@@ -418,10 +492,11 @@ public class Sender implements Runnable {
} else {
completeBatch(batch, response);
- if (transactionState != null && transactionState.pidAndEpoch().producerId == batch.producerId()) {
- transactionState.incrementSequenceNumber(batch.topicPartition, batch.recordCount);
+ if (transactionManager != null && transactionManager.pidAndEpoch().producerId == batch.producerId()
+ && transactionManager.pidAndEpoch().epoch == batch.producerEpoch()) {
+ transactionManager.incrementSequenceNumber(batch.topicPartition, batch.recordCount);
log.debug("Incremented sequence number for topic-partition {} to {}", batch.topicPartition,
- transactionState.sequenceNumber(batch.topicPartition));
+ transactionManager.sequenceNumber(batch.topicPartition));
}
}
@@ -441,11 +516,12 @@ public class Sender implements Runnable {
}
private void failBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, RuntimeException exception) {
- if (transactionState != null && batch.producerId() == transactionState.pidAndEpoch().producerId) {
+ if (transactionManager != null && !transactionManager.isTransactional()
+ && batch.producerId() == transactionManager.pidAndEpoch().producerId) {
// Reset the transaction state since we have hit an irrecoverable exception and cannot make any guarantees
// about the previously committed message. Note that this will discard the producer id and sequence
// numbers for all existing partitions.
- transactionState.resetProducerId();
+ transactionManager.resetProducerId();
}
batch.done(response.baseOffset, response.logAppendTime, exception);
this.accumulator.deallocate(batch);
@@ -500,8 +576,12 @@ public class Sender implements Runnable {
recordsByPartition.put(tp, batch);
}
+ String transactionalId = null;
+ if (transactionManager != null && transactionManager.isTransactional()) {
+ transactionalId = transactionManager.transactionalId();
+ }
ProduceRequest.Builder requestBuilder = new ProduceRequest.Builder(minUsedMagic, acks, timeout,
- produceRecordsByPartition);
+ produceRecordsByPartition, transactionalId);
RequestCompletionHandler callback = new RequestCompletionHandler() {
public void onComplete(ClientResponse response) {
handleProduceResponse(response, recordsByPartition, time.milliseconds());
[2/3] kafka git commit: KAFKA-4818; Exactly once transactional clients
Posted by jg...@apache.org.
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
new file mode 100644
index 0000000..644d4f8
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
@@ -0,0 +1,866 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.producer.internals;
+
+import org.apache.kafka.clients.ClientResponse;
+import org.apache.kafka.clients.RequestCompletionHandler;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.Node;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.ProducerFencedException;
+import org.apache.kafka.common.protocol.Errors;
+import org.apache.kafka.common.requests.AbstractRequest;
+import org.apache.kafka.common.requests.AbstractResponse;
+import org.apache.kafka.common.requests.AddOffsetsToTxnRequest;
+import org.apache.kafka.common.requests.AddOffsetsToTxnResponse;
+import org.apache.kafka.common.requests.AddPartitionsToTxnRequest;
+import org.apache.kafka.common.requests.AddPartitionsToTxnResponse;
+import org.apache.kafka.common.requests.EndTxnRequest;
+import org.apache.kafka.common.requests.EndTxnResponse;
+import org.apache.kafka.common.requests.FindCoordinatorRequest;
+import org.apache.kafka.common.requests.FindCoordinatorResponse;
+import org.apache.kafka.common.requests.InitPidRequest;
+import org.apache.kafka.common.requests.InitPidResponse;
+import org.apache.kafka.common.requests.OffsetCommitRequest;
+import org.apache.kafka.common.requests.TransactionResult;
+import org.apache.kafka.common.requests.TxnOffsetCommitRequest;
+import org.apache.kafka.common.requests.TxnOffsetCommitResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_EPOCH;
+import static org.apache.kafka.common.record.RecordBatch.NO_PRODUCER_ID;
+
+/**
+ * A class which maintains state for transactions. Also keeps the state necessary to ensure idempotent production.
+ */
+public class TransactionManager {
+ private static final Logger log = LoggerFactory.getLogger(TransactionManager.class);
+
+ private static final int NO_INFLIGHT_REQUEST_CORRELATION_ID = -1;
+
+ private volatile PidAndEpoch pidAndEpoch;
+ private final Map<TopicPartition, Integer> sequenceNumbers;
+ private final String transactionalId;
+ private final int transactionTimeoutMs;
+ private final PriorityQueue<TransactionalRequest> pendingTransactionalRequests;
+ private final Set<TopicPartition> newPartitionsToBeAddedToTransaction;
+ private final Set<TopicPartition> pendingPartitionsToBeAddedToTransaction;
+ private final Set<TopicPartition> partitionsInTransaction;
+ private final Map<TopicPartition, TxnOffsetCommitRequest.CommittedOffset> pendingTxnOffsetCommits;
+ private int inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID;
+
+ private Node transactionCoordinator;
+ private Node consumerGroupCoordinator;
+ private volatile State currentState = State.UNINITIALIZED;
+ private Exception lastError = null;
+
+ private enum State {
+ UNINITIALIZED,
+ INITIALIZING,
+ READY,
+ IN_TRANSACTION,
+ COMMITTING_TRANSACTION,
+ ABORTING_TRANSACTION,
+ FENCED,
+ ERROR;
+
+ private boolean isTransitionValid(State source, State target) {
+ switch (target) {
+ case INITIALIZING:
+ return source == UNINITIALIZED || source == ERROR;
+ case READY:
+ return source == INITIALIZING || source == COMMITTING_TRANSACTION || source == ABORTING_TRANSACTION;
+ case IN_TRANSACTION:
+ return source == READY;
+ case COMMITTING_TRANSACTION:
+ return source == IN_TRANSACTION;
+ case ABORTING_TRANSACTION:
+ return source == IN_TRANSACTION || source == ERROR;
+ default:
+ // We can transition to FENCED or ERROR unconditionally.
+ // FENCED is never a valid starting state for any transition. So the only option is to close the
+ // producer or do purely non transactional requests.
+ return true;
+ }
+ }
+ }
+
+ public TransactionManager(String transactionalId, int transactionTimeoutMs) {
+ pidAndEpoch = new PidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH);
+ sequenceNumbers = new HashMap<>();
+ this.transactionalId = transactionalId;
+ this.transactionTimeoutMs = transactionTimeoutMs;
+ this.pendingTransactionalRequests = new PriorityQueue<>(10, new Comparator<TransactionalRequest>() {
+ @Override
+ public int compare(TransactionalRequest o1, TransactionalRequest o2) {
+ return Integer.compare(o1.priority().priority(), o2.priority.priority());
+ }
+ });
+ this.transactionCoordinator = null;
+ this.consumerGroupCoordinator = null;
+ this.newPartitionsToBeAddedToTransaction = new HashSet<>();
+ this.pendingPartitionsToBeAddedToTransaction = new HashSet<>();
+ this.partitionsInTransaction = new HashSet<>();
+ this.pendingTxnOffsetCommits = new HashMap<>();
+ }
+
+ public TransactionManager() {
+ this("", 0);
+ }
+
+ static class TransactionalRequest {
+ private enum Priority {
+ FIND_COORDINATOR(0),
+ INIT_PRODUCER_ID(1),
+ ADD_PARTITIONS_OR_OFFSETS(2),
+ END_TXN(4);
+
+ private final int priority;
+
+ Priority(int priority) {
+ this.priority = priority;
+ }
+
+ public int priority() {
+ return this.priority;
+ }
+ }
+
+ private final AbstractRequest.Builder<?> requestBuilder;
+
+ private final FindCoordinatorRequest.CoordinatorType coordinatorType;
+ private final String coordinatorKey;
+ private final RequestCompletionHandler handler;
+ // We use the priority to determine the order in which requests need to be sent out. For instance, if we have
+ // a pending FindCoordinator request, that must always go first. Next, If we need a Pid, that must go second.
+ // The endTxn request must always go last.
+ private final Priority priority;
+ private final TransactionalRequestResult result;
+ private boolean isRetry;
+
+ private TransactionalRequest(AbstractRequest.Builder<?> requestBuilder, RequestCompletionHandler handler,
+ FindCoordinatorRequest.CoordinatorType coordinatorType, Priority priority,
+ boolean isRetry, String coordinatorKey, TransactionalRequestResult result) {
+ this.requestBuilder = requestBuilder;
+ this.handler = handler;
+ this.coordinatorType = coordinatorType;
+ this.priority = priority;
+ this.isRetry = isRetry;
+ this.coordinatorKey = coordinatorKey;
+ this.result = result;
+ }
+
+ AbstractRequest.Builder<?> requestBuilder() {
+ return requestBuilder;
+ }
+
+ boolean needsCoordinator() {
+ return coordinatorType != null;
+ }
+
+ FindCoordinatorRequest.CoordinatorType coordinatorType() {
+ return coordinatorType;
+ }
+
+ RequestCompletionHandler responseHandler() {
+ return handler;
+ }
+
+ boolean isRetry() {
+ return isRetry;
+ }
+
+ boolean isEndTxnRequest() {
+ return priority == Priority.END_TXN;
+ }
+
+ boolean maybeTerminateWithError(RuntimeException error) {
+ if (result != null) {
+ result.setError(error);
+ result.done();
+ return true;
+ }
+ return false;
+ }
+
+ private void setRetry() {
+ isRetry = true;
+ }
+
+ private Priority priority() {
+ return priority;
+ }
+ }
+
+ static class PidAndEpoch {
+ public final long producerId;
+ public final short epoch;
+
+ PidAndEpoch(long producerId, short epoch) {
+ this.producerId = producerId;
+ this.epoch = epoch;
+ }
+
+ public boolean isValid() {
+ return NO_PRODUCER_ID < producerId;
+ }
+ }
+
+ public synchronized FutureTransactionalResult initializeTransactions() {
+ ensureTransactional();
+ transitionTo(State.INITIALIZING);
+ setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH);
+ this.sequenceNumbers.clear();
+ if (transactionCoordinator == null)
+ pendingTransactionalRequests.add(findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId, false));
+
+ TransactionalRequestResult result = new TransactionalRequestResult();
+ FutureTransactionalResult resultFuture = new FutureTransactionalResult(result);
+ if (!hasPid())
+ pendingTransactionalRequests.add(initPidRequest(false, result));
+ else
+ result.done();
+
+ return resultFuture;
+ }
+
+ public synchronized void beginTransaction() {
+ ensureTransactional();
+ maybeFailWithError();
+ transitionTo(State.IN_TRANSACTION);
+ }
+
+ public synchronized FutureTransactionalResult beginCommittingTransaction() {
+ ensureTransactional();
+ maybeFailWithError();
+ transitionTo(State.COMMITTING_TRANSACTION);
+ return beginCompletingTransaction(true);
+ }
+
+ public synchronized FutureTransactionalResult beginAbortingTransaction() {
+ ensureTransactional();
+ if (isFenced())
+ throw new ProducerFencedException("There is a newer producer using the same transactional.id.");
+ transitionTo(State.ABORTING_TRANSACTION);
+ return beginCompletingTransaction(false);
+ }
+
+ private FutureTransactionalResult beginCompletingTransaction(boolean isCommit) {
+ TransactionalRequestResult result = new TransactionalRequestResult();
+ FutureTransactionalResult resultFuture = new FutureTransactionalResult(result);
+
+ if (!newPartitionsToBeAddedToTransaction.isEmpty()) {
+ pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false));
+ }
+ pendingTransactionalRequests.add(endTxnRequest(isCommit, false, result));
+ return resultFuture;
+ }
+
+ public synchronized FutureTransactionalResult sendOffsetsToTransaction(Map<TopicPartition, OffsetAndMetadata> offsets,
+ String consumerGroupId) {
+ ensureTransactional();
+ maybeFailWithError();
+ if (currentState != State.IN_TRANSACTION)
+ throw new KafkaException("Cannot send offsets to transaction either because the producer is not in an " +
+ "active transaction");
+
+ TransactionalRequestResult result = new TransactionalRequestResult();
+ FutureTransactionalResult resultFuture = new FutureTransactionalResult(result);
+ pendingTransactionalRequests.add(addOffsetsToTxnRequest(offsets, consumerGroupId, false, result));
+ return resultFuture;
+ }
+
+ public synchronized void maybeAddPartitionToTransaction(TopicPartition topicPartition) {
+ if (!isInTransaction() || partitionsInTransaction.contains(topicPartition))
+ return;
+ newPartitionsToBeAddedToTransaction.add(topicPartition);
+ }
+
+ public Exception lastError() {
+ return lastError;
+ }
+
+ public String transactionalId() {
+ return transactionalId;
+ }
+
+ public boolean hasPid() {
+ return pidAndEpoch.isValid();
+ }
+
+ public boolean isTransactional() {
+ return transactionalId != null && !transactionalId.isEmpty();
+ }
+
+ public boolean isFenced() {
+ return currentState == State.FENCED;
+ }
+
+ public boolean isCompletingTransaction() {
+ return currentState == State.COMMITTING_TRANSACTION || currentState == State.ABORTING_TRANSACTION;
+ }
+
+ public boolean isInTransaction() {
+ return currentState == State.IN_TRANSACTION || isCompletingTransaction();
+ }
+
+ public boolean isInErrorState() {
+ return currentState == State.ERROR;
+ }
+
+ public synchronized boolean maybeSetError(Exception exception) {
+ if (isTransactional() && isInTransaction()) {
+ if (exception instanceof ProducerFencedException)
+ transitionTo(State.FENCED, exception);
+ else
+ transitionTo(State.ERROR, exception);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Get the current pid and epoch without blocking. Callers must use {@link PidAndEpoch#isValid()} to
+ * verify that the result is valid.
+ *
+ * @return the current PidAndEpoch.
+ */
+ PidAndEpoch pidAndEpoch() {
+ return pidAndEpoch;
+ }
+
+ /**
+ * Set the pid and epoch atomically. This method will signal any callers blocked on the `pidAndEpoch` method
+ * once the pid is set. This method will be called on the background thread when the broker responds with the pid.
+ */
+ synchronized void setPidAndEpoch(long pid, short epoch) {
+ this.pidAndEpoch = new PidAndEpoch(pid, epoch);
+ }
+
+ /**
+ * This method is used when the producer needs to reset its internal state because of an irrecoverable exception
+ * from the broker.
+ *
+ * We need to reset the producer id and associated state when we have sent a batch to the broker, but we either get
+ * a non-retriable exception or we run out of retries, or the batch expired in the producer queue after it was already
+ * sent to the broker.
+ *
+ * In all of these cases, we don't know whether batch was actually committed on the broker, and hence whether the
+ * sequence number was actually updated. If we don't reset the producer state, we risk the chance that all future
+ * messages will return an OutOfOrderSequenceException.
+ *
+ * Note that we can't reset the producer state for the transactional producer as this would mean bumping the epoch
+ * for the same pid. This might involve aborting the ongoing transaction during the initPidRequest, and the user
+ * would not have any way of knowing this happened. So for the transactional producer, it's best to return the
+ * produce error to the user and let them abort the transaction and close the producer explicitly.
+ */
+ synchronized void resetProducerId() {
+ if (isTransactional())
+ throw new IllegalStateException("Cannot reset producer state for a transactional producer. " +
+ "You must either abort the ongoing transaction or reinitialize the transactional producer instead");
+ setPidAndEpoch(NO_PRODUCER_ID, NO_PRODUCER_EPOCH);
+ this.sequenceNumbers.clear();
+ }
+
+ /**
+ * Returns the next sequence number to be written to the given TopicPartition.
+ */
+ synchronized Integer sequenceNumber(TopicPartition topicPartition) {
+ Integer currentSequenceNumber = sequenceNumbers.get(topicPartition);
+ if (currentSequenceNumber == null) {
+ currentSequenceNumber = 0;
+ sequenceNumbers.put(topicPartition, currentSequenceNumber);
+ }
+ return currentSequenceNumber;
+ }
+
+ synchronized void incrementSequenceNumber(TopicPartition topicPartition, int increment) {
+ Integer currentSequenceNumber = sequenceNumbers.get(topicPartition);
+ if (currentSequenceNumber == null)
+ throw new IllegalStateException("Attempt to increment sequence number for a partition with no current sequence.");
+
+ currentSequenceNumber += increment;
+ sequenceNumbers.put(topicPartition, currentSequenceNumber);
+ }
+
+ boolean hasPendingTransactionalRequests() {
+ return !(pendingTransactionalRequests.isEmpty()
+ && newPartitionsToBeAddedToTransaction.isEmpty());
+ }
+
+ TransactionalRequest nextTransactionalRequest() {
+ if (!hasPendingTransactionalRequests())
+ return null;
+
+ if (!newPartitionsToBeAddedToTransaction.isEmpty())
+ pendingTransactionalRequests.add(addPartitionsToTransactionRequest(false));
+
+ return pendingTransactionalRequests.poll();
+ }
+
+ void needsRetry(TransactionalRequest request) {
+ request.setRetry();
+ pendingTransactionalRequests.add(request);
+ }
+
+ void reenqueue(TransactionalRequest request) {
+ pendingTransactionalRequests.add(request);
+ }
+
+ Node coordinator(FindCoordinatorRequest.CoordinatorType type) {
+ switch (type) {
+ case GROUP:
+ return consumerGroupCoordinator;
+ case TRANSACTION:
+ return transactionCoordinator;
+ default:
+ throw new IllegalStateException("Received an invalid coordinator type: " + type);
+ }
+ }
+
+ void needsCoordinator(TransactionalRequest request) {
+ needsCoordinator(request.coordinatorType, request.coordinatorKey);
+ }
+
+ void setInFlightRequestCorrelationId(int correlationId) {
+ inFlightRequestCorrelationId = correlationId;
+ }
+
+ void resetInFlightRequestCorrelationId() {
+ inFlightRequestCorrelationId = NO_INFLIGHT_REQUEST_CORRELATION_ID;
+ }
+
+ boolean hasInflightTransactionalRequest() {
+ return inFlightRequestCorrelationId != NO_INFLIGHT_REQUEST_CORRELATION_ID;
+ }
+
+ // visible for testing
+ boolean transactionContainsPartition(TopicPartition topicPartition) {
+ return isInTransaction() && partitionsInTransaction.contains(topicPartition);
+ }
+
+ // visible for testing
+ boolean hasPendingOffsetCommits() {
+ return isInTransaction() && !pendingTxnOffsetCommits.isEmpty();
+ }
+
+ // visible for testing
+ boolean isReadyForTransaction() {
+ return isTransactional() && currentState == State.READY;
+ }
+
+ private void transitionTo(State target) {
+ transitionTo(target, null);
+ }
+
+ private void transitionTo(State target, Exception error) {
+ if (target == State.ERROR && error != null)
+ lastError = error;
+ if (currentState.isTransitionValid(currentState, target)) {
+ currentState = target;
+ } else {
+ throw new KafkaException("Invalid transition attempted from state " + currentState.name() + " to state " + target.name());
+ }
+ }
+
+ private void ensureTransactional() {
+ if (!isTransactional())
+ throw new IllegalStateException("Transactional method invoked on a non-transactional producer.");
+ }
+
+ private void maybeFailWithError() {
+ if (isFenced())
+ throw new ProducerFencedException("There is a newer producer instance using the same transactional id.");
+ if (isInErrorState()) {
+ String errorMessage = "Cannot execute transactional method because we are in an error state.";
+ if (lastError != null)
+ throw new KafkaException(errorMessage, lastError);
+ else
+ throw new KafkaException(errorMessage);
+ }
+ }
+
+ private void needsCoordinator(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) {
+ switch (type) {
+ case GROUP:
+ consumerGroupCoordinator = null;
+ break;
+ case TRANSACTION:
+ transactionCoordinator = null;
+ break;
+ default:
+ throw new IllegalStateException("Got an invalid coordinator type: " + type);
+ }
+ pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, false));
+ }
+
+
+ private void completeTransaction() {
+ transitionTo(State.READY);
+ lastError = null;
+ partitionsInTransaction.clear();
+ }
+
+ private TransactionalRequest initPidRequest(boolean isRetry, TransactionalRequestResult result) {
+ InitPidRequest.Builder builder = new InitPidRequest.Builder(transactionalId, transactionTimeoutMs);
+ return new TransactionalRequest(builder, new InitPidCallback(result),
+ FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.INIT_PRODUCER_ID, isRetry, transactionalId, result);
+ }
+
+ private synchronized TransactionalRequest addPartitionsToTransactionRequest(boolean isRetry) {
+ pendingPartitionsToBeAddedToTransaction.addAll(newPartitionsToBeAddedToTransaction);
+ newPartitionsToBeAddedToTransaction.clear();
+ AddPartitionsToTxnRequest.Builder builder = new AddPartitionsToTxnRequest.Builder(transactionalId,
+ pidAndEpoch.producerId, pidAndEpoch.epoch, new ArrayList<>(pendingPartitionsToBeAddedToTransaction));
+ return new TransactionalRequest(builder, new AddPartitionsToTransactionCallback(),
+ FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, transactionalId, null);
+ }
+
+ private TransactionalRequest findCoordinatorRequest(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey, boolean isRetry) {
+ FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder(type, coordinatorKey);
+ return new TransactionalRequest(builder, new FindCoordinatorCallback(type, coordinatorKey),
+ null, TransactionalRequest.Priority.FIND_COORDINATOR, isRetry, null, null);
+ }
+
+ private TransactionalRequest endTxnRequest(boolean isCommit, boolean isRetry, TransactionalRequestResult result) {
+ EndTxnRequest.Builder builder = new EndTxnRequest.Builder(transactionalId,
+ pidAndEpoch.producerId, pidAndEpoch.epoch, isCommit ? TransactionResult.COMMIT : TransactionResult.ABORT);
+ return new TransactionalRequest(builder, new EndTxnCallback(isCommit, result),
+ FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.END_TXN, isRetry, transactionalId, result);
+ }
+
+ private TransactionalRequest addOffsetsToTxnRequest(Map<TopicPartition, OffsetAndMetadata> offsets,
+ String consumerGroupId, boolean isRetry, TransactionalRequestResult result) {
+ AddOffsetsToTxnRequest.Builder builder = new AddOffsetsToTxnRequest.Builder(transactionalId,
+ pidAndEpoch.producerId, pidAndEpoch.epoch, consumerGroupId);
+ return new TransactionalRequest(builder, new AddOffsetsToTxnCallback(offsets, consumerGroupId, result),
+ FindCoordinatorRequest.CoordinatorType.TRANSACTION, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, transactionalId, result);
+ }
+
+ private TransactionalRequest txnOffsetCommitRequest(Map<TopicPartition, OffsetAndMetadata> offsets,
+ String consumerGroupId, boolean isRetry, TransactionalRequestResult result) {
+ for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
+ OffsetAndMetadata offsetAndMetadata = entry.getValue();
+ pendingTxnOffsetCommits.put(entry.getKey(),
+ new TxnOffsetCommitRequest.CommittedOffset(offsetAndMetadata.offset(), offsetAndMetadata.metadata()));
+ }
+ return txnOffsetCommitRequest(consumerGroupId, isRetry, result);
+ }
+
+ private TransactionalRequest txnOffsetCommitRequest(String consumerGroupId, boolean isRetry, TransactionalRequestResult result) {
+ TxnOffsetCommitRequest.Builder builder = new TxnOffsetCommitRequest.Builder(consumerGroupId,
+ pidAndEpoch.producerId, pidAndEpoch.epoch, OffsetCommitRequest.DEFAULT_RETENTION_TIME, pendingTxnOffsetCommits);
+ return new TransactionalRequest(builder, new TxnOffsetCommitCallback(consumerGroupId, result),
+ FindCoordinatorRequest.CoordinatorType.GROUP, TransactionalRequest.Priority.ADD_PARTITIONS_OR_OFFSETS, isRetry, consumerGroupId, result);
+ }
+
+ private abstract class TransactionalRequestCallBack implements RequestCompletionHandler {
+ protected final TransactionalRequestResult result;
+
+ TransactionalRequestCallBack(TransactionalRequestResult result) {
+ this.result = result;
+ }
+
+ @Override
+ public void onComplete(ClientResponse response) {
+ if (response.requestHeader().correlationId() != inFlightRequestCorrelationId) {
+ log.error("Detected more than one inflight transactional request. This should never happen.");
+ transitionTo(State.ERROR, new RuntimeException("Detected more than one inflight transactional request. This should never happen."));
+ return;
+ }
+
+ resetInFlightRequestCorrelationId();
+ if (response.wasDisconnected()) {
+ reenqueue();
+ } else if (response.versionMismatch() != null) {
+ if (result != null) {
+ result.setError(Errors.UNSUPPORTED_VERSION.exception());
+ result.done();
+ }
+ log.error("Could not execute transactional request because the broker isn't on the right version.");
+ transitionTo(State.ERROR, Errors.UNSUPPORTED_VERSION.exception());
+ } else if (response.hasResponse()) {
+ handleResponse(response.responseBody());
+ } else {
+ if (result != null) {
+ result.setError(Errors.UNKNOWN.exception());
+ result.done();
+ }
+ log.error("Could not execute transactional request for unknown reasons");
+ transitionTo(State.ERROR, Errors.UNKNOWN.exception());
+ }
+ }
+
+ public abstract void handleResponse(AbstractResponse responseBody);
+
+ public abstract void reenqueue();
+ }
+
+ private class InitPidCallback extends TransactionalRequestCallBack {
+
+ InitPidCallback(TransactionalRequestResult result) {
+ super(result);
+ }
+
+ @Override
+ public void handleResponse(AbstractResponse responseBody) {
+ InitPidResponse initPidResponse = (InitPidResponse) responseBody;
+ Errors error = initPidResponse.error();
+ if (error == Errors.NONE) {
+ setPidAndEpoch(initPidResponse.producerId(), initPidResponse.epoch());
+ transitionTo(State.READY);
+ lastError = null;
+ } else if (error == Errors.NOT_COORDINATOR || error == Errors.COORDINATOR_NOT_AVAILABLE) {
+ needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ reenqueue();
+ } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) {
+ reenqueue();
+ } else {
+ result.setError(error.exception());
+ transitionTo(State.ERROR, error.exception());
+ }
+
+ if (error == Errors.NONE || !result.isSuccessful())
+ result.done();
+ }
+
+ @Override
+ public void reenqueue() {
+ pendingTransactionalRequests.add(initPidRequest(true, result));
+ }
+ }
+
+ private class AddPartitionsToTransactionCallback extends TransactionalRequestCallBack {
+
+ AddPartitionsToTransactionCallback() {
+ super(null);
+ }
+
+ @Override
+ public void handleResponse(AbstractResponse response) {
+ AddPartitionsToTxnResponse addPartitionsToTxnResponse = (AddPartitionsToTxnResponse) response;
+ Errors error = addPartitionsToTxnResponse.error();
+ if (error == Errors.NONE) {
+ partitionsInTransaction.addAll(pendingPartitionsToBeAddedToTransaction);
+ pendingPartitionsToBeAddedToTransaction.clear();
+ } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) {
+ needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ reenqueue();
+ } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) {
+ reenqueue();
+ } else if (error == Errors.INVALID_PID_MAPPING || error == Errors.INVALID_TXN_STATE) {
+ log.error("Seems like the broker has bad transaction state. producerId: {}, error: {}. message: {}",
+ pidAndEpoch.producerId, error, error.message());
+ transitionTo(State.ERROR, error.exception());
+ } else if (error == Errors.INVALID_PRODUCER_EPOCH) {
+ transitionTo(State.FENCED, error.exception());
+ log.error("Epoch has become invalid: producerId: {}. epoch: {}. Message: {}", pidAndEpoch.producerId, pidAndEpoch.epoch, error.message());
+ } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) {
+ transitionTo(State.ERROR, error.exception());
+ log.error("No permissions add some partitions to the transaction: {}", error.message());
+ } else {
+ transitionTo(State.ERROR, error.exception());
+ log.error("Could not add partitions to transaction due to unknown error: {}", error.message());
+ }
+ }
+
+ @Override
+ public void reenqueue() {
+ pendingTransactionalRequests.add(addPartitionsToTransactionRequest(true));
+ }
+ }
+
+ private class FindCoordinatorCallback extends TransactionalRequestCallBack {
+ private final FindCoordinatorRequest.CoordinatorType type;
+ private final String coordinatorKey;
+
+ FindCoordinatorCallback(FindCoordinatorRequest.CoordinatorType type, String coordinatorKey) {
+ super(null);
+ this.type = type;
+ this.coordinatorKey = coordinatorKey;
+ }
+ @Override
+ public void handleResponse(AbstractResponse responseBody) {
+ FindCoordinatorResponse findCoordinatorResponse = (FindCoordinatorResponse) responseBody;
+ if (findCoordinatorResponse.error() == Errors.NONE) {
+ Node node = findCoordinatorResponse.node();
+ switch (type) {
+ case GROUP:
+ consumerGroupCoordinator = node;
+ break;
+ case TRANSACTION:
+ transactionCoordinator = node;
+ }
+ } else if (findCoordinatorResponse.error() == Errors.COORDINATOR_NOT_AVAILABLE) {
+ reenqueue();
+ } else if (findCoordinatorResponse.error() == Errors.GROUP_AUTHORIZATION_FAILED) {
+ transitionTo(State.ERROR, findCoordinatorResponse.error().exception());
+ log.error("Not authorized to access the group with type {} and key {}. Message: {} ", type,
+ coordinatorKey, findCoordinatorResponse.error().message());
+ } else {
+ transitionTo(State.ERROR, findCoordinatorResponse.error().exception());
+ log.error("Could not find a coordinator with type {} for unknown reasons. coordinatorKey: {}", type,
+ coordinatorKey, findCoordinatorResponse.error().message());
+ }
+ }
+
+ @Override
+ public void reenqueue() {
+ pendingTransactionalRequests.add(findCoordinatorRequest(type, coordinatorKey, true));
+ }
+ }
+
+ private class EndTxnCallback extends TransactionalRequestCallBack {
+ private final boolean isCommit;
+
+ EndTxnCallback(boolean isCommit, TransactionalRequestResult result) {
+ super(result);
+ this.isCommit = isCommit;
+ }
+
+ @Override
+ public void handleResponse(AbstractResponse responseBody) {
+ EndTxnResponse endTxnResponse = (EndTxnResponse) responseBody;
+ Errors error = endTxnResponse.error();
+ if (error == Errors.NONE) {
+ completeTransaction();
+ } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) {
+ needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ reenqueue();
+ } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) {
+ reenqueue();
+ } else if (error == Errors.INVALID_PRODUCER_EPOCH) {
+ transitionTo(State.FENCED, error.exception());
+ result.setError(error.exception());
+ } else {
+ result.setError(error.exception());
+ transitionTo(State.ERROR, error.exception());
+ }
+
+ if (error == Errors.NONE || !result.isSuccessful())
+ result.done();
+ }
+
+ @Override
+ public void reenqueue() {
+ pendingTransactionalRequests.add(endTxnRequest(isCommit, true, result));
+ }
+ }
+
+ private class AddOffsetsToTxnCallback extends TransactionalRequestCallBack {
+ String consumerGroupId;
+ Map<TopicPartition, OffsetAndMetadata> offsets;
+
+ AddOffsetsToTxnCallback(Map<TopicPartition, OffsetAndMetadata> offsets, String consumerGroupId, TransactionalRequestResult result) {
+ super(result);
+ this.offsets = offsets;
+ this.consumerGroupId = consumerGroupId;
+ }
+
+ @Override
+ public void handleResponse(AbstractResponse responseBody) {
+ AddOffsetsToTxnResponse addOffsetsToTxnResponse = (AddOffsetsToTxnResponse) responseBody;
+ Errors error = addOffsetsToTxnResponse.error();
+ if (error == Errors.NONE) {
+ pendingTransactionalRequests.add(txnOffsetCommitRequest(offsets, consumerGroupId, false, result));
+ } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) {
+ needsCoordinator(FindCoordinatorRequest.CoordinatorType.TRANSACTION, transactionalId);
+ reenqueue();
+ } else if (error == Errors.COORDINATOR_LOAD_IN_PROGRESS || error == Errors.CONCURRENT_TRANSACTIONS) {
+ reenqueue();
+ } else if (error == Errors.INVALID_PRODUCER_EPOCH) {
+ transitionTo(State.FENCED, error.exception());
+ result.setError(error.exception());
+ } else {
+ transitionTo(State.ERROR, error.exception());
+ result.setError(error.exception());
+ }
+
+ if (!result.isSuccessful())
+ result.done();
+ }
+
+ @Override
+ public void reenqueue() {
+ pendingTransactionalRequests.add(addOffsetsToTxnRequest(offsets, consumerGroupId, true, result));
+ }
+ }
+
+ private class TxnOffsetCommitCallback extends TransactionalRequestCallBack {
+ private final String consumerGroupId;
+
+ TxnOffsetCommitCallback(String consumerGroupId, TransactionalRequestResult result) {
+ super(result);
+ this.consumerGroupId = consumerGroupId;
+ }
+
+ @Override
+ public void handleResponse(AbstractResponse responseBody) {
+ TxnOffsetCommitResponse txnOffsetCommitResponse = (TxnOffsetCommitResponse) responseBody;
+ boolean coordinatorReloaded = false;
+ boolean hadFailure = false;
+ for (Map.Entry<TopicPartition, Errors> entry : txnOffsetCommitResponse.errors().entrySet()) {
+ TopicPartition topicPartition = entry.getKey();
+ Errors error = entry.getValue();
+ if (error == Errors.NONE) {
+ pendingTxnOffsetCommits.remove(topicPartition);
+ } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) {
+ hadFailure = true;
+ if (!coordinatorReloaded) {
+ coordinatorReloaded = true;
+ needsCoordinator(FindCoordinatorRequest.CoordinatorType.GROUP, consumerGroupId);
+ }
+ } else if (error == Errors.INVALID_PRODUCER_EPOCH) {
+ transitionTo(State.FENCED, error.exception());
+ result.setError(error.exception());
+ break;
+ } else {
+ result.setError(error.exception());
+ transitionTo(State.ERROR, error.exception());
+ break;
+ }
+ }
+
+ if (!hadFailure || !result.isSuccessful()) {
+ // all attempted partitions were either successful, or there was a fatal failure.
+ // either way, we are not retrying, so complete the request.
+ result.done();
+ return;
+ }
+
+ // retry the commits which failed with a retriable error.
+ if (!pendingTxnOffsetCommits.isEmpty())
+ pendingTransactionalRequests.add(txnOffsetCommitRequest(consumerGroupId, true, result));
+
+ }
+
+ @Override
+ public void reenqueue() {
+ pendingTransactionalRequests.add(txnOffsetCommitRequest(consumerGroupId, true, result));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java
new file mode 100644
index 0000000..88f146b
--- /dev/null
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionalRequestResult.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.clients.producer.internals;
+
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public final class TransactionalRequestResult {
+ private final CountDownLatch latch = new CountDownLatch(1);
+ private RuntimeException error = null;
+
+ public void setError(RuntimeException error) {
+ this.error = error;
+ }
+
+ public void done() {
+ this.latch.countDown();
+ }
+
+ public void await() {
+ boolean completed = false;
+
+ while (!completed) {
+ try {
+ latch.await();
+ completed = true;
+ } catch (InterruptedException e) {
+ // Keep waiting until done, we have no other option for these transactional requests.
+ }
+ }
+ }
+
+ public boolean await(long timeout, TimeUnit unit) throws InterruptedException {
+ return latch.await(timeout, unit);
+ }
+
+ public RuntimeException error() {
+ return error;
+ }
+
+ public boolean isSuccessful() {
+ return error == null;
+ }
+
+ public boolean isCompleted() {
+ return latch.getCount() == 0L;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
index aac5b53..5afc9cf 100644
--- a/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
+++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigDef.java
@@ -899,6 +899,16 @@ public class ConfigDef {
}
}
+ public static class NonEmptyString implements Validator {
+ @Override
+ public void ensureValid(String name, Object o) {
+ String s = (String) o;
+ if (s != null && s.isEmpty()) {
+ throw new ConfigException(name, o, "String must be non-empty");
+ }
+ }
+ }
+
public static class ConfigKey {
public final String name;
public final Type type;
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
index e7e155f..548cd45 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecords.java
@@ -177,7 +177,7 @@ public class MemoryRecords extends AbstractRecords {
long logAppendTime = timestampType == TimestampType.LOG_APPEND_TIME ? batch.maxTimestamp() : RecordBatch.NO_TIMESTAMP;
MemoryRecordsBuilder builder = builder(slice, batch.magic(), batch.compressionType(), timestampType,
firstOffset, logAppendTime, batch.producerId(), batch.producerEpoch(), batch.baseSequence(),
- batch.partitionLeaderEpoch());
+ batch.isTransactional(), batch.partitionLeaderEpoch());
for (Record record : retainedRecords)
builder.append(record);
@@ -297,10 +297,7 @@ public class MemoryRecords extends AbstractRecords {
CompressionType compressionType,
TimestampType timestampType,
long baseOffset) {
- long logAppendTime = RecordBatch.NO_TIMESTAMP;
- if (timestampType == TimestampType.LOG_APPEND_TIME)
- logAppendTime = System.currentTimeMillis();
- return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime);
+ return builder(buffer, magic, compressionType, timestampType, baseOffset, false);
}
public static MemoryRecordsBuilder builder(ByteBuffer buffer,
@@ -310,7 +307,20 @@ public class MemoryRecords extends AbstractRecords {
long baseOffset,
long logAppendTime) {
return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime,
- RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, RecordBatch.NO_PARTITION_LEADER_EPOCH);
+ RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, RecordBatch.NO_PARTITION_LEADER_EPOCH);
+ }
+
+ public static MemoryRecordsBuilder builder(ByteBuffer buffer,
+ byte magic,
+ CompressionType compressionType,
+ TimestampType timestampType,
+ long baseOffset,
+ boolean isTransactional) {
+ long logAppendTime = RecordBatch.NO_TIMESTAMP;
+ if (timestampType == TimestampType.LOG_APPEND_TIME)
+ logAppendTime = System.currentTimeMillis();
+ return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime,
+ RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, isTransactional, RecordBatch.NO_PARTITION_LEADER_EPOCH);
}
public static MemoryRecordsBuilder builder(ByteBuffer buffer,
@@ -321,7 +331,7 @@ public class MemoryRecords extends AbstractRecords {
long logAppendTime,
int partitionLeaderEpoch) {
return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime,
- RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, partitionLeaderEpoch);
+ RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, false, partitionLeaderEpoch);
}
public static MemoryRecordsBuilder builder(ByteBuffer buffer,
@@ -334,7 +344,7 @@ public class MemoryRecords extends AbstractRecords {
short producerEpoch,
int baseSequence) {
return builder(buffer, magic, compressionType, timestampType, baseOffset, logAppendTime,
- producerId, producerEpoch, baseSequence, RecordBatch.NO_PARTITION_LEADER_EPOCH);
+ producerId, producerEpoch, baseSequence, false, RecordBatch.NO_PARTITION_LEADER_EPOCH);
}
public static MemoryRecordsBuilder builder(ByteBuffer buffer,
@@ -346,12 +356,14 @@ public class MemoryRecords extends AbstractRecords {
long producerId,
short producerEpoch,
int baseSequence,
+ boolean isTransactional,
int partitionLeaderEpoch) {
return new MemoryRecordsBuilder(buffer, magic, compressionType, timestampType, baseOffset,
- logAppendTime, producerId, producerEpoch, baseSequence, false, partitionLeaderEpoch,
+ logAppendTime, producerId, producerEpoch, baseSequence, isTransactional, partitionLeaderEpoch,
buffer.remaining());
}
+
public static MemoryRecords withRecords(CompressionType compressionType, SimpleRecord... records) {
return withRecords(RecordBatch.CURRENT_MAGIC_VALUE, compressionType, records);
}
@@ -398,7 +410,7 @@ public class MemoryRecords extends AbstractRecords {
if (timestampType == TimestampType.LOG_APPEND_TIME)
logAppendTime = System.currentTimeMillis();
MemoryRecordsBuilder builder = builder(buffer, magic, compressionType, timestampType, initialOffset,
- logAppendTime, producerId, producerEpoch, baseSequence, partitionLeaderEpoch);
+ logAppendTime, producerId, producerEpoch, baseSequence, false, partitionLeaderEpoch);
for (SimpleRecord record : records)
builder.append(record);
return builder.build();
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
index 549804a..eb52134 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java
@@ -117,23 +117,10 @@ public class MemoryRecordsBuilder {
throw new IllegalArgumentException("TimestampType must be set for magic >= 0");
if (isTransactional) {
- if (producerId == RecordBatch.NO_PRODUCER_ID)
- throw new IllegalArgumentException("Cannot write transactional messages without a valid producer ID");
-
if (magic < RecordBatch.MAGIC_VALUE_V2)
throw new IllegalArgumentException("Transactional messages are not supported for magic " + magic);
}
- if (producerId != RecordBatch.NO_PRODUCER_ID) {
- if (producerEpoch < 0)
- throw new IllegalArgumentException("Invalid negative producer epoch");
-
- if (baseSequence < 0)
- throw new IllegalArgumentException("Invalid negative sequence number used");
-
- if (magic < RecordBatch.MAGIC_VALUE_V2)
- throw new IllegalArgumentException("Idempotent messages are not supported for magic " + magic);
- }
this.magic = magic;
this.timestampType = timestampType;
@@ -259,6 +246,20 @@ public class MemoryRecordsBuilder {
if (builtRecords != null)
return;
+ if (isTransactional && producerId == RecordBatch.NO_PRODUCER_ID)
+ throw new IllegalArgumentException("Cannot write transactional messages without a valid producer ID");
+
+ if (producerId != RecordBatch.NO_PRODUCER_ID) {
+ if (producerEpoch == RecordBatch.NO_PRODUCER_EPOCH)
+ throw new IllegalArgumentException("Invalid negative producer epoch");
+
+ if (baseSequence == RecordBatch.NO_SEQUENCE)
+ throw new IllegalArgumentException("Invalid negative sequence number used");
+
+ if (magic < RecordBatch.MAGIC_VALUE_V2)
+ throw new IllegalArgumentException("Idempotent messages are not supported for magic " + magic);
+ }
+
closeForRecordAppends();
if (numRecords == 0L) {
@@ -659,4 +660,8 @@ public class MemoryRecordsBuilder {
public long producerId() {
return this.producerId;
}
+
+ public short producerEpoch() {
+ return this.producerEpoch;
+ }
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java
index 6ac49fb..2426514 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java
@@ -32,6 +32,7 @@ public class AddOffsetsToTxnResponse extends AbstractResponse {
// InvalidPidMapping
// InvalidTxnState
// GroupAuthorizationFailed
+ // InvalidProducerEpoch
private final Errors error;
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
index 3de6295..0337044 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java
@@ -32,6 +32,7 @@ public class AddPartitionsToTxnResponse extends AbstractResponse {
// InvalidTxnState
// InvalidPidMapping
// TopicAuthorizationFailed
+ // InvalidProducerEpoch
private final Errors error;
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java
index 627eb64..3ff6aca 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java
@@ -31,6 +31,7 @@ public class EndTxnResponse extends AbstractResponse {
// CoordinatorLoadInProgress
// InvalidTxnState
// InvalidPidMapping
+ // InvalidProducerEpoch
private final Errors error;
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
index 8cd2818..78715be 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java
@@ -104,24 +104,30 @@ public class FetchRequest extends AbstractRequest {
private final int minBytes;
private final int replicaId;
private final LinkedHashMap<TopicPartition, PartitionData> fetchData;
+ private final IsolationLevel isolationLevel;
private int maxBytes = DEFAULT_RESPONSE_MAX_BYTES;
public static Builder forConsumer(int maxWait, int minBytes, LinkedHashMap<TopicPartition, PartitionData> fetchData) {
- return new Builder(null, CONSUMER_REPLICA_ID, maxWait, minBytes, fetchData);
+ return new Builder(null, CONSUMER_REPLICA_ID, maxWait, minBytes, fetchData, IsolationLevel.READ_UNCOMMITTED);
+ }
+
+ public static Builder forConsumer(int maxWait, int minBytes, LinkedHashMap<TopicPartition, PartitionData> fetchData, IsolationLevel isolationLevel) {
+ return new Builder(null, CONSUMER_REPLICA_ID, maxWait, minBytes, fetchData, isolationLevel);
}
public static Builder forReplica(short desiredVersion, int replicaId, int maxWait, int minBytes,
LinkedHashMap<TopicPartition, PartitionData> fetchData) {
- return new Builder(desiredVersion, replicaId, maxWait, minBytes, fetchData);
+ return new Builder(desiredVersion, replicaId, maxWait, minBytes, fetchData, IsolationLevel.READ_UNCOMMITTED);
}
private Builder(Short desiredVersion, int replicaId, int maxWait, int minBytes,
- LinkedHashMap<TopicPartition, PartitionData> fetchData) {
+ LinkedHashMap<TopicPartition, PartitionData> fetchData, IsolationLevel isolationLevel) {
super(ApiKeys.FETCH, desiredVersion);
this.replicaId = replicaId;
this.maxWait = maxWait;
this.minBytes = minBytes;
this.fetchData = fetchData;
+ this.isolationLevel = isolationLevel;
}
public LinkedHashMap<TopicPartition, PartitionData> fetchData() {
@@ -139,7 +145,7 @@ public class FetchRequest extends AbstractRequest {
maxBytes = DEFAULT_RESPONSE_MAX_BYTES;
}
- return new FetchRequest(version, replicaId, maxWait, minBytes, maxBytes, fetchData);
+ return new FetchRequest(version, replicaId, maxWait, minBytes, maxBytes, fetchData, IsolationLevel.READ_UNCOMMITTED);
}
@Override
@@ -158,13 +164,18 @@ public class FetchRequest extends AbstractRequest {
private FetchRequest(short version, int replicaId, int maxWait, int minBytes, int maxBytes,
LinkedHashMap<TopicPartition, PartitionData> fetchData) {
+ this(version, replicaId, maxWait, minBytes, maxBytes, fetchData, IsolationLevel.READ_UNCOMMITTED);
+ }
+
+ private FetchRequest(short version, int replicaId, int maxWait, int minBytes, int maxBytes,
+ LinkedHashMap<TopicPartition, PartitionData> fetchData, IsolationLevel isolationLevel) {
super(version);
this.replicaId = replicaId;
this.maxWait = maxWait;
this.minBytes = minBytes;
this.maxBytes = maxBytes;
this.fetchData = fetchData;
- this.isolationLevel = IsolationLevel.READ_UNCOMMITTED;
+ this.isolationLevel = isolationLevel;
}
public FetchRequest(Struct struct, short version) {
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
----------------------------------------------------------------------
diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
index 7631391..482811e 100644
--- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
+++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java
@@ -55,16 +55,26 @@ public class ProduceRequest extends AbstractRequest {
private final short acks;
private final int timeout;
private final Map<TopicPartition, MemoryRecords> partitionRecords;
+ private final String transactionalId;
public Builder(byte magic,
short acks,
int timeout,
- Map<TopicPartition, MemoryRecords> partitionRecords) {
+ Map<TopicPartition, MemoryRecords> partitionRecords,
+ String transactionalId) {
super(ApiKeys.PRODUCE, (short) (magic == RecordBatch.MAGIC_VALUE_V2 ? 3 : 2));
this.magic = magic;
this.acks = acks;
this.timeout = timeout;
this.partitionRecords = partitionRecords;
+ this.transactionalId = transactionalId;
+ }
+
+ public Builder(byte magic,
+ short acks,
+ int timeout,
+ Map<TopicPartition, MemoryRecords> partitionRecords) {
+ this(magic, acks, timeout, partitionRecords, null);
}
@Override
@@ -72,7 +82,7 @@ public class ProduceRequest extends AbstractRequest {
if (version < 2)
throw new UnsupportedVersionException("ProduceRequest versions older than 2 are not supported.");
- return new ProduceRequest(version, acks, timeout, partitionRecords);
+ return new ProduceRequest(version, acks, timeout, partitionRecords, transactionalId);
}
@Override
@@ -83,7 +93,8 @@ public class ProduceRequest extends AbstractRequest {
.append(", acks=").append(acks)
.append(", timeout=").append(timeout)
.append(", partitionRecords=(").append(partitionRecords)
- .append("))");
+ .append("), transactionalId='").append(transactionalId != null ? transactionalId : "")
+ .append("'");
return bld.toString();
}
}
@@ -99,13 +110,12 @@ public class ProduceRequest extends AbstractRequest {
// Care should be taken in methods that use this field.
private volatile Map<TopicPartition, MemoryRecords> partitionRecords;
- private ProduceRequest(short version, short acks, int timeout, Map<TopicPartition, MemoryRecords> partitionRecords) {
+ private ProduceRequest(short version, short acks, int timeout, Map<TopicPartition, MemoryRecords> partitionRecords, String transactionalId) {
super(version);
this.acks = acks;
this.timeout = timeout;
- // TODO: Include transactional id in constructor once transactions are supported
- this.transactionalId = null;
+ this.transactionalId = transactionalId;
this.partitionRecords = partitionRecords;
this.partitionSizes = createPartitionSizes(partitionRecords);
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index b9600b4..8e49350 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -49,6 +49,7 @@ import org.apache.kafka.common.requests.FetchResponse;
import org.apache.kafka.common.requests.FetchResponse.PartitionData;
import org.apache.kafka.common.requests.FindCoordinatorResponse;
import org.apache.kafka.common.requests.HeartbeatResponse;
+import org.apache.kafka.common.requests.IsolationLevel;
import org.apache.kafka.common.requests.JoinGroupRequest;
import org.apache.kafka.common.requests.JoinGroupResponse;
import org.apache.kafka.common.requests.LeaveGroupResponse;
@@ -1548,7 +1549,8 @@ public class KafkaConsumerTest {
metrics,
metricGroupPrefix,
time,
- retryBackoffMs);
+ retryBackoffMs,
+ IsolationLevel.READ_UNCOMMITTED);
return new KafkaConsumer<>(
clientId,
@@ -1563,7 +1565,8 @@ public class KafkaConsumerTest {
subscriptions,
metadata,
retryBackoffMs,
- requestTimeoutMs);
+ requestTimeoutMs
+ );
}
private static class FetchInfo {
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
index b8f493a..6059180 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java
@@ -48,6 +48,8 @@ import org.apache.kafka.common.record.MemoryRecordsBuilder;
import org.apache.kafka.common.record.Record;
import org.apache.kafka.common.record.RecordBatch;
import org.apache.kafka.common.record.SimpleRecord;
+import org.apache.kafka.common.requests.IsolationLevel;
+import org.apache.kafka.common.utils.ByteBufferOutputStream;
import org.apache.kafka.common.record.TimestampType;
import org.apache.kafka.common.requests.AbstractRequest;
import org.apache.kafka.common.requests.ApiVersionsResponse;
@@ -59,7 +61,6 @@ import org.apache.kafka.common.requests.MetadataRequest;
import org.apache.kafka.common.requests.MetadataResponse;
import org.apache.kafka.common.serialization.ByteArrayDeserializer;
import org.apache.kafka.common.serialization.Deserializer;
-import org.apache.kafka.common.utils.ByteBufferOutputStream;
import org.apache.kafka.common.utils.MockTime;
import org.apache.kafka.common.utils.Utils;
import org.apache.kafka.test.TestUtils;
@@ -69,13 +70,16 @@ import org.junit.Test;
import java.io.DataOutputStream;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static java.util.Collections.singleton;
import static org.junit.Assert.assertArrayEquals;
@@ -1105,6 +1109,292 @@ public class FetcherTest {
testGetOffsetsForTimesWithError(Errors.BROKER_NOT_AVAILABLE, Errors.NONE, 10L, 100L, 10L, 100L);
}
+ @Test
+ public void testSkippingAbortedTransactions() {
+ Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(),
+ new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+ int currentOffset = 0;
+
+ currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset,
+ new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()),
+ new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()));
+
+ currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds());
+
+ buffer.flip();
+
+ List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>();
+ abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0));
+ MemoryRecords records = MemoryRecords.readableRecords(buffer);
+ subscriptions.assignFromUser(singleton(tp1));
+
+ subscriptions.seek(tp1, 0);
+
+ // normal fetch
+ assertEquals(1, fetcher.sendFetches());
+ assertFalse(fetcher.hasCompletedFetches());
+
+ client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
+ consumerClient.poll(0);
+ assertTrue(fetcher.hasCompletedFetches());
+
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords();
+ assertFalse(fetchedRecords.containsKey(tp1));
+ }
+
+ @Test
+ public void testReturnCommittedTransactions() {
+ Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(),
+ new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+ int currentOffset = 0;
+
+ currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset,
+ new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()),
+ new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()));
+
+ currentOffset += commitTransaction(buffer, 1L, currentOffset, time.milliseconds());
+ buffer.flip();
+
+ List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>();
+ MemoryRecords records = MemoryRecords.readableRecords(buffer);
+ subscriptions.assignFromUser(singleton(tp1));
+
+ subscriptions.seek(tp1, 0);
+
+ // normal fetch
+ assertEquals(1, fetcher.sendFetches());
+ assertFalse(fetcher.hasCompletedFetches());
+
+ client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
+ consumerClient.poll(0);
+ assertTrue(fetcher.hasCompletedFetches());
+
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords();
+ assertTrue(fetchedRecords.containsKey(tp1));
+ assertEquals(fetchedRecords.get(tp1).size(), 2);
+ }
+
+ @Test
+ public void testWithCommittedAndAbortedTransactions() {
+ Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(),
+ new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+
+ List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>();
+
+ int currOffset = 0;
+ // Appends for producer 1 (evetually committed)
+ currOffset += appendTransactionalRecords(buffer, 1L, currOffset,
+ new SimpleRecord(time.milliseconds(), "commit1-1".getBytes(), "value".getBytes()),
+ new SimpleRecord(time.milliseconds(), "commit1-2".getBytes(), "value".getBytes()));
+
+ // Appends for producer 2 (eventually aborted)
+ currOffset += appendTransactionalRecords(buffer, 2L, currOffset,
+ new SimpleRecord(time.milliseconds(), "abort2-1".getBytes(), "value".getBytes()));
+
+ // commit producer 1
+ currOffset += commitTransaction(buffer, 1L, currOffset, time.milliseconds());
+ // append more for producer 2 (eventually aborted)
+ currOffset += appendTransactionalRecords(buffer, 2L, currOffset,
+ new SimpleRecord(time.milliseconds(), "abort2-2".getBytes(), "value".getBytes()));
+
+ // abort producer 2
+ currOffset += abortTransaction(buffer, 2L, currOffset, time.milliseconds());
+ abortedTransactions.add(new FetchResponse.AbortedTransaction(2, 2));
+
+ // New transaction for producer 1 (eventually aborted)
+ currOffset += appendTransactionalRecords(buffer, 1L, currOffset,
+ new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()));
+
+ // New transaction for producer 2 (eventually committed)
+ currOffset += appendTransactionalRecords(buffer, 2L, currOffset,
+ new SimpleRecord(time.milliseconds(), "commit2-1".getBytes(), "value".getBytes()));
+
+ // Add messages for producer 1 (eventually aborted)
+ currOffset += appendTransactionalRecords(buffer, 1L, currOffset,
+ new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes()));
+
+ // abort producer 1
+ currOffset += abortTransaction(buffer, 1L, currOffset, time.milliseconds());
+ abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 6));
+
+ // commit producer 2
+ currOffset += commitTransaction(buffer, 2L, currOffset, time.milliseconds());
+
+ buffer.flip();
+
+ MemoryRecords records = MemoryRecords.readableRecords(buffer);
+ subscriptions.assignFromUser(singleton(tp1));
+
+ subscriptions.seek(tp1, 0);
+
+ // normal fetch
+ assertEquals(1, fetcher.sendFetches());
+ assertFalse(fetcher.hasCompletedFetches());
+
+ client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
+ consumerClient.poll(0);
+ assertTrue(fetcher.hasCompletedFetches());
+
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords();
+ assertTrue(fetchedRecords.containsKey(tp1));
+ // There are only 3 committed records
+ List<ConsumerRecord<byte[], byte[]>> fetchedConsumerRecords = fetchedRecords.get(tp1);
+ Set<String> committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2", "commit2-1"));
+ Set<String> actuallyCommittedKeys = new HashSet<>();
+ for (ConsumerRecord<byte[], byte[]> consumerRecord : fetchedConsumerRecords) {
+ actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8));
+ }
+ assertTrue(actuallyCommittedKeys.equals(committedKeys));
+ }
+
+ @Test
+ public void testMultipleAbortMarkers() {
+ Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(),
+ new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+ int currentOffset = 0;
+
+ currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset,
+ new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()),
+ new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes()));
+
+ currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds());
+ // Duplicate abort -- should be ignored.
+ currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds());
+ // Now commit a transaction.
+ currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset,
+ new SimpleRecord(time.milliseconds(), "commit1-1".getBytes(), "value".getBytes()),
+ new SimpleRecord(time.milliseconds(), "commit1-2".getBytes(), "value".getBytes()));
+ currentOffset += commitTransaction(buffer, 1L, currentOffset, time.milliseconds());
+ buffer.flip();
+
+ List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>();
+ abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0));
+ MemoryRecords records = MemoryRecords.readableRecords(buffer);
+ subscriptions.assignFromUser(singleton(tp1));
+
+ subscriptions.seek(tp1, 0);
+
+ // normal fetch
+ assertEquals(1, fetcher.sendFetches());
+ assertFalse(fetcher.hasCompletedFetches());
+
+ client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
+ consumerClient.poll(0);
+ assertTrue(fetcher.hasCompletedFetches());
+
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords();
+ assertTrue(fetchedRecords.containsKey(tp1));
+ assertEquals(fetchedRecords.get(tp1).size(), 2);
+ List<ConsumerRecord<byte[], byte[]>> fetchedConsumerRecords = fetchedRecords.get(tp1);
+ Set<String> committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2"));
+ Set<String> actuallyCommittedKeys = new HashSet<>();
+ for (ConsumerRecord<byte[], byte[]> consumerRecord : fetchedConsumerRecords) {
+ actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8));
+ }
+ assertTrue(actuallyCommittedKeys.equals(committedKeys));
+ }
+
+ @Test
+ public void testReturnAbortedTransactionsinUncommittedMode() {
+ Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(),
+ new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED);
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+ int currentOffset = 0;
+
+ currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset,
+ new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()),
+ new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()));
+
+ currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds());
+
+ buffer.flip();
+
+ List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>();
+ abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0));
+ MemoryRecords records = MemoryRecords.readableRecords(buffer);
+ subscriptions.assignFromUser(singleton(tp1));
+
+ subscriptions.seek(tp1, 0);
+
+ // normal fetch
+ assertEquals(1, fetcher.sendFetches());
+ assertFalse(fetcher.hasCompletedFetches());
+
+ client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
+ consumerClient.poll(0);
+ assertTrue(fetcher.hasCompletedFetches());
+
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords();
+ assertTrue(fetchedRecords.containsKey(tp1));
+ }
+
+ @Test
+ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() {
+ Fetcher<byte[], byte[]> fetcher = createFetcher(subscriptions, new Metrics(), new ByteArrayDeserializer(),
+ new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+ long currentOffset = 0;
+
+ currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset,
+ new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()),
+ new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes()));
+
+ currentOffset += abortTransaction(buffer, 1L, currentOffset, time.milliseconds());
+ buffer.flip();
+
+ List<FetchResponse.AbortedTransaction> abortedTransactions = new ArrayList<>();
+ abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0));
+ MemoryRecords records = MemoryRecords.readableRecords(buffer);
+ subscriptions.assignFromUser(singleton(tp1));
+
+ subscriptions.seek(tp1, 0);
+
+ // normal fetch
+ assertEquals(1, fetcher.sendFetches());
+ assertFalse(fetcher.hasCompletedFetches());
+
+ client.prepareResponse(fetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0));
+ consumerClient.poll(0);
+ assertTrue(fetcher.hasCompletedFetches());
+
+ Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetcher.fetchedRecords();
+
+ // Ensure that we don't return any of the aborted records, but yet advance the consumer position.
+ assertFalse(fetchedRecords.containsKey(tp1));
+ assertEquals(currentOffset, (long) subscriptions.position(tp1));
+ }
+
+ private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOffset, SimpleRecord... records) {
+ MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE,
+ TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, (int) baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH);
+
+ for (SimpleRecord record : records) {
+ builder.append(record);
+ }
+ builder.build();
+ return records.length;
+ }
+
+ private int commitTransaction(ByteBuffer buffer, long pid, int baseOffset, long timestamp) {
+ MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE,
+ TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH);
+ builder.appendControlRecord(timestamp, ControlRecordType.COMMIT, null);
+ builder.build();
+ return 1;
+ }
+
+ private int abortTransaction(ByteBuffer buffer, long pid, long baseOffset, long timestamp) {
+ MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE,
+ TimestampType.LOG_APPEND_TIME, baseOffset, time.milliseconds(), pid, (short) 0, (int) baseOffset, true, RecordBatch.NO_PARTITION_LEADER_EPOCH);
+ builder.appendControlRecord(timestamp, ControlRecordType.ABORT, null);
+ builder.build();
+ return 1;
+ }
+
private void testGetOffsetsForTimesWithError(Errors errorForTp0,
Errors errorForTp1,
long offsetForTp0,
@@ -1167,6 +1457,15 @@ public class FetcherTest {
return new ListOffsetResponse(allPartitionData);
}
+ private FetchResponse fetchResponseWithAbortedTransactions(MemoryRecords records,
+ List<FetchResponse.AbortedTransaction> abortedTransactions,
+ Errors error,
+ long lastStableOffset, long hw, int throttleTime) {
+ Map<TopicPartition, FetchResponse.PartitionData> partitions = Collections.singletonMap(tp1,
+ new FetchResponse.PartitionData(error, hw, lastStableOffset, 0L, abortedTransactions, records));
+ return new FetchResponse(new LinkedHashMap<>(partitions), throttleTime);
+ }
+
private FetchResponse fetchResponse(MemoryRecords records, Errors error, long hw, int throttleTime) {
Map<TopicPartition, FetchResponse.PartitionData> partitions = Collections.singletonMap(tp1,
new FetchResponse.PartitionData(error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records));
@@ -1193,7 +1492,8 @@ public class FetcherTest {
private Fetcher<byte[], byte[]> createFetcher(SubscriptionState subscriptions,
Metrics metrics,
int maxPollRecords) {
- return createFetcher(subscriptions, metrics, new ByteArrayDeserializer(), new ByteArrayDeserializer(), maxPollRecords);
+ return createFetcher(subscriptions, metrics, new ByteArrayDeserializer(), new ByteArrayDeserializer(),
+ maxPollRecords, IsolationLevel.READ_UNCOMMITTED);
}
private Fetcher<byte[], byte[]> createFetcher(SubscriptionState subscriptions, Metrics metrics) {
@@ -1204,14 +1504,16 @@ public class FetcherTest {
Metrics metrics,
Deserializer<K> keyDeserializer,
Deserializer<V> valueDeserializer) {
- return createFetcher(subscriptions, metrics, keyDeserializer, valueDeserializer, Integer.MAX_VALUE);
+ return createFetcher(subscriptions, metrics, keyDeserializer, valueDeserializer, Integer.MAX_VALUE,
+ IsolationLevel.READ_UNCOMMITTED);
}
private <K, V> Fetcher<K, V> createFetcher(SubscriptionState subscriptions,
Metrics metrics,
Deserializer<K> keyDeserializer,
Deserializer<V> valueDeserializer,
- int maxPollRecords) {
+ int maxPollRecords,
+ IsolationLevel isolationLevel) {
return new Fetcher<>(consumerClient,
minBytes,
maxBytes,
@@ -1226,7 +1528,8 @@ public class FetcherTest {
metrics,
"consumer" + groupId,
time,
- retryBackoffMs);
+ retryBackoffMs,
+ isolationLevel);
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/a82f194b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
----------------------------------------------------------------------
diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
index d152117..0f1cb54 100644
--- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/RecordAccumulatorTest.java
@@ -20,7 +20,6 @@ import org.apache.kafka.clients.ApiVersions;
import org.apache.kafka.clients.NodeApiVersions;
import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.RecordMetadata;
-import org.apache.kafka.clients.producer.TransactionState;
import org.apache.kafka.common.Cluster;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
@@ -549,7 +548,7 @@ public class RecordAccumulatorTest {
apiVersions.update("foobar", NodeApiVersions.create(Arrays.asList(new ApiVersionsResponse.ApiVersion(ApiKeys.PRODUCE.id,
(short) 0, (short) 2))));
RecordAccumulator accum = new RecordAccumulator(batchSize + DefaultRecordBatch.RECORD_BATCH_OVERHEAD, 10 * batchSize,
- CompressionType.NONE, 10, 100L, metrics, time, apiVersions, new TransactionState(time));
+ CompressionType.NONE, 10, 100L, metrics, time, apiVersions, new TransactionManager());
accum.append(tp1, 0L, key, value, null, 0);
}