You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by ac...@apache.org on 2023/12/19 10:41:18 UTC
(camel) branch main updated: CAMEL-20250: AWS Kinesis consumer - add KinesisResumeStrategy (#12462)
This is an automated email from the ASF dual-hosted git repository.
acosentino pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new a65705d76a7 CAMEL-20250: AWS Kinesis consumer - add KinesisResumeStrategy (#12462)
a65705d76a7 is described below
commit a65705d76a797c3640102c08b9472d963c7c0fdc
Author: Alphonse Bendt <37...@users.noreply.github.com>
AuthorDate: Tue Dec 19 11:41:11 2023 +0100
CAMEL-20250: AWS Kinesis consumer - add KinesisResumeStrategy (#12462)
---
.../org/apache/camel/kinesis-resume-strategy | 2 +
.../component/aws2/kinesis/Kinesis2Consumer.java | 17 ++--
...er.java => KinesisConsumerOffsetProcessor.java} | 32 +++----
.../consumer/KinesisDefaultResumeAdapter.java | 48 +++-------
.../kinesis/consumer/KinesisResumeAdapter.java | 17 +---
.../kinesis/consumer/KinesisResumeStrategy.java | 89 +++++++++++++++++
.../KinesisResumeStrategyConfiguration.java | 68 +++++++++++++
.../kinesis/integration/KinesisConsumerIT.java | 3 +
.../KinesisConsumerResumeAfterRestartIT.java | 106 +++++++++++++++++++++
.../integration/KinesisConsumerResumeIT.java | 19 ++--
10 files changed, 316 insertions(+), 85 deletions(-)
diff --git a/components/camel-aws/camel-aws2-kinesis/src/generated/resources/META-INF/services/org/apache/camel/kinesis-resume-strategy b/components/camel-aws/camel-aws2-kinesis/src/generated/resources/META-INF/services/org/apache/camel/kinesis-resume-strategy
new file mode 100644
index 00000000000..ab2569b1804
--- /dev/null
+++ b/components/camel-aws/camel-aws2-kinesis/src/generated/resources/META-INF/services/org/apache/camel/kinesis-resume-strategy
@@ -0,0 +1,2 @@
+# Generated by camel build tools - do NOT edit this file!
+class=org.apache.camel.component.aws2.kinesis.consumer.KinesisResumeStrategy
diff --git a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/Kinesis2Consumer.java b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/Kinesis2Consumer.java
index 0e750f8268c..f913870826e 100644
--- a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/Kinesis2Consumer.java
+++ b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/Kinesis2Consumer.java
@@ -161,7 +161,7 @@ public class Kinesis2Consumer extends ScheduledBatchPollingConsumer implements R
}
try {
- Queue<Exchange> exchanges = createExchanges(result.records());
+ Queue<Exchange> exchanges = createExchanges(shard, result.records());
processedExchangeCount.getAndSet(processBatch(CastUtils.cast(exchanges)));
} catch (Exception e) {
throw new RuntimeException(e);
@@ -220,7 +220,7 @@ public class Kinesis2Consumer extends ScheduledBatchPollingConsumer implements R
request.startingSequenceNumber(getEndpoint().getConfiguration().getSequenceNumber());
}
- resume(request);
+ resume(shardId, request);
GetShardIteratorResponse result;
if (getEndpoint().getConfiguration().isAsyncClient()) {
@@ -265,7 +265,7 @@ public class Kinesis2Consumer extends ScheduledBatchPollingConsumer implements R
}
}
- private void resume(GetShardIteratorRequest.Builder req) {
+ private void resume(String shardId, GetShardIteratorRequest.Builder req) {
if (resumeStrategy == null) {
return;
}
@@ -277,26 +277,25 @@ public class Kinesis2Consumer extends ScheduledBatchPollingConsumer implements R
return;
}
- adapter.setRequestBuilder(req);
- adapter.setStreamName(getEndpoint().getConfiguration().getStreamName());
- adapter.resume();
+ adapter.configureGetShardIteratorRequest(req, getEndpoint().getConfiguration().getStreamName(), shardId);
}
- private Queue<Exchange> createExchanges(List<Record> records) {
+ private Queue<Exchange> createExchanges(Shard shard, List<Record> records) {
Queue<Exchange> exchanges = new ArrayDeque<>();
for (Record dataRecord : records) {
- exchanges.add(createExchange(dataRecord));
+ exchanges.add(createExchange(shard, dataRecord));
}
return exchanges;
}
- protected Exchange createExchange(Record dataRecord) {
+ protected Exchange createExchange(Shard shard, Record dataRecord) {
LOG.debug("Received Kinesis record with partition_key={}", dataRecord.partitionKey());
Exchange exchange = createExchange(true);
exchange.getIn().setBody(dataRecord.data().asInputStream());
exchange.getIn().setHeader(Kinesis2Constants.APPROX_ARRIVAL_TIME, dataRecord.approximateArrivalTimestamp());
exchange.getIn().setHeader(Kinesis2Constants.PARTITION_KEY, dataRecord.partitionKey());
exchange.getIn().setHeader(Kinesis2Constants.SEQUENCE_NUMBER, dataRecord.sequenceNumber());
+ exchange.getIn().setHeader(Kinesis2Constants.SHARD_ID, shard.shardId());
if (dataRecord.approximateArrivalTimestamp() != null) {
long ts = dataRecord.approximateArrivalTimestamp().getEpochSecond() * 1000;
exchange.getIn().setHeader(Kinesis2Constants.MESSAGE_TIMESTAMP, ts);
diff --git a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeAdapter.java b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisConsumerOffsetProcessor.java
similarity index 57%
copy from components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeAdapter.java
copy to components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisConsumerOffsetProcessor.java
index 8ed7ea914db..71f143d56be 100644
--- a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeAdapter.java
+++ b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisConsumerOffsetProcessor.java
@@ -17,24 +17,18 @@
package org.apache.camel.component.aws2.kinesis.consumer;
-import org.apache.camel.resume.ResumeAdapter;
-import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest;
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.component.aws2.kinesis.Kinesis2Constants;
+import org.apache.camel.support.resume.Resumables;
-/**
- * The resume adapter for Kinesis
- */
-public interface KinesisResumeAdapter extends ResumeAdapter {
- /**
- * Sets the shard iterator request builder that can be used to customize the call and set the exact resume point
- *
- * @param builder the builder instance
- */
- void setRequestBuilder(GetShardIteratorRequest.Builder builder);
-
- /**
- * Sets the stream name being worked on
- *
- * @param streamName the stream name
- */
- void setStreamName(String streamName);
+public class KinesisConsumerOffsetProcessor implements Processor {
+ @Override
+ public void process(Exchange exchange) throws Exception {
+ exchange.getMessage().setHeader(
+ Exchange.OFFSET,
+ Resumables.of(
+ exchange.getMessage().getHeader(Kinesis2Constants.SHARD_ID),
+ exchange.getMessage().getHeader(Kinesis2Constants.SEQUENCE_NUMBER)));
+ }
}
diff --git a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisDefaultResumeAdapter.java b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisDefaultResumeAdapter.java
index 0c472acc8cd..8a1c47a2cf3 100644
--- a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisDefaultResumeAdapter.java
+++ b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisDefaultResumeAdapter.java
@@ -17,10 +17,7 @@
package org.apache.camel.component.aws2.kinesis.consumer;
-import java.nio.ByteBuffer;
-
import org.apache.camel.resume.Cacheable;
-import org.apache.camel.resume.Deserializable;
import org.apache.camel.resume.Offset;
import org.apache.camel.resume.OffsetKey;
import org.apache.camel.resume.ResumeAdapter;
@@ -32,35 +29,14 @@ import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest;
import software.amazon.awssdk.services.kinesis.model.ShardIteratorType;
@JdkService(ResumeAdapter.RESUME_ADAPTER_FACTORY)
-public class KinesisDefaultResumeAdapter implements KinesisResumeAdapter, Cacheable, Deserializable {
+public class KinesisDefaultResumeAdapter implements KinesisResumeAdapter, Cacheable {
private static final Logger LOG = LoggerFactory.getLogger(KinesisDefaultResumeAdapter.class);
private ResumeCache<String> cache;
- private GetShardIteratorRequest.Builder resumable;
- private String streamName;
-
- public void setRequestBuilder(GetShardIteratorRequest.Builder resumable) {
- this.resumable = resumable;
- }
-
@Override
public void resume() {
- assert streamName != null;
- assert resumable != null;
-
- KinesisOffset offset = cache.get(streamName, KinesisOffset.class);
-
- if (offset == null) {
- LOG.info("There is no offset for the stream {}", streamName);
- return;
- }
-
- final String sequenceNumber = offset.getValue();
- LOG.info("Resuming from offset {} for key {}", sequenceNumber, streamName);
-
- resumable.shardIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER);
- resumable.startingSequenceNumber(sequenceNumber);
+ throw new UnsupportedOperationException();
}
private void add(Object key, Object offset) {
@@ -87,17 +63,19 @@ public class KinesisDefaultResumeAdapter implements KinesisResumeAdapter, Cachea
}
@Override
- public boolean deserialize(ByteBuffer keyBuffer, ByteBuffer valueBuffer) {
- Object keyObj = deserializeKey(keyBuffer);
- Object valueObj = deserializeValue(valueBuffer);
+ public void configureGetShardIteratorRequest(GetShardIteratorRequest.Builder builder, String streamName, String shardId) {
+ KinesisOffset offset = cache.get(shardId, KinesisOffset.class);
- add(keyObj, valueObj);
+ if (offset == null) {
+ LOG.info("There is no offset for the stream {}", streamName);
+ return;
+ }
- return true;
- }
+ final String sequenceNumber = offset.getValue();
+ LOG.info("Resuming from offset {} for key {}", sequenceNumber, streamName);
- @Override
- public void setStreamName(String streamName) {
- this.streamName = streamName;
+ builder.shardId(shardId);
+ builder.shardIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER);
+ builder.startingSequenceNumber(sequenceNumber);
}
}
diff --git a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeAdapter.java b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeAdapter.java
index 8ed7ea914db..a9f031f1866 100644
--- a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeAdapter.java
+++ b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeAdapter.java
@@ -24,17 +24,10 @@ import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest;
* The resume adapter for Kinesis
*/
public interface KinesisResumeAdapter extends ResumeAdapter {
- /**
- * Sets the shard iterator request builder that can be used to customize the call and set the exact resume point
- *
- * @param builder the builder instance
+ /*
+ When consuming from multiple shards the KinesisResumeAdapter is potentially accessed by multiple threads.
+ To avoid any concurrency issues the configuration of the GetShardIteratorRequest should be done in one operation
+ and not using multiple calls like in the previous version of this interface
*/
- void setRequestBuilder(GetShardIteratorRequest.Builder builder);
-
- /**
- * Sets the stream name being worked on
- *
- * @param streamName the stream name
- */
- void setStreamName(String streamName);
+ void configureGetShardIteratorRequest(GetShardIteratorRequest.Builder builder, String streamName, String shardId);
}
diff --git a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeStrategy.java b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeStrategy.java
new file mode 100644
index 00000000000..e6f1059fcdd
--- /dev/null
+++ b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeStrategy.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.aws2.kinesis.consumer;
+
+import org.apache.camel.resume.Offset;
+import org.apache.camel.resume.OffsetKey;
+import org.apache.camel.resume.Resumable;
+import org.apache.camel.resume.ResumeAdapter;
+import org.apache.camel.resume.ResumeStrategy;
+import org.apache.camel.resume.ResumeStrategyConfiguration;
+import org.apache.camel.resume.cache.ResumeCache;
+import org.apache.camel.spi.annotations.JdkService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@JdkService("kinesis-resume-strategy")
+public class KinesisResumeStrategy implements ResumeStrategy {
+ private static final Logger LOG = LoggerFactory.getLogger(KinesisResumeStrategy.class);
+
+ private ResumeStrategyConfiguration configuration = KinesisResumeStrategyConfiguration.builder().build();
+ private ResumeCache resumeCache;
+ private ResumeAdapter adapter;
+
+ @Override
+ public void start() {
+ LOG.info("start");
+ this.resumeCache = configuration.getResumeCache();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("stop");
+ }
+
+ @Override
+ public void setAdapter(ResumeAdapter adapter) {
+ this.adapter = adapter;
+ }
+
+ @Override
+ public ResumeAdapter getAdapter() {
+ return adapter;
+ }
+
+ @Override
+ public <T extends Resumable> void updateLastOffset(T offset) {
+ resumeCache.add(offset.getOffsetKey().getValue().toString(),
+ new KinesisOffset(offset.getLastOffset().getValue(String.class)));
+ }
+
+ @Override
+ public <T extends Resumable> void updateLastOffset(T offset, UpdateCallBack updateCallBack) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void updateLastOffset(OffsetKey<?> offsetKey, Offset<?> offsetValue) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void updateLastOffset(OffsetKey<?> offsetKey, Offset<?> offset, UpdateCallBack updateCallBack) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setResumeStrategyConfiguration(ResumeStrategyConfiguration resumeStrategyConfiguration) {
+ this.configuration = resumeStrategyConfiguration;
+ }
+
+ @Override
+ public ResumeStrategyConfiguration getResumeStrategyConfiguration() {
+ return configuration;
+ }
+}
diff --git a/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeStrategyConfiguration.java b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeStrategyConfiguration.java
new file mode 100644
index 00000000000..d9b9aebeb32
--- /dev/null
+++ b/components/camel-aws/camel-aws2-kinesis/src/main/java/org/apache/camel/component/aws2/kinesis/consumer/KinesisResumeStrategyConfiguration.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.aws2.kinesis.consumer;
+
+import org.apache.camel.resume.Cacheable;
+import org.apache.camel.resume.ResumeStrategyConfiguration;
+import org.apache.camel.resume.ResumeStrategyConfigurationBuilder;
+import org.apache.camel.resume.cache.ResumeCache;
+
+public class KinesisResumeStrategyConfiguration extends ResumeStrategyConfiguration {
+
+ private KinesisResumeStrategyConfiguration() {
+ }
+
+ @Override
+ public String resumeStrategyService() {
+ return "kinesis-resume-strategy";
+ }
+
+ public static KinesisResumeStrategyConfigurationBuilder builder() {
+ return new KinesisResumeStrategyConfigurationBuilder();
+ }
+
+ public static class KinesisResumeStrategyConfigurationBuilder
+ implements
+ ResumeStrategyConfigurationBuilder<KinesisResumeStrategyConfigurationBuilder, KinesisResumeStrategyConfiguration> {
+
+ private KinesisResumeStrategyConfigurationBuilder() {
+ }
+
+ private Cacheable.FillPolicy fillPolicy = Cacheable.FillPolicy.MAXIMIZING;
+ private ResumeCache resumeCache;
+
+ @Override
+ public KinesisResumeStrategyConfigurationBuilder withCacheFillPolicy(Cacheable.FillPolicy cacheFillPolicy) {
+ this.fillPolicy = cacheFillPolicy;
+ return this;
+ }
+
+ @Override
+ public KinesisResumeStrategyConfigurationBuilder withResumeCache(ResumeCache<?> resumeCache) {
+ this.resumeCache = resumeCache;
+ return this;
+ }
+
+ @Override
+ public KinesisResumeStrategyConfiguration build() {
+ KinesisResumeStrategyConfiguration result = new KinesisResumeStrategyConfiguration();
+ result.setResumeCache(resumeCache);
+ result.setCacheFillPolicy(fillPolicy);
+ return result;
+ }
+ }
+}
diff --git a/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerIT.java b/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerIT.java
index 46a62640dc6..aa7c9da76bf 100644
--- a/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerIT.java
+++ b/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerIT.java
@@ -54,6 +54,7 @@ public class KinesisConsumerIT extends CamelTestSupport {
private static class KinesisData {
private String partition;
private String body;
+ private String shardId;
@Override
public String toString() {
@@ -97,6 +98,7 @@ public class KinesisConsumerIT extends CamelTestSupport {
if (message != null) {
data.body = message.getBody(String.class);
data.partition = message.getHeader(Kinesis2Constants.PARTITION_KEY, String.class);
+ data.shardId = message.getHeader(Kinesis2Constants.SHARD_ID, String.class);
}
receivedMessages.add(data);
@@ -135,6 +137,7 @@ public class KinesisConsumerIT extends CamelTestSupport {
assertTrue(data.partition.endsWith(data.body), "The data/partition mismatch for record: " + data);
assertNotEquals(partitionKey, data.partition);
partitionKey = data.partition;
+ assertNotNull(data.shardId);
}
}
}
diff --git a/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerResumeAfterRestartIT.java b/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerResumeAfterRestartIT.java
new file mode 100644
index 00000000000..01b71632295
--- /dev/null
+++ b/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerResumeAfterRestartIT.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.aws2.kinesis.integration;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.aws2.kinesis.consumer.KinesisConsumerOffsetProcessor;
+import org.apache.camel.component.aws2.kinesis.consumer.KinesisResumeStrategyConfiguration;
+import org.apache.camel.processor.resume.TransientResumeStrategy;
+import org.apache.camel.resume.cache.ResumeCache;
+import org.apache.camel.test.infra.aws.common.services.AWSService;
+import org.apache.camel.test.infra.aws2.clients.AWSSDKClientUtils;
+import org.apache.camel.test.infra.aws2.services.AWSServiceFactory;
+import org.apache.camel.test.junit5.CamelTestSupport;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.testcontainers.shaded.org.awaitility.Awaitility;
+import software.amazon.awssdk.core.SdkBytes;
+import software.amazon.awssdk.services.kinesis.KinesisClient;
+import software.amazon.awssdk.services.kinesis.model.PutRecordRequest;
+
+import static org.apache.camel.test.infra.aws2.clients.KinesisUtils.createStream;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class KinesisConsumerResumeAfterRestartIT extends CamelTestSupport {
+
+ @RegisterExtension
+ public static AWSService awsService = AWSServiceFactory.createKinesisService();
+ private KinesisClient client;
+
+ String streamName = "my-stream";
+
+ List<String> receivedMessages = new ArrayList<>();
+
+ ResumeCache resumeCache = TransientResumeStrategy.createSimpleCache();
+
+ @Override
+ protected RouteBuilder createRouteBuilder() {
+ client = AWSSDKClientUtils.newKinesisClient();
+
+ context.getRegistry().bind("amazonKinesisClient", AWSSDKClientUtils.newKinesisClient());
+
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+ String kinesisEndpointUri = "aws2-kinesis://%s?amazonKinesisClient=#amazonKinesisClient";
+
+ fromF(kinesisEndpointUri, streamName)
+ // commenting out the strategy will cause the test to fail as event "First" will be consumed twice
+ .resumable().configuration(KinesisResumeStrategyConfiguration.builder().withResumeCache(resumeCache))
+ .process(new KinesisConsumerOffsetProcessor())
+ .process(exchange -> receivedMessages.add(exchange.getMessage().getBody(String.class)));
+ }
+ };
+ }
+
+ @BeforeEach
+ public void prepareEnvironment() {
+ createStream(client, streamName);
+ }
+
+ private void sendEvent(String payload) {
+ client.putRecord(PutRecordRequest.builder().streamName(streamName).partitionKey("my-key")
+ .data(SdkBytes.fromUtf8String(payload)).build());
+ }
+
+ @Test
+ void shouldResumeConsumptionAfterRestart() {
+
+ sendEvent("First");
+ Awaitility.await().until(() -> receivedMessages.contains("First"));
+
+ restartContext();
+
+ sendEvent("Second");
+ Awaitility.await().until(() -> receivedMessages.contains("Second"));
+
+ assertEquals(2, receivedMessages.size());
+ }
+
+ private void restartContext() {
+ context.stop();
+
+ // stop also seems to close the kinesis client, therefor we need to provide a new one
+ context.getRegistry().bind("amazonKinesisClient", AWSSDKClientUtils.newKinesisClient());
+
+ context.start();
+ }
+}
diff --git a/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerResumeIT.java b/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerResumeIT.java
index 26686b3cb33..af890f72faf 100644
--- a/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerResumeIT.java
+++ b/components/camel-aws/camel-aws2-kinesis/src/test/java/org/apache/camel/component/aws2/kinesis/integration/KinesisConsumerResumeIT.java
@@ -86,17 +86,20 @@ public class KinesisConsumerResumeIT extends CamelTestSupport {
}
@Override
- public void setRequestBuilder(GetShardIteratorRequest.Builder builder) {
- this.builder = ObjectHelper.notNull(builder, "builder");
+ public void resume() {
}
- @Override
- public void setStreamName(String streamName) {
- ObjectHelper.notNull(streamName, "streamName");
+ public void setPreviousRecords(List<PutRecordsResponse> previousRecords) {
+ this.previousRecords = previousRecords;
}
@Override
- public void resume() {
+ public void configureGetShardIteratorRequest(
+ GetShardIteratorRequest.Builder builder, String streamName, String shardId) {
+ ObjectHelper.notNull(builder, "builder");
+ ObjectHelper.notNull(streamName, "streamName");
+ ObjectHelper.notNull(shardId, "shardId");
+
LOG.debug("Waiting for data");
Awaitility.await().atMost(1, TimeUnit.MINUTES).until(() -> !previousRecords.isEmpty());
@@ -107,10 +110,6 @@ public class KinesisConsumerResumeIT extends CamelTestSupport {
builder.startingSequenceNumber(putRecordsResultEntry.sequenceNumber());
builder.shardIteratorType(ShardIteratorType.AT_SEQUENCE_NUMBER);
}
-
- public void setPreviousRecords(List<PutRecordsResponse> previousRecords) {
- this.previousRecords = previousRecords;
- }
}
@RegisterExtension