You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/16 11:29:42 UTC
(iotdb) branch master updated: Subscription: improve event consistency by operating within the `compute` method (#13787)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d6a64705d6f Subscription: improve event consistency by operating within the `compute` method (#13787)
d6a64705d6f is described below
commit d6a64705d6f62c5c38abffd8bd89fc3de730c276
Author: V_Galaxy <vg...@apache.org>
AuthorDate: Wed Oct 16 19:29:36 2024 +0800
Subscription: improve event consistency by operating within the `compute` method (#13787)
---
.../broker/SubscriptionPrefetchingQueue.java | 152 +++++++-----
.../broker/SubscriptionPrefetchingTabletQueue.java | 148 ++++++-----
.../broker/SubscriptionPrefetchingTsFileQueue.java | 274 +++++++++++----------
3 files changed, 318 insertions(+), 256 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java
index ac5b3fad279..ed2e6f2efa0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java
@@ -48,6 +48,7 @@ import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -82,6 +83,12 @@ public abstract class SubscriptionPrefetchingQueue {
* SubscriptionPrefetchingQueue#ack}, etc. However, it does not enforce mutual exclusion among the
* other operations themselves.
*
+ * <p>Under the premise of obtaining this lock, to avoid inconsistencies, updates to the {@link
+ * SubscriptionEvent} in both {@link SubscriptionPrefetchingQueue#prefetchingQueue} and {@link
+ * SubscriptionPrefetchingQueue#inFlightEvents} MUST be performed within the {@link
+ * ConcurrentHashMap#compute} method of inFlightEvents in the {@link
+ * SubscriptionPrefetchingQueue}.
+ *
* <p>This lock is created with fairness set to true, which means threads acquire the lock in the
* order they requested it, to avoid thread starvation.
*/
@@ -404,50 +411,60 @@ public abstract class SubscriptionPrefetchingQueue {
*/
private boolean ackInternal(
final String consumerId, final SubscriptionCommitContext commitContext) {
- final SubscriptionEvent event = inFlightEvents.get(new Pair<>(consumerId, commitContext));
- if (Objects.isNull(event)) {
- LOGGER.warn(
- "Subscription: subscription commit context {} does not exist, it may have been committed or something unexpected happened, prefetching queue: {}",
- commitContext,
- this);
- return false;
- }
-
- if (event.isCommitted()) {
- LOGGER.warn(
- "Subscription: subscription event {} is committed, subscription commit context {}, prefetching queue: {}",
- event,
- commitContext,
- this);
- return false;
- }
-
- if (!event.isCommittable()) {
- LOGGER.warn(
- "Subscription: subscription event {} is not committable, subscription commit context {}, prefetching queue: {}",
- event,
- commitContext,
- this);
- return false;
- }
+ final AtomicBoolean acked = new AtomicBoolean(false);
+ inFlightEvents.compute(
+ new Pair<>(consumerId, commitContext),
+ (key, ev) -> {
+ if (Objects.isNull(ev)) {
+ LOGGER.warn(
+ "Subscription: subscription commit context {} does not exist, it may have been committed or something unexpected happened, prefetching queue: {}",
+ commitContext,
+ this);
+ return null;
+ }
+
+ if (ev.isCommitted()) {
+ LOGGER.warn(
+ "Subscription: subscription event {} is committed, subscription commit context {}, prefetching queue: {}",
+ ev,
+ commitContext,
+ this);
+ return ev;
+ }
+
+ if (ev.isCommitted()) {
+ LOGGER.warn(
+ "Subscription: subscription event {} is committed, subscription commit context {}, prefetching queue: {}",
+ ev,
+ commitContext,
+ this);
+ // clean up committed event
+ ev.cleanUp();
+ return null; // remove this entry
+ }
+
+ // check if a consumer acks event from another consumer group...
+ final String consumerGroupId = commitContext.getConsumerGroupId();
+ if (!Objects.equals(consumerGroupId, brokerId)) {
+ LOGGER.warn(
+ "inconsistent consumer group when acking event, current: {}, incoming: {}, consumer id: {}, event commit context: {}, prefetching queue: {}, commit it anyway...",
+ brokerId,
+ consumerGroupId,
+ consumerId,
+ commitContext,
+ this);
+ }
- // check if a consumer acks event from another consumer group...
- final String consumerGroupId = commitContext.getConsumerGroupId();
- if (!Objects.equals(consumerGroupId, brokerId)) {
- LOGGER.warn(
- "inconsistent consumer group when acking event, current: {}, incoming: {}, consumer id: {}, event commit context: {}, prefetching queue: {}, commit it anyway...",
- brokerId,
- consumerGroupId,
- consumerId,
- commitContext,
- this);
- }
+ ev.ack();
+ ev.recordCommittedTimestamp(); // now committed
+ acked.set(true);
- event.ack();
- event.recordCommittedTimestamp(); // now committed
+ // clean up committed event
+ ev.cleanUp();
+ return null; // remove this entry
+ });
- // no need to update inFlightEvents
- return true;
+ return acked.get();
}
/**
@@ -467,31 +484,38 @@ public abstract class SubscriptionPrefetchingQueue {
*/
public boolean nackInternal(
final String consumerId, final SubscriptionCommitContext commitContext) {
- final SubscriptionEvent event = inFlightEvents.get(new Pair<>(consumerId, commitContext));
- if (Objects.isNull(event)) {
- LOGGER.warn(
- "Subscription: subscription commit context [{}] does not exist, it may have been committed or something unexpected happened, prefetching queue: {}",
- commitContext,
- this);
- return false;
- }
+ final AtomicBoolean nacked = new AtomicBoolean(false);
+ inFlightEvents.compute(
+ new Pair<>(consumerId, commitContext),
+ (key, ev) -> {
+ if (Objects.isNull(ev)) {
+ LOGGER.warn(
+ "Subscription: subscription commit context [{}] does not exist, it may have been committed or something unexpected happened, prefetching queue: {}",
+ commitContext,
+ this);
+ return null;
+ }
+
+ // check if a consumer nacks event from another consumer group...
+ final String consumerGroupId = commitContext.getConsumerGroupId();
+ if (!Objects.equals(consumerGroupId, brokerId)) {
+ LOGGER.warn(
+ "inconsistent consumer group when nacking event, current: {}, incoming: {}, consumer id: {}, event commit context: {}, prefetching queue: {}, commit it anyway...",
+ brokerId,
+ consumerGroupId,
+ consumerId,
+ commitContext,
+ this);
+ }
- // check if a consumer nacks event from another consumer group...
- final String consumerGroupId = commitContext.getConsumerGroupId();
- if (!Objects.equals(consumerGroupId, brokerId)) {
- LOGGER.warn(
- "inconsistent consumer group when nacking event, current: {}, incoming: {}, consumer id: {}, event commit context: {}, prefetching queue: {}, commit it anyway...",
- brokerId,
- consumerGroupId,
- consumerId,
- commitContext,
- this);
- }
+ ev.nack(); // now pollable
+ nacked.set(true);
- event.nack(); // now pollable
+ // no need to update inFlightEvents and prefetchingQueue
+ return ev;
+ });
- // no need to update inFlightEvents and prefetchingQueue
- return true;
+ return nacked.get();
}
public SubscriptionCommitContext generateSubscriptionCommitContext() {
@@ -545,7 +569,7 @@ public abstract class SubscriptionPrefetchingQueue {
.map(SubscriptionEvent::getPipeEventCount)
.reduce(Integer::sum)
.orElse(0)
- + +inFlightEvents.values().stream()
+ + inFlightEvents.values().stream()
.map(SubscriptionEvent::getPipeEventCount)
.reduce(Integer::sum)
.orElse(0);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java
index 28ff3771349..ea7a652e461 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletQueue.java
@@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
public class SubscriptionPrefetchingTabletQueue extends SubscriptionPrefetchingQueue {
@@ -69,77 +70,92 @@ public class SubscriptionPrefetchingTabletQueue extends SubscriptionPrefetchingQ
private @NonNull SubscriptionEvent pollTabletsInternal(
final String consumerId, final SubscriptionCommitContext commitContext, final int offset) {
- // 1. Extract current event and check it
- final SubscriptionEvent event =
- inFlightEvents.compute(
- new Pair<>(consumerId, commitContext),
- (key, ev) -> {
- if (Objects.nonNull(ev) && ev.isCommitted()) {
- ev.cleanUp();
- return null; // remove this entry
+ final AtomicReference<SubscriptionEvent> eventRef = new AtomicReference<>();
+ inFlightEvents.compute(
+ new Pair<>(consumerId, commitContext),
+ (key, ev) -> {
+ // 1. Extract current event and check it
+ if (Objects.isNull(ev)) {
+ final String errorMessage =
+ String.format(
+ "SubscriptionPrefetchingTabletQueue %s is currently not transferring any tablet to consumer %s, commit context: %s, offset: %s",
+ this, consumerId, commitContext, offset);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return null;
+ }
+
+ if (ev.isCommitted()) {
+ ev.cleanUp();
+ final String errorMessage =
+ String.format(
+ "outdated poll request after commit, consumer id: %s, commit context: %s, offset: %s, prefetching queue: %s",
+ consumerId, commitContext, offset, this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return null; // remove this entry
+ }
+
+ // check consumer id
+ if (!Objects.equals(ev.getLastPolledConsumerId(), consumerId)) {
+ final String errorMessage =
+ String.format(
+ "inconsistent polled consumer id, current: %s, incoming: %s, commit context: %s, offset: %s, prefetching queue: %s",
+ ev.getLastPolledConsumerId(), consumerId, commitContext, offset, this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+
+ final SubscriptionPollResponse response = ev.getCurrentResponse();
+ final SubscriptionPollPayload payload = response.getPayload();
+
+ // 2. Check previous response type and offset
+ final short responseType = response.getResponseType();
+ if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) {
+ final String errorMessage = String.format("unexpected response type: %s", responseType);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+
+ switch (SubscriptionPollResponseType.valueOf(responseType)) {
+ case TABLETS:
+ // check offset
+ if (!Objects.equals(offset, ((TabletsPayload) payload).getNextOffset())) {
+ final String errorMessage =
+ String.format(
+ "inconsistent offset, current: %s, incoming: %s, consumer: %s, prefetching queue: %s",
+ ((TabletsPayload) payload).getNextOffset(), offset, consumerId, this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
}
- return ev;
- });
-
- if (Objects.isNull(event)) {
- final String errorMessage =
- String.format(
- "SubscriptionPrefetchingTabletQueue %s is currently not transferring any tablet to consumer %s, commit context: %s, offset: %s",
- this, consumerId, commitContext, offset);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
-
- // check consumer id
- if (!Objects.equals(event.getLastPolledConsumerId(), consumerId)) {
- final String errorMessage =
- String.format(
- "inconsistent polled consumer id, current: %s, incoming: %s, commit context: %s, offset: %s, prefetching queue: %s",
- event.getLastPolledConsumerId(), consumerId, commitContext, offset, this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
+ break;
+ default:
+ {
+ final String errorMessage =
+ String.format("unexpected response type: %s", responseType);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+ }
- final SubscriptionPollResponse response = event.getCurrentResponse();
- final SubscriptionPollPayload payload = response.getPayload();
+ // 3. Poll next tablets
+ try {
+ ev.fetchNextResponse();
+ } catch (final Exception ignored) {
+ // no exceptions will be thrown
+ }
- // 2. Check previous response type and offset
- final short responseType = response.getResponseType();
- if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) {
- final String errorMessage = String.format("unexpected response type: %s", responseType);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
+ ev.recordLastPolledTimestamp();
+ eventRef.set(ev);
- switch (SubscriptionPollResponseType.valueOf(responseType)) {
- case TABLETS:
- // check offset
- if (!Objects.equals(offset, ((TabletsPayload) payload).getNextOffset())) {
- final String errorMessage =
- String.format(
- "inconsistent offset, current: %s, incoming: %s, consumer: %s, prefetching queue: %s",
- ((TabletsPayload) payload).getNextOffset(), offset, consumerId, this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- break;
- default:
- {
- final String errorMessage = String.format("unexpected response type: %s", responseType);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- }
-
- // 3. Poll next tablets
- try {
- event.fetchNextResponse();
- } catch (final Exception ignored) {
- // no exceptions will be thrown
- }
+ return ev;
+ });
- event.recordLastPolledTimestamp();
- return event;
+ return eventRef.get();
}
/////////////////////////////// prefetch ///////////////////////////////
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java
index 40969d72bd9..64daaca4b4c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java
@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
public class SubscriptionPrefetchingTsFileQueue extends SubscriptionPrefetchingQueue {
@@ -76,138 +77,159 @@ public class SubscriptionPrefetchingTsFileQueue extends SubscriptionPrefetchingQ
final String consumerId,
final SubscriptionCommitContext commitContext,
final long writingOffset) {
- // 1. Extract current event and check it
- final SubscriptionEvent event =
- inFlightEvents.compute(
- new Pair<>(consumerId, commitContext),
- (key, ev) -> {
- if (Objects.nonNull(ev) && ev.isCommitted()) {
- ev.cleanUp();
- return null; // remove this entry
- }
- return ev;
- });
-
- if (Objects.isNull(event)) {
- final String errorMessage =
- String.format(
- "SubscriptionPrefetchingTsFileQueue %s is currently not transferring any file to consumer %s, commit context: %s, writing offset: %s",
- this, consumerId, commitContext, writingOffset);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
+ final AtomicReference<SubscriptionEvent> eventRef = new AtomicReference<>();
+ inFlightEvents.compute(
+ new Pair<>(consumerId, commitContext),
+ (key, ev) -> {
+ // 1. Extract current event and check it
+ if (Objects.isNull(ev)) {
+ final String errorMessage =
+ String.format(
+ "SubscriptionPrefetchingTsFileQueue %s is currently not transferring any file to consumer %s, commit context: %s, writing offset: %s",
+ this, consumerId, commitContext, writingOffset);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return null;
+ }
- // check consumer id
- if (!Objects.equals(event.getLastPolledConsumerId(), consumerId)) {
- final String errorMessage =
- String.format(
- "inconsistent polled consumer id, current: %s, incoming: %s, commit context: %s, writing offset: %s, prefetching queue: %s",
- event.getLastPolledConsumerId(), consumerId, commitContext, writingOffset, this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
+ if (ev.isCommitted()) {
+ ev.cleanUp();
+ final String errorMessage =
+ String.format(
+ "outdated poll request after commit, consumer id: %s, commit context: %s, writing offset: %s, prefetching queue: %s",
+ consumerId, commitContext, writingOffset, this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return null; // remove this entry
+ }
- final SubscriptionPollResponse response = event.getCurrentResponse();
- final SubscriptionPollPayload payload = response.getPayload();
+ // check consumer id
+ if (!Objects.equals(ev.getLastPolledConsumerId(), consumerId)) {
+ final String errorMessage =
+ String.format(
+ "inconsistent polled consumer id, current: %s, incoming: %s, commit context: %s, writing offset: %s, prefetching queue: %s",
+ ev.getLastPolledConsumerId(), consumerId, commitContext, writingOffset, this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
- // 2. Check previous response type, file name and offset
- final short responseType = response.getResponseType();
- if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) {
- final String errorMessage = String.format("unexpected response type: %s", responseType);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
+ final SubscriptionPollResponse response = ev.getCurrentResponse();
+ final SubscriptionPollPayload payload = response.getPayload();
- final String fileName = event.getFileName();
- switch (SubscriptionPollResponseType.valueOf(responseType)) {
- case FILE_INIT:
- // check file name
- if (!Objects.equals(fileName, ((FileInitPayload) payload).getFileName())) {
- final String errorMessage =
- String.format(
- "inconsistent file name, current: %s, incoming: %s, consumer: %s, writing offset: %s, prefetching queue: %s",
- ((FileInitPayload) payload).getFileName(),
- fileName,
- consumerId,
- writingOffset,
- this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- // check offset
- if (writingOffset != 0) {
- final String errorMessage =
- String.format(
- "inconsistent offset, current: %s, incoming: %s, consumer: %s, file name: %s, prefetching queue: %s",
- 0, writingOffset, consumerId, fileName, this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- break;
- case FILE_PIECE:
- // check file name
- if (!Objects.equals(fileName, ((FilePiecePayload) payload).getFileName())) {
- final String errorMessage =
- String.format(
- "inconsistent file name, current: %s, incoming: %s, consumer: %s, writing offset: %s, prefetching queue: %s",
- ((FilePiecePayload) payload).getFileName(),
- fileName,
- consumerId,
- writingOffset,
- this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- // check offset
- if (writingOffset != ((FilePiecePayload) payload).getNextWritingOffset()) {
- final String errorMessage =
- String.format(
- "inconsistent offset, current: %s, incoming: %s, consumer: %s, file name: %s, prefetching queue: %s",
- ((FilePiecePayload) payload).getNextWritingOffset(),
- writingOffset,
- consumerId,
- fileName,
- this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- break;
- case FILE_SEAL:
- {
- final String errorMessage =
- String.format(
- "poll after sealing, consumer: %s, file name: %s, writing offset: %s, prefetching queue: %s",
- consumerId, fileName, writingOffset, this);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- default:
- {
- final String errorMessage = String.format("unexpected response type: %s", responseType);
- LOGGER.warn(errorMessage);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
- }
+ // 2. Check previous response type, file name and offset
+ final short responseType = response.getResponseType();
+ if (!SubscriptionPollResponseType.isValidatedResponseType(responseType)) {
+ final String errorMessage = String.format("unexpected response type: %s", responseType);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
- // 3. Poll tsfile piece or tsfile seal
- try {
- event.fetchNextResponse();
- } catch (final Exception e) {
- LOGGER.warn(
- "Exception occurred when SubscriptionPrefetchingTsFileQueue {} transferring file (with event {}) to consumer {}",
- this,
- event,
- consumerId,
- e);
- final String errorMessage =
- String.format(
- "Exception occurred when SubscriptionPrefetchingTsFileQueue %s transferring file (with event %s) to consumer %s: %s",
- this, event, consumerId, e);
- return generateSubscriptionPollErrorResponse(errorMessage);
- }
+ final String fileName = ev.getFileName();
+ switch (SubscriptionPollResponseType.valueOf(responseType)) {
+ case FILE_INIT:
+ // check file name
+ if (!Objects.equals(fileName, ((FileInitPayload) payload).getFileName())) {
+ final String errorMessage =
+ String.format(
+ "inconsistent file name, current: %s, incoming: %s, consumer: %s, writing offset: %s, prefetching queue: %s",
+ ((FileInitPayload) payload).getFileName(),
+ fileName,
+ consumerId,
+ writingOffset,
+ this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+ // check offset
+ if (writingOffset != 0) {
+ final String errorMessage =
+ String.format(
+ "inconsistent offset, current: %s, incoming: %s, consumer: %s, file name: %s, prefetching queue: %s",
+ 0, writingOffset, consumerId, fileName, this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+ break;
+ case FILE_PIECE:
+ // check file name
+ if (!Objects.equals(fileName, ((FilePiecePayload) payload).getFileName())) {
+ final String errorMessage =
+ String.format(
+ "inconsistent file name, current: %s, incoming: %s, consumer: %s, writing offset: %s, prefetching queue: %s",
+ ((FilePiecePayload) payload).getFileName(),
+ fileName,
+ consumerId,
+ writingOffset,
+ this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+ // check offset
+ if (writingOffset != ((FilePiecePayload) payload).getNextWritingOffset()) {
+ final String errorMessage =
+ String.format(
+ "inconsistent offset, current: %s, incoming: %s, consumer: %s, file name: %s, prefetching queue: %s",
+ ((FilePiecePayload) payload).getNextWritingOffset(),
+ writingOffset,
+ consumerId,
+ fileName,
+ this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+ break;
+ case FILE_SEAL:
+ {
+ final String errorMessage =
+ String.format(
+ "poll after sealing, consumer: %s, file name: %s, writing offset: %s, prefetching queue: %s",
+ consumerId, fileName, writingOffset, this);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+ default:
+ {
+ final String errorMessage =
+ String.format("unexpected response type: %s", responseType);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+ }
+
+ // 3. Poll tsfile piece or tsfile seal
+ try {
+ ev.fetchNextResponse();
+ } catch (final Exception e) {
+ LOGGER.warn(
+ "Exception occurred when SubscriptionPrefetchingTsFileQueue {} transferring file (with event {}) to consumer {}",
+ this,
+ ev,
+ consumerId,
+ e);
+ final String errorMessage =
+ String.format(
+ "Exception occurred when SubscriptionPrefetchingTsFileQueue %s transferring file (with event %s) to consumer %s: %s",
+ this, ev, consumerId, e);
+ LOGGER.warn(errorMessage);
+ eventRef.set(generateSubscriptionPollErrorResponse(errorMessage));
+ return ev;
+ }
+
+ ev.recordLastPolledTimestamp();
+ eventRef.set(ev);
+
+ return ev;
+ });
- event.recordLastPolledTimestamp();
- return event;
+ return eventRef.get();
}
/////////////////////////////// prefetch ///////////////////////////////