You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jo...@apache.org on 2024/02/28 14:46:01 UTC
(beam) branch master updated: Wire error handling into PubSubIO and add initial tests (#30372)
This is an automated email from the ASF dual-hosted git repository.
johncasey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2b9d958807d Wire error handling into PubSubIO and add initial tests (#30372)
2b9d958807d is described below
commit 2b9d958807def109f92dc23c9761e5636693cab3
Author: johnjcasey <95...@users.noreply.github.com>
AuthorDate: Wed Feb 28 09:45:54 2024 -0500
Wire error handling into PubSubIO and add initial tests (#30372)
* Wire error handling into PubSubIO and add initial tests
* update comments to reflect that schema mismatches are not routed to DLQs
* spotless
* address nits
---
CHANGES.md | 2 +
.../sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java | 51 ++++-
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 205 ++++++++++++++++-----
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 104 +++++++++++
4 files changed, 312 insertions(+), 50 deletions(-)
diff --git a/CHANGES.md b/CHANGES.md
index 28ca6543006..fbbc12ef0a9 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -66,6 +66,8 @@
* Full Support for Storage Read and Write APIs
* Partial Support for File Loads (Failures writing to files supported, failures loading files to BQ unsupported)
* No Support for Extract or Streaming Inserts
+* Added support for handling bad records to PubSubIO ([#30372](https://github.com/apache/beam/pull/30372)).
+ * Support is not available for handling schema mismatches, and enabling error handling for writing to pubsub topics with schemas is not recommended
* `--enableBundling` pipeline option for BigQueryIO DIRECT_READ is replaced by `--enableStorageReadApiV2`. Both were considered experimental and may subject to change (Java) ([#26354](https://github.com/apache/beam/issues/26354)).
* Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java
index 26f4fb5d076..47033451ab8 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PreparePubsubWriteDoFn.java
@@ -20,10 +20,13 @@ package org.apache.beam.sdk.io.gcp.pubsub;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import javax.naming.SizeLimitExceededException;
+import org.apache.beam.sdk.coders.Coder;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.values.TupleTag;
import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.joda.time.Instant;
@@ -41,6 +44,12 @@ public class PreparePubsubWriteDoFn<InputT> extends DoFn<InputT, PubsubMessage>
private SerializableFunction<ValueInSingleWindow<InputT>, PubsubMessage> formatFunction;
@Nullable SerializableFunction<ValueInSingleWindow<InputT>, PubsubIO.PubsubTopic> topicFunction;
+ private final BadRecordRouter badRecordRouter;
+
+ private final Coder<InputT> inputCoder;
+
+ private final TupleTag<PubsubMessage> outputTag;
+
static int validatePubsubMessageSize(PubsubMessage message, int maxPublishBatchSize)
throws SizeLimitExceededException {
int payloadSize = message.getPayload().length;
@@ -113,10 +122,16 @@ public class PreparePubsubWriteDoFn<InputT> extends DoFn<InputT, PubsubMessage>
SerializableFunction<ValueInSingleWindow<InputT>, PubsubMessage> formatFunction,
@Nullable
SerializableFunction<ValueInSingleWindow<InputT>, PubsubIO.PubsubTopic> topicFunction,
- int maxPublishBatchSize) {
+ int maxPublishBatchSize,
+ BadRecordRouter badRecordRouter,
+ Coder<InputT> inputCoder,
+ TupleTag<PubsubMessage> outputTag) {
this.formatFunction = formatFunction;
this.topicFunction = topicFunction;
this.maxPublishBatchSize = maxPublishBatchSize;
+ this.badRecordRouter = badRecordRouter;
+ this.inputCoder = inputCoder;
+ this.outputTag = outputTag;
}
@ProcessElement
@@ -125,18 +140,42 @@ public class PreparePubsubWriteDoFn<InputT> extends DoFn<InputT, PubsubMessage>
@Timestamp Instant ts,
BoundedWindow window,
PaneInfo paneInfo,
- OutputReceiver<PubsubMessage> o) {
+ MultiOutputReceiver o)
+ throws Exception {
ValueInSingleWindow<InputT> valueInSingleWindow =
ValueInSingleWindow.of(element, ts, window, paneInfo);
- PubsubMessage message = formatFunction.apply(valueInSingleWindow);
+ PubsubMessage message;
+ try {
+ message = formatFunction.apply(valueInSingleWindow);
+ } catch (Exception e) {
+ badRecordRouter.route(
+ o,
+ element,
+ inputCoder,
+ e,
+ "Failed to serialize PubSub message with provided format function");
+ return;
+ }
if (topicFunction != null) {
- message = message.withTopic(topicFunction.apply(valueInSingleWindow).asPath());
+ try {
+ message = message.withTopic(topicFunction.apply(valueInSingleWindow).asPath());
+ } catch (Exception e) {
+ badRecordRouter.route(
+ o, element, inputCoder, e, "Failed to determine PubSub topic using topic function");
+ return;
+ }
}
try {
validatePubsubMessageSize(message, maxPublishBatchSize);
} catch (SizeLimitExceededException e) {
- throw new IllegalArgumentException(e);
+ badRecordRouter.route(
+ o,
+ element,
+ inputCoder,
+ new IllegalArgumentException(e),
+ "PubSub message limit exceeded, see exception for details");
+ return;
}
- o.output(message);
+ o.get(outputTag).output(message);
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index 1080999271d..1d687812560 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.io.gcp.pubsub;
+import static org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter.BAD_RECORD_TAG;
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState;
import com.google.api.client.util.Clock;
@@ -62,6 +63,11 @@ import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.transforms.WithFailures;
import org.apache.beam.sdk.transforms.WithFailures.Result;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter.ThrowingBadRecordRouter;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.DefaultErrorHandler;
import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.util.Preconditions;
@@ -69,8 +75,11 @@ import org.apache.beam.sdk.values.EncodableThrowable;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
import org.apache.beam.sdk.values.PDone;
import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
@@ -830,6 +839,10 @@ public class PubsubIO {
abstract boolean getNeedsOrderingKey();
+ abstract BadRecordRouter getBadRecordRouter();
+
+ abstract ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
abstract Builder<T> toBuilder();
static <T> Builder<T> newBuilder(SerializableFunction<PubsubMessage, T> parseFn) {
@@ -839,6 +852,8 @@ public class PubsubIO {
builder.setNeedsAttributes(false);
builder.setNeedsMessageId(false);
builder.setNeedsOrderingKey(false);
+ builder.setBadRecordRouter(BadRecordRouter.THROWING_ROUTER);
+ builder.setBadRecordErrorHandler(new DefaultErrorHandler<>());
return builder;
}
@@ -881,6 +896,11 @@ public class PubsubIO {
abstract Builder<T> setClock(Clock clock);
+ abstract Builder<T> setBadRecordRouter(BadRecordRouter badRecordRouter);
+
+ abstract Builder<T> setBadRecordErrorHandler(
+ ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
abstract Read<T> build();
}
@@ -959,6 +979,8 @@ public class PubsubIO {
*
* <p>See {@link PubsubIO.PubsubTopic#fromPath(String)} for more details on the format of the
* {@code deadLetterTopic} string.
+ *
+ * <p>This functionality is mutually exclusive with {@link Read#withErrorHandler(ErrorHandler)}
*/
public Read<T> withDeadLetterTopic(String deadLetterTopic) {
return withDeadLetterTopic(StaticValueProvider.of(deadLetterTopic));
@@ -1045,6 +1067,19 @@ public class PubsubIO {
return toBuilder().setCoder(coder).setParseFn(parseFn).build();
}
+ /**
+ * Configures the PubSub read with an alternate error handler. When a message is read from
+ * PubSub, but fails to parse, the message and the parse failure information will be sent to the
+ * error handler. See {@link ErrorHandler} for more details on configuring an Error Handler.
+ * This functionality is mutually exclusive with {@link Read#withDeadLetterTopic(String)}.
+ */
+ public Read<T> withErrorHandler(ErrorHandler<BadRecord, ?> badRecordErrorHandler) {
+ return toBuilder()
+ .setBadRecordErrorHandler(badRecordErrorHandler)
+ .setBadRecordRouter(BadRecordRouter.RECORDING_ROUTER)
+ .build();
+ }
+
@VisibleForTesting
/**
* Set's the internal Clock.
@@ -1066,6 +1101,12 @@ public class PubsubIO {
"Can't set both the topic and the subscription for " + "a PubsubIO.Read transform");
}
+ if (getDeadLetterTopicProvider() != null
+ && !(getBadRecordRouter() instanceof ThrowingBadRecordRouter)) {
+ throw new IllegalArgumentException(
+ "PubSubIO cannot be configured with both a dead letter topic and a bad record router");
+ }
+
@Nullable
ValueProvider<TopicPath> topicPath =
getTopicProvider() == null
@@ -1089,57 +1130,73 @@ public class PubsubIO {
getNeedsMessageId(),
getNeedsOrderingKey());
- PCollection<T> read;
PCollection<PubsubMessage> preParse = input.apply(source);
TypeDescriptor<T> typeDescriptor = new TypeDescriptor<T>() {};
- if (getDeadLetterTopicProvider() == null) {
+ PCollection<T> read;
+ if (getDeadLetterTopicProvider() == null
+ && (getBadRecordRouter() instanceof ThrowingBadRecordRouter)) {
read = preParse.apply(MapElements.into(typeDescriptor).via(getParseFn()));
} else {
+ // parse PubSub messages, separating out exceptions
Result<PCollection<T>, KV<PubsubMessage, EncodableThrowable>> result =
preParse.apply(
"PubsubIO.Read/Map/Parse-Incoming-Messages",
MapElements.into(typeDescriptor)
.via(getParseFn())
.exceptionsVia(new WithFailures.ThrowableHandler<PubsubMessage>() {}));
+
+ // Emit parsed records
read = result.output();
- // Write out failures to the provided dead-letter topic.
- result
- .failures()
- // Since the stack trace could easily exceed Pub/Sub limits, we need to remove it from
- // the attributes.
- .apply(
- "PubsubIO.Read/Map/Remove-Stack-Trace-Attribute",
- MapElements.into(new TypeDescriptor<KV<PubsubMessage, Map<String, String>>>() {})
- .via(
- kv -> {
- PubsubMessage message = kv.getKey();
- String messageId =
- message.getMessageId() == null ? "<null>" : message.getMessageId();
- Throwable throwable = kv.getValue().throwable();
-
- // In order to stay within Pub/Sub limits, we aren't adding the stack
- // trace to the attributes. Therefore, we need to log the throwable.
- LOG.error(
- "Error parsing Pub/Sub message with id '{}'", messageId, throwable);
-
- ImmutableMap<String, String> attributes =
- ImmutableMap.<String, String>builder()
- .put("exceptionClassName", throwable.getClass().getName())
- .put("exceptionMessage", throwable.getMessage())
- .put("pubsubMessageId", messageId)
- .build();
-
- return KV.of(kv.getKey(), attributes);
- }))
- .apply(
- "PubsubIO.Read/Map/Create-Dead-Letter-Payload",
- MapElements.into(TypeDescriptor.of(PubsubMessage.class))
- .via(kv -> new PubsubMessage(kv.getKey().getPayload(), kv.getValue())))
- .apply(
- writeMessages()
- .to(getDeadLetterTopicProvider().get().asPath())
- .withClientFactory(getPubsubClientFactory()));
+ // Send exceptions to either the bad record router or the dead letter topic
+ if (!(getBadRecordRouter() instanceof ThrowingBadRecordRouter)) {
+ PCollection<BadRecord> badRecords =
+ result
+ .failures()
+ .apply(
+ "Map Failures To BadRecords",
+ ParDo.of(new ParseReadFailuresToBadRecords(preParse.getCoder())));
+ getBadRecordErrorHandler()
+ .addErrorCollection(badRecords.setCoder(BadRecord.getCoder(input.getPipeline())));
+ } else {
+ // Write out failures to the provided dead-letter topic.
+ result
+ .failures()
+ // Since the stack trace could easily exceed Pub/Sub limits, we need to remove it from
+ // the attributes.
+ .apply(
+ "PubsubIO.Read/Map/Remove-Stack-Trace-Attribute",
+ MapElements.into(new TypeDescriptor<KV<PubsubMessage, Map<String, String>>>() {})
+ .via(
+ kv -> {
+ PubsubMessage message = kv.getKey();
+ String messageId =
+ message.getMessageId() == null ? "<null>" : message.getMessageId();
+ Throwable throwable = kv.getValue().throwable();
+
+ // In order to stay within Pub/Sub limits, we aren't adding the stack
+ // trace to the attributes. Therefore, we need to log the throwable.
+ LOG.error(
+ "Error parsing Pub/Sub message with id '{}'", messageId, throwable);
+
+ ImmutableMap<String, String> attributes =
+ ImmutableMap.<String, String>builder()
+ .put("exceptionClassName", throwable.getClass().getName())
+ .put("exceptionMessage", throwable.getMessage())
+ .put("pubsubMessageId", messageId)
+ .build();
+
+ return KV.of(kv.getKey(), attributes);
+ }))
+ .apply(
+ "PubsubIO.Read/Map/Create-Dead-Letter-Payload",
+ MapElements.into(TypeDescriptor.of(PubsubMessage.class))
+ .via(kv -> new PubsubMessage(kv.getKey().getPayload(), kv.getValue())))
+ .apply(
+ writeMessages()
+ .to(getDeadLetterTopicProvider().get().asPath())
+ .withClientFactory(getPubsubClientFactory()));
+ }
}
return read.setCoder(getCoder());
@@ -1156,6 +1213,28 @@ public class PubsubIO {
}
}
+ private static class ParseReadFailuresToBadRecords
+ extends DoFn<KV<PubsubMessage, EncodableThrowable>, BadRecord> {
+ private final Coder<PubsubMessage> coder;
+
+ public ParseReadFailuresToBadRecords(Coder<PubsubMessage> coder) {
+ this.coder = coder;
+ }
+
+ @ProcessElement
+ public void processElement(
+ OutputReceiver<BadRecord> outputReceiver,
+ @Element KV<PubsubMessage, EncodableThrowable> element)
+ throws Exception {
+ outputReceiver.output(
+ BadRecord.fromExceptionInformation(
+ element.getKey(),
+ coder,
+ (Exception) element.getValue().throwable(),
+ "Failed to parse message read from PubSub"));
+ }
+ }
+
/////////////////////////////////////////////////////////////////////////////
/** Disallow construction of utility class. */
@@ -1198,6 +1277,10 @@ public class PubsubIO {
abstract @Nullable String getPubsubRootUrl();
+ abstract BadRecordRouter getBadRecordRouter();
+
+ abstract ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
abstract Builder<T> toBuilder();
static <T> Builder<T> newBuilder(
@@ -1205,6 +1288,8 @@ public class PubsubIO {
Builder<T> builder = new AutoValue_PubsubIO_Write.Builder<T>();
builder.setPubsubClientFactory(FACTORY);
builder.setFormatFn(formatFn);
+ builder.setBadRecordRouter(BadRecordRouter.THROWING_ROUTER);
+ builder.setBadRecordErrorHandler(new DefaultErrorHandler<>());
return builder;
}
@@ -1236,6 +1321,11 @@ public class PubsubIO {
abstract Builder<T> setPubsubRootUrl(String pubsubRootUrl);
+ abstract Builder<T> setBadRecordRouter(BadRecordRouter badRecordRouter);
+
+ abstract Builder<T> setBadRecordErrorHandler(
+ ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
abstract Write<T> build();
}
@@ -1334,6 +1424,19 @@ public class PubsubIO {
return toBuilder().setPubsubRootUrl(pubsubRootUrl).build();
}
+ /**
+ * Writes any serialization failures out to the Error Handler. See {@link ErrorHandler} for
+ * details on how to configure an Error Handler. Error Handlers are not well supported when
+ * writing to topics with schemas, and it is not recommended to configure an error handler if
+ * the target topic has a schema.
+ */
+ public Write<T> withErrorHandler(ErrorHandler<BadRecord, ?> badRecordErrorHandler) {
+ return toBuilder()
+ .setBadRecordErrorHandler(badRecordErrorHandler)
+ .setBadRecordRouter(BadRecordRouter.RECORDING_ROUTER)
+ .build();
+ }
+
@Override
public PDone expand(PCollection<T> input) {
if (getTopicProvider() == null && !getDynamicDestinations()) {
@@ -1355,12 +1458,26 @@ public class PubsubIO {
MoreObjects.firstNonNull(
getMaxBatchBytesSize(), MAX_PUBLISH_BATCH_BYTE_SIZE_DEFAULT));
}
+ TupleTag<PubsubMessage> pubsubMessageTupleTag = new TupleTag<>();
+ PCollectionTuple pubsubMessageTuple =
+ input.apply(
+ ParDo.of(
+ new PreparePubsubWriteDoFn<>(
+ getFormatFn(),
+ topicFunction,
+ maxMessageSize,
+ getBadRecordRouter(),
+ input.getCoder(),
+ pubsubMessageTupleTag))
+ .withOutputTags(pubsubMessageTupleTag, TupleTagList.of(BAD_RECORD_TAG)));
+
+ getBadRecordErrorHandler()
+ .addErrorCollection(
+ pubsubMessageTuple
+ .get(BAD_RECORD_TAG)
+ .setCoder(BadRecord.getCoder(input.getPipeline())));
PCollection<PubsubMessage> pubsubMessages =
- input
- .apply(
- ParDo.of(
- new PreparePubsubWriteDoFn<>(getFormatFn(), topicFunction, maxMessageSize)))
- .setCoder(new PubsubMessageWithTopicCoder());
+ pubsubMessageTuple.get(pubsubMessageTupleTag).setCoder(new PubsubMessageWithTopicCoder());
switch (input.isBounded()) {
case BOUNDED:
pubsubMessages.apply(
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java
index f7f9f5f91b7..abc35d0bb1b 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIOTest.java
@@ -71,11 +71,15 @@ import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.transforms.SimpleFunction;
import org.apache.beam.sdk.transforms.display.DisplayData;
import org.apache.beam.sdk.transforms.display.DisplayDataEvaluator;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandlingTestUtils.ErrorSinkTransform;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.TimestampedValue;
import org.apache.beam.sdk.values.TypeDescriptor;
import org.apache.beam.sdk.values.TypeDescriptors;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists;
@@ -482,6 +486,46 @@ public class PubsubIOTest {
pipeline.run();
}
+ @Test
+ public void testFailedParseWithErrorHandlerConfigured() throws Exception {
+ ByteString data = ByteString.copyFrom("Hello, World!".getBytes(StandardCharsets.UTF_8));
+ RuntimeException exception = new RuntimeException("Some error message");
+ ImmutableList<IncomingMessage> expectedReads =
+ ImmutableList.of(
+ IncomingMessage.of(
+ com.google.pubsub.v1.PubsubMessage.newBuilder().setData(data).build(),
+ 1234L,
+ 0,
+ UUID.randomUUID().toString(),
+ UUID.randomUUID().toString()));
+ ImmutableList<OutgoingMessage> expectedWrites = ImmutableList.of();
+ clientFactory =
+ PubsubTestClient.createFactoryForPullAndPublish(
+ SUBSCRIPTION, TOPIC, CLOCK, 60, expectedReads, expectedWrites, ImmutableList.of());
+
+ ErrorHandler<BadRecord, PCollection<Long>> errorHandler =
+ pipeline.registerBadRecordErrorHandler(new ErrorSinkTransform());
+ PCollection<String> read =
+ pipeline.apply(
+ PubsubIO.readStrings()
+ .fromSubscription(SUBSCRIPTION.getPath())
+ .withErrorHandler(errorHandler)
+ .withClock(CLOCK)
+ .withClientFactory(clientFactory)
+ .withCoderAndParseFn(
+ StringUtf8Coder.of(),
+ SimpleFunction.fromSerializableFunctionWithOutputType(
+ message -> {
+ throw exception;
+ },
+ TypeDescriptors.strings())));
+ errorHandler.close();
+
+ PAssert.thatSingleton(errorHandler.getOutput()).isEqualTo(1L);
+ PAssert.that(read).empty();
+ pipeline.run();
+ }
+
@Test
public void testProto() {
ProtoCoder<Primitive> coder = ProtoCoder.of(Primitive.class);
@@ -654,6 +698,66 @@ public class PubsubIOTest {
}
}
+ @Test
+ public void testWriteMalformedMessagesWithErrorHandler() throws Exception {
+ OutgoingMessage msg =
+ OutgoingMessage.of(
+ com.google.pubsub.v1.PubsubMessage.newBuilder()
+ .setData(ByteString.copyFromUtf8("foo"))
+ .build(),
+ 0,
+ null,
+ "projects/project/topics/topic1");
+
+ try (PubsubTestClientFactory factory =
+ PubsubTestClient.createFactoryForPublish(null, ImmutableList.of(msg), ImmutableList.of())) {
+ TimestampedValue<PubsubMessage> pubsubMsg =
+ TimestampedValue.of(
+ new PubsubMessage(
+ msg.getMessage().getData().toByteArray(),
+ Collections.emptyMap(),
+ msg.recordId())
+ .withTopic(msg.topic()),
+ Instant.ofEpochMilli(msg.getTimestampMsSinceEpoch()));
+
+ TimestampedValue<PubsubMessage> failingPubsubMsg =
+ TimestampedValue.of(
+ new PubsubMessage(
+ "foo".getBytes(StandardCharsets.UTF_8),
+ Collections.emptyMap(),
+ msg.recordId())
+ .withTopic("badTopic"),
+ Instant.ofEpochMilli(msg.getTimestampMsSinceEpoch()));
+
+ PCollection<PubsubMessage> messages =
+ pipeline.apply(
+ Create.timestamped(ImmutableList.of(pubsubMsg, failingPubsubMsg))
+ .withCoder(new PubsubMessageWithTopicCoder()));
+ messages.setIsBoundedInternal(PCollection.IsBounded.BOUNDED);
+ ErrorHandler<BadRecord, PCollection<Long>> badRecordErrorHandler =
+ pipeline.registerBadRecordErrorHandler(new ErrorSinkTransform());
+ // The most straightforward method to simulate a bad message is to have a format function that
+ // deterministically fails based on some value
+ messages.apply(
+ PubsubIO.writeMessages()
+ .toBuilder()
+ .setFormatFn(
+ (ValueInSingleWindow<PubsubMessage> messageAndWindow) -> {
+ if (messageAndWindow.getValue().getTopic().equals("badTopic")) {
+ throw new RuntimeException("expected exception");
+ }
+ return messageAndWindow.getValue();
+ })
+ .build()
+ .to("projects/project/topics/topic1")
+ .withClientFactory(factory)
+ .withErrorHandler(badRecordErrorHandler));
+ badRecordErrorHandler.close();
+ PAssert.thatSingleton(badRecordErrorHandler.getOutput()).isEqualTo(1L);
+ pipeline.run();
+ }
+ }
+
@Test
public void testReadMessagesWithCoderAndParseFn() {
Coder<PubsubMessage> coder = PubsubMessagePayloadOnlyCoder.of();