You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by re...@apache.org on 2023/05/29 03:06:40 UTC
[beam] branch master updated: Merge pull request #26919: warn if BigQuery failed rows collection is not processed
This is an automated email from the ASF dual-hosted git repository.
reuvenlax pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new fbeae980e93 Merge pull request #26919: warn if BigQuery failed rows collection is not processed
fbeae980e93 is described below
commit fbeae980e93ea64fc8cc3ad074cbc8aebd157691
Author: Reuven Lax <re...@google.com>
AuthorDate: Sun May 28 20:06:27 2023 -0700
Merge pull request #26919: warn if BigQuery failed rows collection is not processed
---
.../beam/runners/dataflow/DataflowRunner.java | 56 ++++++++++++++
.../beam/runners/dataflow/DataflowRunnerTest.java | 89 +++++++++++++++++++++-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 4 +-
.../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 5 ++
.../sdk/io/gcp/bigquery/StreamingWriteTables.java | 8 +-
5 files changed, 157 insertions(+), 5 deletions(-)
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index a2f6298ab10..2951b48bab4 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -105,6 +105,8 @@ import org.apache.beam.sdk.io.WriteFiles;
import org.apache.beam.sdk.io.WriteFilesResult;
import org.apache.beam.sdk.io.fs.ResolveOptions;
import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.gcp.bigquery.StorageApiLoads;
+import org.apache.beam.sdk.io.gcp.bigquery.StreamingWriteTables;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesAndMessageIdCoder;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder;
@@ -169,6 +171,7 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Utf8;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files;
@@ -1069,6 +1072,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
}
logWarningIfPCollectionViewHasNonDeterministicKeyCoder(pipeline);
+ logWarningIfBigqueryDLQUnused(pipeline);
if (shouldActAsStreaming(pipeline)) {
options.setStreaming(true);
@@ -1569,6 +1573,58 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
/////////////////////////////////////////////////////////////////////////////
+ private void logWarningIfBigqueryDLQUnused(Pipeline pipeline) {
+ Map<PCollection<?>, String> unconsumedDLQ = Maps.newHashMap();
+ pipeline.traverseTopologically(
+ new PipelineVisitor.Defaults() {
+ @Override
+ public CompositeBehavior enterCompositeTransform(Node node) {
+ PTransform<?, ?> transform = node.getTransform();
+ if (transform != null) {
+ TupleTag<?> failedTag = null;
+ String rootBigQueryTransform = "";
+ if (transform.getClass().equals(StorageApiLoads.class)) {
+ StorageApiLoads<?, ?> storageLoads = (StorageApiLoads<?, ?>) transform;
+ failedTag = storageLoads.getFailedRowsTag();
+ // For storage API the transform that outputs failed rows is nested one layer below
+ // BigQueryIO.
+ rootBigQueryTransform = node.getEnclosingNode().getFullName();
+ } else if (transform.getClass().equals(StreamingWriteTables.class)) {
+ StreamingWriteTables<?> streamingInserts = (StreamingWriteTables<?>) transform;
+ failedTag = streamingInserts.getFailedRowsTupleTag();
+ // For streaming inserts the transform that outputs failed rows is nested two layers
+ // below BigQueryIO.
+ rootBigQueryTransform = node.getEnclosingNode().getEnclosingNode().getFullName();
+ }
+ if (failedTag != null) {
+ PCollection<?> dlq = node.getOutputs().get(failedTag);
+ if (dlq != null) {
+ unconsumedDLQ.put(dlq, rootBigQueryTransform);
+ }
+ }
+ }
+
+ for (PCollection<?> input : node.getInputs().values()) {
+ unconsumedDLQ.remove(input);
+ }
+ return CompositeBehavior.ENTER_TRANSFORM;
+ }
+
+ @Override
+ public void visitPrimitiveTransform(Node node) {
+ for (PCollection<?> input : node.getInputs().values()) {
+ unconsumedDLQ.remove(input);
+ }
+ }
+ });
+ for (String unconsumed : unconsumedDLQ.values()) {
+ LOG.warn(
+ "No transform processes the failed-inserts output from BigQuery sink: "
+ + unconsumed
+ + "! Not processing failed inserts means that those rows will be lost.");
+ }
+ }
+
/** Outputs a warning about PCollection views without deterministic key coders. */
private void logWarningIfPCollectionViewHasNonDeterministicKeyCoder(Pipeline pipeline) {
// We need to wait till this point to determine the names of the transforms since only
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index a37c80a93b9..d16afeb91e0 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -64,6 +64,8 @@ import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
import com.google.api.services.dataflow.Dataflow;
import com.google.api.services.dataflow.model.DataflowPackage;
import com.google.api.services.dataflow.model.Job;
@@ -123,6 +125,8 @@ import org.apache.beam.sdk.io.TextIO;
import org.apache.beam.sdk.io.WriteFiles;
import org.apache.beam.sdk.io.WriteFilesResult;
import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
+import org.apache.beam.sdk.io.gcp.bigquery.WriteResult;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
import org.apache.beam.sdk.options.ExperimentalOptions;
@@ -164,6 +168,7 @@ import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PValues;
import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TypeDescriptors;
import org.apache.beam.sdk.values.WindowingStrategy;
import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
@@ -2414,7 +2419,89 @@ public class DataflowRunnerTest implements Serializable {
}
@Test
- public void testPubinkDynamicOverride() throws IOException {
+ public void testBigQueryDLQWarningStreamingInsertsConsumed() throws Exception {
+ testBigQueryDLQWarning(BigQueryIO.Write.Method.STREAMING_INSERTS, true);
+ }
+
+ @Test
+ public void testBigQueryDLQWarningStreamingInsertsNotConsumed() throws Exception {
+ testBigQueryDLQWarning(BigQueryIO.Write.Method.STREAMING_INSERTS, false);
+ }
+
+ @Test
+ public void testBigQueryDLQWarningStorageApiConsumed() throws Exception {
+ testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_WRITE_API, true);
+ }
+
+ @Test
+ public void testBigQueryDLQWarningStorageApiNotConsumed() throws Exception {
+ testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_WRITE_API, false);
+ }
+
+ @Test
+ public void testBigQueryDLQWarningStorageApiALOConsumed() throws Exception {
+ testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE, true);
+ }
+
+ @Test
+ public void testBigQueryDLQWarningStorageApiALONotConsumed() throws Exception {
+ testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE, false);
+ }
+
+ public void testBigQueryDLQWarning(BigQueryIO.Write.Method method, boolean processFailures)
+ throws IOException {
+ PipelineOptions options = buildPipelineOptions();
+ List<String> experiments =
+ new ArrayList<>(ImmutableList.of(GcpOptions.STREAMING_ENGINE_EXPERIMENT));
+ DataflowPipelineOptions dataflowOptions = options.as(DataflowPipelineOptions.class);
+ dataflowOptions.setExperiments(experiments);
+ dataflowOptions.setStreaming(true);
+ Pipeline p = Pipeline.create(options);
+
+ List<TableRow> testValues = Arrays.asList(new TableRow(), new TableRow());
+ PCollection<TableRow> input =
+ p.apply("CreateValuesBytes", Create.of(testValues))
+ .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
+
+ BigQueryIO.Write<TableRow> write =
+ BigQueryIO.writeTableRows()
+ .to("project:dataset.table")
+ .withSchema(new TableSchema())
+ .withMethod(method)
+ .withoutValidation();
+ if (method == BigQueryIO.Write.Method.STORAGE_WRITE_API) {
+ write = write.withAutoSharding().withTriggeringFrequency(Duration.standardSeconds(1));
+ }
+ WriteResult result = input.apply("BQWrite", write);
+ if (processFailures) {
+ if (method == BigQueryIO.Write.Method.STREAMING_INSERTS) {
+ result
+ .getFailedInserts()
+ .apply(
+ MapElements.into(TypeDescriptors.voids())
+ .via(SerializableFunctions.constant((Void) null)));
+ } else {
+ result
+ .getFailedStorageApiInserts()
+ .apply(
+ MapElements.into(TypeDescriptors.voids())
+ .via(SerializableFunctions.constant((Void) null)));
+ }
+ }
+ p.run();
+
+ final String expectedWarning =
+ "No transform processes the failed-inserts output from BigQuery sink: BQWrite!"
+ + " Not processing failed inserts means that those rows will be lost.";
+ if (processFailures) {
+ expectedLogs.verifyNotLogged(expectedWarning);
+ } else {
+ expectedLogs.verifyWarn(expectedWarning);
+ }
+ }
+
+ @Test
+ public void testPubsubSinkDynamicOverride() throws IOException {
PipelineOptions options = buildPipelineOptions();
DataflowPipelineOptions dataflowOptions = options.as(DataflowPipelineOptions.class);
dataflowOptions.setStreaming(true);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 3b7977484b8..aa4e1555a25 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -950,7 +950,7 @@ public class BigQueryIO {
abstract @Nullable String getQueryTempDataset();
- abstract TypedRead.Method getMethod();
+ public abstract TypedRead.Method getMethod();
abstract DataFormat getFormat();
@@ -2208,7 +2208,7 @@ public class BigQueryIO {
abstract @Nullable Duration getTriggeringFrequency();
- abstract Write.Method getMethod();
+ public abstract Write.Method getMethod();
abstract @Nullable ValueProvider<String> getLoadJobProjectId();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java
index 1a49fd31b2e..d3ce53f8f37 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java
@@ -45,6 +45,7 @@ public class StorageApiLoads<DestinationT, ElementT>
extends PTransform<PCollection<KV<DestinationT, ElementT>>, WriteResult> {
final TupleTag<KV<DestinationT, StorageApiWritePayload>> successfulConvertedRowsTag =
new TupleTag<>("successfulRows");
+
final TupleTag<BigQueryStorageApiInsertError> failedRowsTag = new TupleTag<>("failedRows");
@Nullable TupleTag<TableRow> successfulWrittenRowsTag;
@@ -89,6 +90,10 @@ public class StorageApiLoads<DestinationT, ElementT>
}
}
+ public TupleTag<BigQueryStorageApiInsertError> getFailedRowsTag() {
+ return failedRowsTag;
+ }
+
@Override
public WriteResult expand(PCollection<KV<DestinationT, ElementT>> input) {
Coder<StorageApiWritePayload> payloadCoder;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
index 00779193262..2cc08d7731f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
@@ -303,11 +303,15 @@ public class StreamingWriteTables<ElementT>
deterministicRecordIdFn);
}
+ public <TupleTagT> TupleTag<TupleTagT> getFailedRowsTupleTag() {
+ return new TupleTag<>(FAILED_INSERTS_TAG_ID);
+ }
+
@Override
public WriteResult expand(PCollection<KV<TableDestination, ElementT>> input) {
Preconditions.checkStateNotNull(elementCoder);
if (extendedErrorInfo) {
- TupleTag<BigQueryInsertError> failedInsertsTag = new TupleTag<>(FAILED_INSERTS_TAG_ID);
+ TupleTag<BigQueryInsertError> failedInsertsTag = getFailedRowsTupleTag();
PCollectionTuple result =
writeAndGetErrors(
input,
@@ -321,7 +325,7 @@ public class StreamingWriteTables<ElementT>
failedInserts,
propagateSuccessful ? result.get(BatchedStreamingWrite.SUCCESSFUL_ROWS_TAG) : null);
} else {
- TupleTag<TableRow> failedInsertsTag = new TupleTag<>(FAILED_INSERTS_TAG_ID);
+ TupleTag<TableRow> failedInsertsTag = getFailedRowsTupleTag();
PCollectionTuple result =
writeAndGetErrors(
input,