You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by re...@apache.org on 2023/05/29 03:06:40 UTC

[beam] branch master updated: Merge pull request #26919: warn if BigQuery failed rows collection is not processed

This is an automated email from the ASF dual-hosted git repository.

reuvenlax pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new fbeae980e93 Merge pull request #26919: warn if BigQuery failed rows collection is not processed
fbeae980e93 is described below

commit fbeae980e93ea64fc8cc3ad074cbc8aebd157691
Author: Reuven Lax <re...@google.com>
AuthorDate: Sun May 28 20:06:27 2023 -0700

    Merge pull request #26919: warn if BigQuery failed rows collection is not processed
---
 .../beam/runners/dataflow/DataflowRunner.java      | 56 ++++++++++++++
 .../beam/runners/dataflow/DataflowRunnerTest.java  | 89 +++++++++++++++++++++-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java       |  4 +-
 .../beam/sdk/io/gcp/bigquery/StorageApiLoads.java  |  5 ++
 .../sdk/io/gcp/bigquery/StreamingWriteTables.java  |  8 +-
 5 files changed, 157 insertions(+), 5 deletions(-)

diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index a2f6298ab10..2951b48bab4 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -105,6 +105,8 @@ import org.apache.beam.sdk.io.WriteFiles;
 import org.apache.beam.sdk.io.WriteFilesResult;
 import org.apache.beam.sdk.io.fs.ResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.gcp.bigquery.StorageApiLoads;
+import org.apache.beam.sdk.io.gcp.bigquery.StreamingWriteTables;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesAndMessageIdCoder;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageWithAttributesCoder;
@@ -169,6 +171,7 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Utf8;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.HashCode;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.hash.Hashing;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.Files;
@@ -1069,6 +1072,7 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
     }
 
     logWarningIfPCollectionViewHasNonDeterministicKeyCoder(pipeline);
+    logWarningIfBigqueryDLQUnused(pipeline);
     if (shouldActAsStreaming(pipeline)) {
       options.setStreaming(true);
 
@@ -1569,6 +1573,58 @@ public class DataflowRunner extends PipelineRunner<DataflowPipelineJob> {
 
   /////////////////////////////////////////////////////////////////////////////
 
+  private void logWarningIfBigqueryDLQUnused(Pipeline pipeline) {
+    Map<PCollection<?>, String> unconsumedDLQ = Maps.newHashMap();
+    pipeline.traverseTopologically(
+        new PipelineVisitor.Defaults() {
+          @Override
+          public CompositeBehavior enterCompositeTransform(Node node) {
+            PTransform<?, ?> transform = node.getTransform();
+            if (transform != null) {
+              TupleTag<?> failedTag = null;
+              String rootBigQueryTransform = "";
+              if (transform.getClass().equals(StorageApiLoads.class)) {
+                StorageApiLoads<?, ?> storageLoads = (StorageApiLoads<?, ?>) transform;
+                failedTag = storageLoads.getFailedRowsTag();
+                // For storage API the transform that outputs failed rows is nested one layer below
+                // BigQueryIO.
+                rootBigQueryTransform = node.getEnclosingNode().getFullName();
+              } else if (transform.getClass().equals(StreamingWriteTables.class)) {
+                StreamingWriteTables<?> streamingInserts = (StreamingWriteTables<?>) transform;
+                failedTag = streamingInserts.getFailedRowsTupleTag();
+                // For streaming inserts the transform that outputs failed rows is nested two layers
+                // below BigQueryIO.
+                rootBigQueryTransform = node.getEnclosingNode().getEnclosingNode().getFullName();
+              }
+              if (failedTag != null) {
+                PCollection<?> dlq = node.getOutputs().get(failedTag);
+                if (dlq != null) {
+                  unconsumedDLQ.put(dlq, rootBigQueryTransform);
+                }
+              }
+            }
+
+            for (PCollection<?> input : node.getInputs().values()) {
+              unconsumedDLQ.remove(input);
+            }
+            return CompositeBehavior.ENTER_TRANSFORM;
+          }
+
+          @Override
+          public void visitPrimitiveTransform(Node node) {
+            for (PCollection<?> input : node.getInputs().values()) {
+              unconsumedDLQ.remove(input);
+            }
+          }
+        });
+    for (String unconsumed : unconsumedDLQ.values()) {
+      LOG.warn(
+          "No transform processes the failed-inserts output from BigQuery sink: "
+              + unconsumed
+              + "! Not processing failed inserts means that those rows will be lost.");
+    }
+  }
+
   /** Outputs a warning about PCollection views without deterministic key coders. */
   private void logWarningIfPCollectionViewHasNonDeterministicKeyCoder(Pipeline pipeline) {
     // We need to wait till this point to determine the names of the transforms since only
diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
index a37c80a93b9..d16afeb91e0 100644
--- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java
@@ -64,6 +64,8 @@ import com.fasterxml.jackson.databind.SerializerProvider;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.api.services.bigquery.model.TableSchema;
 import com.google.api.services.dataflow.Dataflow;
 import com.google.api.services.dataflow.model.DataflowPackage;
 import com.google.api.services.dataflow.model.Job;
@@ -123,6 +125,8 @@ import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.io.WriteFiles;
 import org.apache.beam.sdk.io.WriteFilesResult;
 import org.apache.beam.sdk.io.fs.ResourceId;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
+import org.apache.beam.sdk.io.gcp.bigquery.WriteResult;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
 import org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage;
 import org.apache.beam.sdk.options.ExperimentalOptions;
@@ -164,6 +168,7 @@ import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PValues;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.TypeDescriptors;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.InvalidProtocolBufferException;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
@@ -2414,7 +2419,89 @@ public class DataflowRunnerTest implements Serializable {
   }
 
   @Test
-  public void testPubinkDynamicOverride() throws IOException {
+  public void testBigQueryDLQWarningStreamingInsertsConsumed() throws Exception {
+    testBigQueryDLQWarning(BigQueryIO.Write.Method.STREAMING_INSERTS, true);
+  }
+
+  @Test
+  public void testBigQueryDLQWarningStreamingInsertsNotConsumed() throws Exception {
+    testBigQueryDLQWarning(BigQueryIO.Write.Method.STREAMING_INSERTS, false);
+  }
+
+  @Test
+  public void testBigQueryDLQWarningStorageApiConsumed() throws Exception {
+    testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_WRITE_API, true);
+  }
+
+  @Test
+  public void testBigQueryDLQWarningStorageApiNotConsumed() throws Exception {
+    testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_WRITE_API, false);
+  }
+
+  @Test
+  public void testBigQueryDLQWarningStorageApiALOConsumed() throws Exception {
+    testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE, true);
+  }
+
+  @Test
+  public void testBigQueryDLQWarningStorageApiALONotConsumed() throws Exception {
+    testBigQueryDLQWarning(BigQueryIO.Write.Method.STORAGE_API_AT_LEAST_ONCE, false);
+  }
+
+  public void testBigQueryDLQWarning(BigQueryIO.Write.Method method, boolean processFailures)
+      throws IOException {
+    PipelineOptions options = buildPipelineOptions();
+    List<String> experiments =
+        new ArrayList<>(ImmutableList.of(GcpOptions.STREAMING_ENGINE_EXPERIMENT));
+    DataflowPipelineOptions dataflowOptions = options.as(DataflowPipelineOptions.class);
+    dataflowOptions.setExperiments(experiments);
+    dataflowOptions.setStreaming(true);
+    Pipeline p = Pipeline.create(options);
+
+    List<TableRow> testValues = Arrays.asList(new TableRow(), new TableRow());
+    PCollection<TableRow> input =
+        p.apply("CreateValuesBytes", Create.of(testValues))
+            .setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED);
+
+    BigQueryIO.Write<TableRow> write =
+        BigQueryIO.writeTableRows()
+            .to("project:dataset.table")
+            .withSchema(new TableSchema())
+            .withMethod(method)
+            .withoutValidation();
+    if (method == BigQueryIO.Write.Method.STORAGE_WRITE_API) {
+      write = write.withAutoSharding().withTriggeringFrequency(Duration.standardSeconds(1));
+    }
+    WriteResult result = input.apply("BQWrite", write);
+    if (processFailures) {
+      if (method == BigQueryIO.Write.Method.STREAMING_INSERTS) {
+        result
+            .getFailedInserts()
+            .apply(
+                MapElements.into(TypeDescriptors.voids())
+                    .via(SerializableFunctions.constant((Void) null)));
+      } else {
+        result
+            .getFailedStorageApiInserts()
+            .apply(
+                MapElements.into(TypeDescriptors.voids())
+                    .via(SerializableFunctions.constant((Void) null)));
+      }
+    }
+    p.run();
+
+    final String expectedWarning =
+        "No transform processes the failed-inserts output from BigQuery sink: BQWrite!"
+            + " Not processing failed inserts means that those rows will be lost.";
+    if (processFailures) {
+      expectedLogs.verifyNotLogged(expectedWarning);
+    } else {
+      expectedLogs.verifyWarn(expectedWarning);
+    }
+  }
+
+  @Test
+  public void testPubsubSinkDynamicOverride() throws IOException {
     PipelineOptions options = buildPipelineOptions();
     DataflowPipelineOptions dataflowOptions = options.as(DataflowPipelineOptions.class);
     dataflowOptions.setStreaming(true);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 3b7977484b8..aa4e1555a25 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -950,7 +950,7 @@ public class BigQueryIO {
 
     abstract @Nullable String getQueryTempDataset();
 
-    abstract TypedRead.Method getMethod();
+    public abstract TypedRead.Method getMethod();
 
     abstract DataFormat getFormat();
 
@@ -2208,7 +2208,7 @@ public class BigQueryIO {
 
     abstract @Nullable Duration getTriggeringFrequency();
 
-    abstract Write.Method getMethod();
+    public abstract Write.Method getMethod();
 
     abstract @Nullable ValueProvider<String> getLoadJobProjectId();
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java
index 1a49fd31b2e..d3ce53f8f37 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiLoads.java
@@ -45,6 +45,7 @@ public class StorageApiLoads<DestinationT, ElementT>
     extends PTransform<PCollection<KV<DestinationT, ElementT>>, WriteResult> {
   final TupleTag<KV<DestinationT, StorageApiWritePayload>> successfulConvertedRowsTag =
       new TupleTag<>("successfulRows");
+
   final TupleTag<BigQueryStorageApiInsertError> failedRowsTag = new TupleTag<>("failedRows");
 
   @Nullable TupleTag<TableRow> successfulWrittenRowsTag;
@@ -89,6 +90,10 @@ public class StorageApiLoads<DestinationT, ElementT>
     }
   }
 
+  public TupleTag<BigQueryStorageApiInsertError> getFailedRowsTag() {
+    return failedRowsTag;
+  }
+
   @Override
   public WriteResult expand(PCollection<KV<DestinationT, ElementT>> input) {
     Coder<StorageApiWritePayload> payloadCoder;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
index 00779193262..2cc08d7731f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StreamingWriteTables.java
@@ -303,11 +303,15 @@ public class StreamingWriteTables<ElementT>
         deterministicRecordIdFn);
   }
 
+  public <TupleTagT> TupleTag<TupleTagT> getFailedRowsTupleTag() {
+    return new TupleTag<>(FAILED_INSERTS_TAG_ID);
+  }
+
   @Override
   public WriteResult expand(PCollection<KV<TableDestination, ElementT>> input) {
     Preconditions.checkStateNotNull(elementCoder);
     if (extendedErrorInfo) {
-      TupleTag<BigQueryInsertError> failedInsertsTag = new TupleTag<>(FAILED_INSERTS_TAG_ID);
+      TupleTag<BigQueryInsertError> failedInsertsTag = getFailedRowsTupleTag();
       PCollectionTuple result =
           writeAndGetErrors(
               input,
@@ -321,7 +325,7 @@ public class StreamingWriteTables<ElementT>
           failedInserts,
           propagateSuccessful ? result.get(BatchedStreamingWrite.SUCCESSFUL_ROWS_TAG) : null);
     } else {
-      TupleTag<TableRow> failedInsertsTag = new TupleTag<>(FAILED_INSERTS_TAG_ID);
+      TupleTag<TableRow> failedInsertsTag = getFailedRowsTupleTag();
       PCollectionTuple result =
           writeAndGetErrors(
               input,