You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ah...@apache.org on 2024/10/15 19:28:30 UTC
(beam) branch master updated: Enable BigQuery CDC configuration for Python BigQuery sink (#32529)
This is an automated email from the ASF dual-hosted git repository.
ahmedabualsaud pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e52868c29f9 Enable BigQuery CDC configuration for Python BigQuery sink (#32529)
e52868c29f9 is described below
commit e52868c29f9d6cae3c91aedff9814f90de241b36
Author: pablo rodriguez defino <pr...@gmail.com>
AuthorDate: Tue Oct 15 12:28:22 2024 -0700
Enable BigQuery CDC configuration for Python BigQuery sink (#32529)
* include CDC configuration on the storage write transform provider
* adding the primary key configuration for CDC and tests
* fixing List.of references to use ImmutableList
* fixing test, missing calling the cdc info row builder() method
* fix test, add config validations
* added the xlang params to storage write python wrapper
* adding missing comma
* shortening property name
* changing xlang config property
* set use cdc schema property as nullable, added safe retrieval method
* fixes property name reference and argument type definition
* python format fix
* adding xlang IT with BQ
* adding missing primary key column to test
* python format fix
* format xlang test
* more format xlang test fixes
* and more format xlang test fixes
* adding missing import
* missing self reference
* enabled create if needed functionality for CDC python integration, implemented table constraint support on the bigquery fake dataset services
* Update bigquery.py
* triggering the xlang tests
* fixing lint
* addressing few comments
* cdc info is added after row transformation now
* remove not used param
* removed typing information for callable
* adding test for cdc using dicts as input and cdc write callable
* simplifying the xlang configuration from python perspective, will add callable on a future PR
* spotless apply
* wrong property passed to xlang builder
* missing self
* fixing xlang it
* fixes wrong property reference
* change cdc xlang test to use beam.io.WriteToBigQuery
* force another build
* modifying comment to trigger build.
* addressing PR comments, included new dicts based test for xlang python tests, included the CDC configurations into the existing RowDynamicDestinations object, improved error message for mutation information schema checks.
---
.../beam_PostCommit_Python_Xlang_Gcp_Dataflow.json | 3 +-
.../beam_PostCommit_Python_Xlang_Gcp_Direct.json | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 8 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 122 +++++++++++++++++++--
.../beam/sdk/io/gcp/testing/TableContainer.java | 32 +++++-
...StorageWriteApiSchemaTransformProviderTest.java | 115 ++++++++++++++++++-
.../io/external/xlang_bigqueryio_it_test.py | 122 +++++++++++++++++++++
sdks/python/apache_beam/io/gcp/bigquery.py | 30 ++++-
8 files changed, 414 insertions(+), 20 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json b/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json
index 6b3a9dc134e..27c1f3ae26c 100644
--- a/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json
+++ b/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json
@@ -1,4 +1,5 @@
{
- "comment": "Modify this file in a trivial way to cause this test suite to run"
+ "comment": "Modify this file in a trivial way to cause this test suite to run",
+ "modification": 1
}
diff --git a/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json b/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
index e3d6056a5de..b2683333323 100644
--- a/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
+++ b/.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to run",
- "modification": 1
+ "modification": 2
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 88dfa2c2634..84bf90bd412 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -2259,6 +2259,7 @@ public class BigQueryIO {
.withFormatFunction(RowMutation::getTableRow)
.withRowMutationInformationFn(RowMutation::getMutationInformation);
}
+
/**
* A {@link PTransform} that writes a {@link PCollection} containing {@link GenericRecord
* GenericRecords} to a BigQuery table.
@@ -2367,8 +2368,10 @@ public class BigQueryIO {
abstract WriteDisposition getWriteDisposition();
abstract Set<SchemaUpdateOption> getSchemaUpdateOptions();
+
/** Table description. Default is empty. */
abstract @Nullable String getTableDescription();
+
/** An option to indicate if table validation is desired. Default is true. */
abstract boolean getValidate();
@@ -3455,7 +3458,10 @@ public class BigQueryIO {
LOG.error("The Storage API sink does not support the WRITE_TRUNCATE write disposition.");
}
if (getRowMutationInformationFn() != null) {
- checkArgument(getMethod() == Method.STORAGE_API_AT_LEAST_ONCE);
+ checkArgument(
+ getMethod() == Method.STORAGE_API_AT_LEAST_ONCE,
+ "When using row updates on BigQuery, StorageWrite API should execute using"
+ + " \"at least once\" mode.");
checkArgument(
getCreateDisposition() == CreateDisposition.CREATE_NEVER || getPrimaryKey() != null,
"If specifying CREATE_IF_NEEDED along with row updates, a primary key needs to be specified");
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
index 980d783ec43..c1c06fc592f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
@@ -20,6 +20,7 @@ package org.apache.beam.sdk.io.gcp.bigquery.providers;
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
+import com.google.api.services.bigquery.model.TableConstraints;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.auto.service.AutoService;
import com.google.auto.value.AutoValue;
@@ -27,6 +28,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import javax.annotation.Nullable;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
@@ -37,6 +39,7 @@ import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryStorageApiInsertError;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils;
import org.apache.beam.sdk.io.gcp.bigquery.DynamicDestinations;
+import org.apache.beam.sdk.io.gcp.bigquery.RowMutationInformation;
import org.apache.beam.sdk.io.gcp.bigquery.TableDestination;
import org.apache.beam.sdk.io.gcp.bigquery.WriteResult;
import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryStorageWriteApiSchemaTransformProvider.BigQueryStorageWriteApiSchemaTransformConfiguration;
@@ -87,6 +90,14 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
private static final String FAILED_ROWS_WITH_ERRORS_TAG = "FailedRowsWithErrors";
// magic string that tells us to write to dynamic destinations
protected static final String DYNAMIC_DESTINATIONS = "DYNAMIC_DESTINATIONS";
+ protected static final String ROW_PROPERTY_MUTATION_INFO = "row_mutation_info";
+ protected static final String ROW_PROPERTY_MUTATION_TYPE = "mutation_type";
+ protected static final String ROW_PROPERTY_MUTATION_SQN = "change_sequence_number";
+ protected static final Schema ROW_SCHEMA_MUTATION_INFO =
+ Schema.builder()
+ .addStringField("mutation_type")
+ .addStringField("change_sequence_number")
+ .build();
@Override
protected SchemaTransform from(
@@ -257,6 +268,20 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
@Nullable
public abstract ErrorHandling getErrorHandling();
+ @SchemaFieldDescription(
+ "This option enables the use of BigQuery CDC functionality. The expected PCollection"
+ + " should contain Beam Rows with a schema wrapping the record to be inserted and"
+ + " adding the CDC info similar to: {row_mutation_info: {mutation_type:\"...\", "
+ + "change_sequence_number:\"...\"}, record: {...}}")
+ @Nullable
+ public abstract Boolean getUseCdcWrites();
+
+ @SchemaFieldDescription(
+ "If CREATE_IF_NEEDED disposition is set, BigQuery table(s) will be created with this"
+ + " columns as primary key. Required when CDC writes are enabled with CREATE_IF_NEEDED.")
+ @Nullable
+ public abstract List<String> getPrimaryKey();
+
/** Builder for {@link BigQueryStorageWriteApiSchemaTransformConfiguration}. */
@AutoValue.Builder
public abstract static class Builder {
@@ -277,6 +302,10 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
public abstract Builder setErrorHandling(ErrorHandling errorHandling);
+ public abstract Builder setUseCdcWrites(Boolean cdcWrites);
+
+ public abstract Builder setPrimaryKey(List<String> pkColumns);
+
/** Builds a {@link BigQueryStorageWriteApiSchemaTransformConfiguration} instance. */
public abstract BigQueryStorageWriteApiSchemaTransformProvider
.BigQueryStorageWriteApiSchemaTransformConfiguration
@@ -343,15 +372,27 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
}
private static class RowDynamicDestinations extends DynamicDestinations<Row, String> {
- Schema schema;
+ final Schema schema;
+ final String fixedDestination;
+ final List<String> primaryKey;
RowDynamicDestinations(Schema schema) {
this.schema = schema;
+ this.fixedDestination = null;
+ this.primaryKey = null;
+ }
+
+ public RowDynamicDestinations(
+ Schema schema, String fixedDestination, List<String> primaryKey) {
+ this.schema = schema;
+ this.fixedDestination = fixedDestination;
+ this.primaryKey = primaryKey;
}
@Override
public String getDestination(ValueInSingleWindow<Row> element) {
- return element.getValue().getString("destination");
+ return Optional.ofNullable(fixedDestination)
+ .orElseGet(() -> element.getValue().getString("destination"));
}
@Override
@@ -363,6 +404,17 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
public TableSchema getSchema(String destination) {
return BigQueryUtils.toTableSchema(schema);
}
+
+ @Override
+ public TableConstraints getTableConstraints(String destination) {
+ return Optional.ofNullable(this.primaryKey)
+ .filter(pk -> !pk.isEmpty())
+ .map(
+ pk ->
+ new TableConstraints()
+ .setPrimaryKey(new TableConstraints.PrimaryKey().setColumns(pk)))
+ .orElse(null);
+ }
}
@Override
@@ -453,6 +505,13 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
}
}
+ void validateDynamicDestinationsExpectedSchema(Schema schema) {
+ checkArgument(
+ schema.getFieldNames().containsAll(Arrays.asList("destination", "record")),
+ "When writing to dynamic destinations, we expect Row Schema with a "
+ + "\"destination\" string field and a \"record\" Row field.");
+ }
+
BigQueryIO.Write<Row> createStorageWriteApiTransform(Schema schema) {
Method writeMethod =
configuration.getUseAtLeastOnceSemantics() != null
@@ -466,11 +525,11 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
.withFormatFunction(BigQueryUtils.toTableRow())
.withWriteDisposition(WriteDisposition.WRITE_APPEND);
- if (configuration.getTable().equals(DYNAMIC_DESTINATIONS)) {
- checkArgument(
- schema.getFieldNames().equals(Arrays.asList("destination", "record")),
- "When writing to dynamic destinations, we expect Row Schema with a "
- + "\"destination\" string field and a \"record\" Row field.");
+ // in case CDC writes are configured we validate and include them in the configuration
+ if (Optional.ofNullable(configuration.getUseCdcWrites()).orElse(false)) {
+ write = validateAndIncludeCDCInformation(write, schema);
+ } else if (configuration.getTable().equals(DYNAMIC_DESTINATIONS)) {
+ validateDynamicDestinationsExpectedSchema(schema);
write =
write
.to(new RowDynamicDestinations(schema.getField("record").getType().getRowSchema()))
@@ -485,6 +544,7 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
configuration.getCreateDisposition().toUpperCase());
write = write.withCreateDisposition(createDisposition);
}
+
if (!Strings.isNullOrEmpty(configuration.getWriteDisposition())) {
WriteDisposition writeDisposition =
BigQueryStorageWriteApiSchemaTransformConfiguration.WRITE_DISPOSITIONS.get(
@@ -498,5 +558,53 @@ public class BigQueryStorageWriteApiSchemaTransformProvider
return write;
}
+
+ BigQueryIO.Write<Row> validateAndIncludeCDCInformation(
+ BigQueryIO.Write<Row> write, Schema schema) {
+ checkArgument(
+ schema.getFieldNames().containsAll(Arrays.asList(ROW_PROPERTY_MUTATION_INFO, "record")),
+ "When writing using CDC functionality, we expect Row Schema with a "
+ + "\""
+ + ROW_PROPERTY_MUTATION_INFO
+ + "\" Row field and a \"record\" Row field.");
+
+ Schema rowSchema = schema.getField(ROW_PROPERTY_MUTATION_INFO).getType().getRowSchema();
+
+ checkArgument(
+ rowSchema.equals(ROW_SCHEMA_MUTATION_INFO),
+ "When writing using CDC functionality, we expect a \""
+ + ROW_PROPERTY_MUTATION_INFO
+ + "\" field of Row type with schema:\n"
+ + ROW_SCHEMA_MUTATION_INFO.toString()
+ + "\n"
+ + "Received \""
+ + ROW_PROPERTY_MUTATION_INFO
+ + "\" field with schema:\n"
+ + rowSchema.toString());
+
+ String tableDestination = null;
+
+ if (configuration.getTable().equals(DYNAMIC_DESTINATIONS)) {
+ validateDynamicDestinationsExpectedSchema(schema);
+ } else {
+ tableDestination = configuration.getTable();
+ }
+
+ return write
+ .to(
+ new RowDynamicDestinations(
+ schema.getField("record").getType().getRowSchema(),
+ tableDestination,
+ configuration.getPrimaryKey()))
+ .withFormatFunction(row -> BigQueryUtils.toTableRow(row.getRow("record")))
+ .withPrimaryKey(configuration.getPrimaryKey())
+ .withRowMutationInformationFn(
+ row ->
+ RowMutationInformation.of(
+ RowMutationInformation.MutationType.valueOf(
+ row.getRow(ROW_PROPERTY_MUTATION_INFO)
+ .getString(ROW_PROPERTY_MUTATION_TYPE)),
+ row.getRow(ROW_PROPERTY_MUTATION_INFO).getString(ROW_PROPERTY_MUTATION_SQN)));
+ }
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java
index b50aa4d32d7..b44b9596cc1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/TableContainer.java
@@ -18,11 +18,13 @@
package org.apache.beam.sdk.io.gcp.testing;
import com.google.api.services.bigquery.model.Table;
+import com.google.api.services.bigquery.model.TableConstraints;
import com.google.api.services.bigquery.model.TableRow;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.beam.sdk.io.gcp.bigquery.TableRowJsonCoder;
@@ -51,12 +53,24 @@ class TableContainer {
this.keyedRows = Maps.newHashMap();
this.ids = new ArrayList<>();
this.sizeBytes = 0L;
+ // extract primary key information from Table if present
+ List<String> pkColumns = primaryKeyColumns(table);
+ this.primaryKeyColumns = pkColumns;
+ this.primaryKeyColumnIndices = primaryColumnFieldIndices(pkColumns, table);
}
- // Only top-level columns supported.
- void setPrimaryKeyColumns(List<String> primaryKeyColumns) {
- this.primaryKeyColumns = primaryKeyColumns;
+ static @Nullable List<String> primaryKeyColumns(Table table) {
+ return Optional.ofNullable(table.getTableConstraints())
+ .flatMap(constraints -> Optional.ofNullable(constraints.getPrimaryKey()))
+ .map(TableConstraints.PrimaryKey::getColumns)
+ .orElse(null);
+ }
+ static @Nullable List<Integer> primaryColumnFieldIndices(
+ @Nullable List<String> primaryKeyColumns, Table table) {
+ if (primaryKeyColumns == null) {
+ return null;
+ }
Map<String, Integer> indices =
IntStream.range(0, table.getSchema().getFields().size())
.boxed()
@@ -65,7 +79,13 @@ class TableContainer {
for (String columnName : primaryKeyColumns) {
primaryKeyColumnIndices.add(Preconditions.checkStateNotNull(indices.get(columnName)));
}
- this.primaryKeyColumnIndices = primaryKeyColumnIndices;
+ return primaryKeyColumnIndices;
+ }
+
+ // Only top-level columns supported.
+ void setPrimaryKeyColumns(List<String> primaryKeyColumns) {
+ this.primaryKeyColumns = primaryKeyColumns;
+ this.primaryKeyColumnIndices = primaryColumnFieldIndices(primaryKeyColumns, table);
}
@Nullable
@@ -80,7 +100,7 @@ class TableContainer {
.stream()
.map(cell -> Preconditions.checkStateNotNull(cell.get("v")))
.collect(Collectors.toList());
- ;
+
return Preconditions.checkStateNotNull(primaryKeyColumnIndices).stream()
.map(cellValues::get)
.collect(Collectors.toList());
@@ -91,7 +111,7 @@ class TableContainer {
long addRow(TableRow row, String id) {
List<Object> primaryKey = getPrimaryKey(row);
- if (primaryKey != null) {
+ if (primaryKey != null && !primaryKey.isEmpty()) {
if (keyedRows.putIfAbsent(primaryKey, row) != null) {
throw new RuntimeException(
"Primary key validation error! Multiple inserts with the same primary key.");
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java
index 64ea0b11d1b..87ba2961461 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProviderTest.java
@@ -30,6 +30,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
import org.apache.beam.sdk.PipelineResult;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers;
import org.apache.beam.sdk.io.gcp.bigquery.providers.BigQueryStorageWriteApiSchemaTransformProvider.BigQueryStorageWriteApiSchemaTransform;
@@ -54,6 +56,7 @@ import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PCollectionRowTuple;
import org.apache.beam.sdk.values.Row;
import org.apache.beam.sdk.values.TypeDescriptors;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -221,6 +224,117 @@ public class BigQueryStorageWriteApiSchemaTransformProviderTest {
fakeDatasetService.getAllRows("project", "dataset", "dynamic_write_3").get(0)));
}
+ List<Row> createCDCUpsertRows(List<Row> rows, boolean dynamicDestination, String tablePrefix) {
+
+ Schema.Builder schemaBuilder =
+ Schema.builder()
+ .addRowField("record", SCHEMA)
+ .addRowField(
+ BigQueryStorageWriteApiSchemaTransformProvider.ROW_PROPERTY_MUTATION_INFO,
+ BigQueryStorageWriteApiSchemaTransformProvider.ROW_SCHEMA_MUTATION_INFO);
+
+ if (dynamicDestination) {
+ schemaBuilder = schemaBuilder.addStringField("destination");
+ }
+
+ Schema schemaWithCDC = schemaBuilder.build();
+ return IntStream.range(0, rows.size())
+ .mapToObj(
+ idx -> {
+ Row row = rows.get(idx);
+ Row.FieldValueBuilder rowBuilder =
+ Row.withSchema(schemaWithCDC)
+ .withFieldValue(
+ BigQueryStorageWriteApiSchemaTransformProvider.ROW_PROPERTY_MUTATION_INFO,
+ Row.withSchema(
+ BigQueryStorageWriteApiSchemaTransformProvider
+ .ROW_SCHEMA_MUTATION_INFO)
+ .withFieldValue(
+ BigQueryStorageWriteApiSchemaTransformProvider
+ .ROW_PROPERTY_MUTATION_TYPE,
+ "UPSERT")
+ .withFieldValue(
+ BigQueryStorageWriteApiSchemaTransformProvider
+ .ROW_PROPERTY_MUTATION_SQN,
+ "AAA" + idx)
+ .build())
+ .withFieldValue("record", row);
+ if (dynamicDestination) {
+ rowBuilder =
+ rowBuilder.withFieldValue("destination", tablePrefix + row.getInt64("number"));
+ }
+ return rowBuilder.build();
+ })
+ .collect(Collectors.toList());
+ }
+
+ @Test
+ public void testCDCWrites() throws Exception {
+ String tableSpec = "project:dataset.cdc_write";
+ List<String> primaryKeyColumns = ImmutableList.of("name");
+
+ BigQueryStorageWriteApiSchemaTransformConfiguration config =
+ BigQueryStorageWriteApiSchemaTransformConfiguration.builder()
+ .setUseAtLeastOnceSemantics(true)
+ .setTable(tableSpec)
+ .setUseCdcWrites(true)
+ .setPrimaryKey(primaryKeyColumns)
+ .build();
+
+ List<Row> rowsDuplicated =
+ Stream.concat(ROWS.stream(), ROWS.stream()).collect(Collectors.toList());
+
+ runWithConfig(config, createCDCUpsertRows(rowsDuplicated, false, ""));
+ p.run().waitUntilFinish();
+
+ assertTrue(
+ rowEquals(
+ rowsDuplicated.get(3),
+ fakeDatasetService.getAllRows("project", "dataset", "cdc_write").get(0)));
+ assertTrue(
+ rowEquals(
+ rowsDuplicated.get(4),
+ fakeDatasetService.getAllRows("project", "dataset", "cdc_write").get(1)));
+ assertTrue(
+ rowEquals(
+ rowsDuplicated.get(5),
+ fakeDatasetService.getAllRows("project", "dataset", "cdc_write").get(2)));
+ }
+
+ @Test
+ public void testCDCWriteToDynamicDestinations() throws Exception {
+ List<String> primaryKeyColumns = ImmutableList.of("name");
+ String dynamic = BigQueryStorageWriteApiSchemaTransformProvider.DYNAMIC_DESTINATIONS;
+ BigQueryStorageWriteApiSchemaTransformConfiguration config =
+ BigQueryStorageWriteApiSchemaTransformConfiguration.builder()
+ .setUseAtLeastOnceSemantics(true)
+ .setTable(dynamic)
+ .setUseCdcWrites(true)
+ .setPrimaryKey(primaryKeyColumns)
+ .build();
+
+ String baseTableSpec = "project:dataset.cdc_dynamic_write_";
+
+ List<Row> rowsDuplicated =
+ Stream.concat(ROWS.stream(), ROWS.stream()).collect(Collectors.toList());
+
+ runWithConfig(config, createCDCUpsertRows(rowsDuplicated, true, baseTableSpec));
+ p.run().waitUntilFinish();
+
+ assertTrue(
+ rowEquals(
+ rowsDuplicated.get(3),
+ fakeDatasetService.getAllRows("project", "dataset", "cdc_dynamic_write_1").get(0)));
+ assertTrue(
+ rowEquals(
+ rowsDuplicated.get(4),
+ fakeDatasetService.getAllRows("project", "dataset", "cdc_dynamic_write_2").get(0)));
+ assertTrue(
+ rowEquals(
+ rowsDuplicated.get(5),
+ fakeDatasetService.getAllRows("project", "dataset", "cdc_dynamic_write_3").get(0)));
+ }
+
@Test
public void testInputElementCount() throws Exception {
String tableSpec = "project:dataset.input_count";
@@ -292,7 +406,6 @@ public class BigQueryStorageWriteApiSchemaTransformProviderTest {
MapElements.into(TypeDescriptors.rows())
.via((rowAndError) -> rowAndError.<Row>getValue("failed_row")))
.setRowSchema(SCHEMA);
- ;
PAssert.that(failedRows).containsInAnyOrder(expectedFailedRows);
p.run().waitUntilFinish();
diff --git a/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py b/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py
index cfbb411b4e5..7f3a16e02aa 100644
--- a/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py
+++ b/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py
@@ -245,6 +245,128 @@ class BigQueryXlangStorageWriteIT(unittest.TestCase):
| StorageWriteToBigQuery(table=table_id))
hamcrest_assert(p, bq_matcher)
+ def test_write_with_beam_rows_cdc(self):
+ table = 'write_with_beam_rows_cdc'
+ table_id = '{}:{}.{}'.format(self.project, self.dataset_id, table)
+
+ expected_data_on_bq = [
+ # (name, value)
+ {
+ "name": "cdc_test",
+ "value": 5,
+ }
+ ]
+
+ rows_with_cdc = [
+ beam.Row(
+ row_mutation_info=beam.Row(
+ mutation_type="UPSERT", change_sequence_number="AAA/2"),
+ record=beam.Row(name="cdc_test", value=5)),
+ beam.Row(
+ row_mutation_info=beam.Row(
+ mutation_type="UPSERT", change_sequence_number="AAA/1"),
+ record=beam.Row(name="cdc_test", value=3))
+ ]
+
+ bq_matcher = BigqueryFullResultMatcher(
+ project=self.project,
+ query="SELECT * FROM {}.{}".format(self.dataset_id, table),
+ data=self.parse_expected_data(expected_data_on_bq))
+
+ with beam.Pipeline(argv=self.args) as p:
+ _ = (
+ p
+ | beam.Create(rows_with_cdc)
+ | beam.io.WriteToBigQuery(
+ table=table_id,
+ method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API,
+ use_at_least_once=True,
+ use_cdc_writes=True,
+ primary_key=["name"]))
+ hamcrest_assert(p, bq_matcher)
+
+ def test_write_with_dicts_cdc(self):
+ table = 'write_with_dicts_cdc'
+ table_id = '{}:{}.{}'.format(self.project, self.dataset_id, table)
+
+ expected_data_on_bq = [
+ # (name, value)
+ {
+ "name": "cdc_test",
+ "value": 5,
+ }
+ ]
+
+ data_with_cdc = [
+ # record: (name, value)
+ {
+ 'row_mutation_info': {
+ 'mutation_type': 'UPSERT', 'change_sequence_number': 'AAA/2'
+ },
+ 'record': {
+ 'name': 'cdc_test', 'value': 5
+ }
+ },
+ {
+ 'row_mutation_info': {
+ 'mutation_type': 'UPSERT', 'change_sequence_number': 'AAA/1'
+ },
+ 'record': {
+ 'name': 'cdc_test', 'value': 3
+ }
+ }
+ ]
+
+ schema = {
+ "fields": [
+ # include both record and mutation info fields as part of the schema
+ {
+ "name": "row_mutation_info",
+ "type": "STRUCT",
+ "fields": [
+ # setting both fields are required
+ {
+ "name": "mutation_type",
+ "type": "STRING",
+ "mode": "REQUIRED"
+ },
+ {
+ "name": "change_sequence_number",
+ "type": "STRING",
+ "mode": "REQUIRED"
+ }
+ ]
+ },
+ {
+ "name": "record",
+ "type": "STRUCT",
+ "fields": [{
+ "name": "name", "type": "STRING"
+ }, {
+ "name": "value", "type": "INTEGER"
+ }]
+ }
+ ]
+ }
+
+ bq_matcher = BigqueryFullResultMatcher(
+ project=self.project,
+ query="SELECT * FROM {}.{}".format(self.dataset_id, table),
+ data=self.parse_expected_data(expected_data_on_bq))
+
+ with beam.Pipeline(argv=self.args) as p:
+ _ = (
+ p
+ | beam.Create(data_with_cdc)
+ | beam.io.WriteToBigQuery(
+ table=table_id,
+ method=beam.io.WriteToBigQuery.Method.STORAGE_WRITE_API,
+ use_at_least_once=True,
+ use_cdc_writes=True,
+ schema=schema,
+ primary_key=["name"]))
+ hamcrest_assert(p, bq_matcher)
+
def test_write_to_dynamic_destinations(self):
base_table_spec = '{}.dynamic_dest_'.format(self.dataset_id)
spec_with_project = '{}:{}'.format(self.project, base_table_spec)
diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py
index 2cb64742f26..11e0d098b2f 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery.py
@@ -1930,6 +1930,8 @@ class WriteToBigQuery(PTransform):
load_job_project_id=None,
max_insert_payload_size=MAX_INSERT_PAYLOAD_SIZE,
num_streaming_keys=DEFAULT_SHARDS_PER_DESTINATION,
+ use_cdc_writes: bool = False,
+ primary_key: List[str] = None,
expansion_service=None):
"""Initialize a WriteToBigQuery transform.
@@ -2095,6 +2097,15 @@ bigquery_v2_messages.TableSchema`. or a `ValueProvider` that has a JSON string,
GCP expansion service. Used for STORAGE_WRITE_API method.
max_insert_payload_size: The maximum byte size for a BigQuery legacy
streaming insert payload.
+ use_cdc_writes: Configure the usage of CDC writes on BigQuery.
+ The argument can be used by passing True and the Beam Rows will be
+ sent as they are to the BigQuery sink which expects a 'record'
+ and 'row_mutation_info' properties.
+ Used for STORAGE_WRITE_API, working on 'at least once' mode.
+ primary_key: When using CDC write on BigQuery and
+ CREATE_IF_NEEDED mode for the underlying tables a list of column names
+ is required to be configured as the primary key. Used for
+ STORAGE_WRITE_API, working on 'at least once' mode.
"""
self._table = table
self._dataset = dataset
@@ -2136,6 +2147,8 @@ bigquery_v2_messages.TableSchema`. or a `ValueProvider` that has a JSON string,
self.load_job_project_id = load_job_project_id
self._max_insert_payload_size = max_insert_payload_size
self._num_streaming_keys = num_streaming_keys
+ self._use_cdc_writes = use_cdc_writes
+ self._primary_key = primary_key
# Dict/schema methods were moved to bigquery_tools, but keep references
# here for backward compatibility.
@@ -2289,8 +2302,9 @@ bigquery_v2_messages.TableSchema`. or a `ValueProvider` that has a JSON string,
use_at_least_once=self.use_at_least_once,
with_auto_sharding=self.with_auto_sharding,
num_storage_api_streams=self._num_storage_api_streams,
+ use_cdc_writes=self._use_cdc_writes,
+ primary_key=self._primary_key,
expansion_service=self.expansion_service)
-
else:
raise ValueError(f"Unsupported method {method_to_use}")
@@ -2518,6 +2532,10 @@ class StorageWriteToBigQuery(PTransform):
# fields for rows sent to Storage API with dynamic destinations
DESTINATION = "destination"
RECORD = "record"
+ # field names for rows sent to Storage API for CDC functionality
+ CDC_INFO = "row_mutation_info"
+ CDC_MUTATION_TYPE = "mutation_type"
+ CDC_SQN = "change_sequence_number"
# magic string to tell Java that these rows are going to dynamic destinations
DYNAMIC_DESTINATIONS = "DYNAMIC_DESTINATIONS"
@@ -2532,6 +2550,8 @@ class StorageWriteToBigQuery(PTransform):
use_at_least_once=False,
with_auto_sharding=False,
num_storage_api_streams=0,
+ use_cdc_writes: bool = False,
+ primary_key: List[str] = None,
expansion_service=None):
self._table = table
self._table_side_inputs = table_side_inputs
@@ -2542,6 +2562,8 @@ class StorageWriteToBigQuery(PTransform):
self._use_at_least_once = use_at_least_once
self._with_auto_sharding = with_auto_sharding
self._num_storage_api_streams = num_storage_api_streams
+ self._use_cdc_writes = use_cdc_writes
+ self._primary_key = primary_key
self._expansion_service = expansion_service or BeamJarExpansionService(
'sdks:java:io:google-cloud-platform:expansion-service:build')
@@ -2552,11 +2574,11 @@ class StorageWriteToBigQuery(PTransform):
is_rows = True
except TypeError as exn:
raise ValueError(
- "A schema is required in order to prepare rows"
+ "A schema is required in order to prepare rows "
"for writing with STORAGE_WRITE_API.") from exn
elif callable(self._schema):
raise NotImplementedError(
- "Writing with dynamic schemas is not"
+ "Writing with dynamic schemas is not "
"supported for this write method.")
elif isinstance(self._schema, vp.ValueProvider):
schema = self._schema.get()
@@ -2624,6 +2646,8 @@ class StorageWriteToBigQuery(PTransform):
auto_sharding=self._with_auto_sharding,
num_streams=self._num_storage_api_streams,
use_at_least_once_semantics=self._use_at_least_once,
+ use_cdc_writes=self._use_cdc_writes,
+ primary_key=self._primary_key,
error_handling={
'output': StorageWriteToBigQuery.FAILED_ROWS_WITH_ERRORS
}))