You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/14 21:16:33 UTC
(hudi) 05/14: [HUDI-7488] The BigQuerySyncTool can't work well when the hudi table schema changed (#10830)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8ed8a20b6902b95a3e698a519822bdb0210a0a7f
Author: steve-xi-awx <84...@users.noreply.github.com>
AuthorDate: Fri Mar 8 08:29:03 2024 +0800
[HUDI-7488] The BigQuerySyncTool can't work well when the hudi table schema changed (#10830)
---
.../gcp/bigquery/HoodieBigQuerySyncClient.java | 25 ++++++++++++++--------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java
index 5a23a4079ae..32430b53329 100644
--- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java
+++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/HoodieBigQuerySyncClient.java
@@ -44,6 +44,7 @@ import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.bigquery.ViewDefinition;
+import com.google.cloud.bigquery.StandardTableDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -198,16 +199,22 @@ public class HoodieBigQuerySyncClient extends HoodieSyncClient {
LOG.info("No table update is needed.");
return; // No need to update schema.
}
- ExternalTableDefinition.Builder builder = definition.toBuilder();
- builder.setSchema(finalSchema);
- builder.setAutodetect(false);
- if (definition.getHivePartitioningOptions() != null) {
- builder.setHivePartitioningOptions(definition.getHivePartitioningOptions().toBuilder().setRequirePartitionFilter(requirePartitionFilter).build());
+ if (!StringUtils.isNullOrEmpty(bigLakeConnectionId)) {
+ Table updatedTable =
+ existingTable.toBuilder().setDefinition(StandardTableDefinition.of(finalSchema)).build();
+ updatedTable.update();
+ } else {
+ ExternalTableDefinition.Builder builder = definition.toBuilder();
+ builder.setSchema(finalSchema);
+ builder.setAutodetect(false);
+ if (definition.getHivePartitioningOptions() != null) {
+ builder.setHivePartitioningOptions(definition.getHivePartitioningOptions().toBuilder().setRequirePartitionFilter(requirePartitionFilter).build());
+ }
+ Table updatedTable = existingTable.toBuilder()
+ .setDefinition(builder.build())
+ .build();
+ bigquery.update(updatedTable);
}
- Table updatedTable = existingTable.toBuilder()
- .setDefinition(builder.build())
- .build();
- bigquery.update(updatedTable);
}
public void createVersionsTable(String tableName, String sourceUri, String sourceUriPrefix, List<String> partitionFields) {