You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/04/15 09:02:12 UTC
(hudi) branch master updated: feat(sync): Map BLOB type to struct in Hive and BigQuery sync (#18482)
This is an automated email from the ASF dual-hosted git repository.
vhs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new da2667c4c6e6 feat(sync): Map BLOB type to struct in Hive and BigQuery sync (#18482)
da2667c4c6e6 is described below
commit da2667c4c6e6b7cd34edf85c29fca271f14aa71d
Author: voonhous <vo...@gmail.com>
AuthorDate: Wed Apr 15 17:02:04 2026 +0800
feat(sync): Map BLOB type to struct in Hive and BigQuery sync (#18482)
* feat(sync): Map BLOB type to struct in Hive and BigQuery sync
- BLOB columns were not handled in HiveSchemaUtil and BigQuerySchemaResolver, causing UnsupportedOperationException when syncing tables with blob columns.
- Map BLOB to its underlying physical type (struct) so external engines can read via Hive metastore and BigQuery.
* Add test for nullable and nested blobs
---
.../hudi/gcp/bigquery/BigQuerySchemaResolver.java | 1 +
.../gcp/bigquery/TestBigQuerySchemaResolver.java | 56 ++++++++++++++++++++++
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 1 +
.../apache/hudi/hive/util/TestHiveSchemaUtil.java | 14 +++++-
4 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
index e32921d7052c..d9d85c1743af 100644
--- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
+++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
@@ -147,6 +147,7 @@ public class BigQuerySchemaResolver {
standardSQLTypeName = StandardSQLTypeName.NUMERIC;
break;
case RECORD:
+ case BLOB:
return Field.newBuilder(name, StandardSQLTypeName.STRUCT,
FieldList.of(getFields(fieldSchema))).setMode(fieldMode).build();
case ARRAY:
diff --git a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
index 10ddfeef5632..f0d7cdc2a65d 100644
--- a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
+++ b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
@@ -281,6 +281,62 @@ public class TestBigQuerySchemaResolver {
Assertions.assertEquals(PRIMITIVE_TYPES_BQ_SCHEMA, resolver.getTableSchema(mockMetaClient, Collections.emptyList()));
}
+ @Test
+ void convertSchema_blobField() {
+ HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
+ HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.INT)),
+ HoodieSchemaField.of("blob_data", HoodieSchema.createBlob())
+ ));
+
+ Field expectedBlobField = Field.newBuilder("blob_data", StandardSQLTypeName.STRUCT,
+ Field.newBuilder("type", StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(),
+ Field.newBuilder("data", StandardSQLTypeName.BYTES).setMode(Field.Mode.NULLABLE).build(),
+ Field.newBuilder("reference", StandardSQLTypeName.STRUCT,
+ Field.newBuilder("external_path", StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(),
+ Field.newBuilder("offset", StandardSQLTypeName.INT64).setMode(Field.Mode.NULLABLE).build(),
+ Field.newBuilder("length", StandardSQLTypeName.INT64).setMode(Field.Mode.NULLABLE).build(),
+ Field.newBuilder("managed", StandardSQLTypeName.BOOL).setMode(Field.Mode.REQUIRED).build())
+ .setMode(Field.Mode.NULLABLE).build())
+ .setMode(Field.Mode.REQUIRED).build();
+
+ Schema result = SCHEMA_RESOLVER.convertSchema(input);
+ Assertions.assertEquals(2, result.getFields().size());
+ Assertions.assertEquals(expectedBlobField, result.getFields().get(1));
+ }
+
+ @Test
+ void convertSchema_nullableBlobField() {
+ HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
+ HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.INT)),
+ HoodieSchemaField.of("blob_data", HoodieSchema.createNullable(HoodieSchema.createBlob()))
+ ));
+
+ Schema result = SCHEMA_RESOLVER.convertSchema(input);
+ Field blobField = result.getFields().get(1);
+ Assertions.assertEquals(Field.Mode.NULLABLE, blobField.getMode());
+ Assertions.assertEquals(StandardSQLTypeName.STRUCT, blobField.getType().getStandardType());
+ }
+
+ @Test
+ void convertSchema_nestedBlobField() {
+ HoodieSchema inner = HoodieSchema.createRecord("media", null, null, false, Arrays.asList(
+ HoodieSchemaField.of("title", HoodieSchema.create(HoodieSchemaType.STRING)),
+ HoodieSchemaField.of("content", HoodieSchema.createBlob())
+ ));
+ HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
+ HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.INT)),
+ HoodieSchemaField.of("media", inner)
+ ));
+
+ Schema result = SCHEMA_RESOLVER.convertSchema(input);
+ Field mediaField = result.getFields().get(1);
+ Assertions.assertEquals(StandardSQLTypeName.STRUCT, mediaField.getType().getStandardType());
+ // Verify the nested "content" sub-field is a STRUCT (BLOB)
+ Field contentField = mediaField.getSubFields().stream()
+ .filter(f -> f.getName().equals("content")).findFirst().get();
+ Assertions.assertEquals(StandardSQLTypeName.STRUCT, contentField.getType().getStandardType());
+ }
+
@Test
void convertSchema_vectorField() {
HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
index 64933d962d22..9ea507f64d14 100644
--- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
+++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
@@ -249,6 +249,7 @@ public class HiveSchemaUtil {
HoodieSchema valueType = nonNullType.getValueType();
return createHiveMap(convertField(keyType, supportTimestamp, doFormat), convertField(valueType, supportTimestamp, doFormat), doFormat);
case RECORD:
+ case BLOB:
return createHiveStruct(nonNullType.getFields(), supportTimestamp, doFormat);
default:
throw new UnsupportedOperationException("Cannot convert type " + nonNullType.getType());
diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java
index 53698fc056be..1a0e6cecf86a 100644
--- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java
+++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java
@@ -149,7 +149,13 @@ public class TestHiveSchemaUtil {
HoodieSchemaField.of("time_micros_field", HoodieSchema.createTimeMicros()),
HoodieSchemaField.of("decimal_field", HoodieSchema.createDecimal(10, 2)),
HoodieSchemaField.of("uuid_field", HoodieSchema.create(HoodieSchemaType.UUID)),
- HoodieSchemaField.of("vector_field", HoodieSchema.createVector(128))
+ HoodieSchemaField.of("vector_field", HoodieSchema.createVector(128)),
+ HoodieSchemaField.of("blob_field", HoodieSchema.createBlob()),
+ HoodieSchemaField.of("nullable_blob_field", HoodieSchema.createNullable(HoodieSchema.createBlob())),
+ HoodieSchemaField.of("nested_blob_field", HoodieSchema.createRecord("media", null, null, false, Arrays.asList(
+ HoodieSchemaField.of("title", HoodieSchema.create(HoodieSchemaType.STRING)),
+ HoodieSchemaField.of("content", HoodieSchema.createBlob())
+ )))
)
);
@@ -174,6 +180,12 @@ public class TestHiveSchemaUtil {
expected.put("`decimal_field`", "DECIMAL(10 , 2)");
expected.put("`uuid_field`", "binary");
expected.put("`vector_field`", "binary");
+ expected.put("`blob_field`",
+ "STRUCT< `type` : string, `data` : binary, `reference` : STRUCT< `external_path` : string, `offset` : bigint, `length` : bigint, `managed` : boolean>>");
+ expected.put("`nullable_blob_field`",
+ "STRUCT< `type` : string, `data` : binary, `reference` : STRUCT< `external_path` : string, `offset` : bigint, `length` : bigint, `managed` : boolean>>");
+ expected.put("`nested_blob_field`",
+ "STRUCT< `title` : string, `content` : STRUCT< `type` : string, `data` : binary, `reference` : STRUCT< `external_path` : string, `offset` : bigint, `length` : bigint, `managed` : boolean>>>");
assertEquals(expected, actual);
}
}