You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/04/15 09:02:12 UTC

(hudi) branch master updated: feat(sync): Map BLOB type to struct in Hive and BigQuery sync (#18482)

This is an automated email from the ASF dual-hosted git repository.

vhs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new da2667c4c6e6 feat(sync): Map BLOB type to struct in Hive and BigQuery sync (#18482)
da2667c4c6e6 is described below

commit da2667c4c6e6b7cd34edf85c29fca271f14aa71d
Author: voonhous <vo...@gmail.com>
AuthorDate: Wed Apr 15 17:02:04 2026 +0800

    feat(sync): Map BLOB type to struct in Hive and BigQuery sync (#18482)
    
    * feat(sync): Map BLOB type to struct in Hive and BigQuery sync
    
    - BLOB columns were not handled in HiveSchemaUtil and BigQuerySchemaResolver, causing UnsupportedOperationException when syncing tables with blob columns.
    - Map BLOB to its underlying physical type (struct) so external engines can read via Hive metastore and BigQuery.
    
    * Add test for nullable and nested blobs
---
 .../hudi/gcp/bigquery/BigQuerySchemaResolver.java  |  1 +
 .../gcp/bigquery/TestBigQuerySchemaResolver.java   | 56 ++++++++++++++++++++++
 .../org/apache/hudi/hive/util/HiveSchemaUtil.java  |  1 +
 .../apache/hudi/hive/util/TestHiveSchemaUtil.java  | 14 +++++-
 4 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
index e32921d7052c..d9d85c1743af 100644
--- a/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
+++ b/hudi-gcp/src/main/java/org/apache/hudi/gcp/bigquery/BigQuerySchemaResolver.java
@@ -147,6 +147,7 @@ public class BigQuerySchemaResolver {
         standardSQLTypeName = StandardSQLTypeName.NUMERIC;
         break;
       case RECORD:
+      case BLOB:
         return Field.newBuilder(name, StandardSQLTypeName.STRUCT,
             FieldList.of(getFields(fieldSchema))).setMode(fieldMode).build();
       case ARRAY:
diff --git a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
index 10ddfeef5632..f0d7cdc2a65d 100644
--- a/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
+++ b/hudi-gcp/src/test/java/org/apache/hudi/gcp/bigquery/TestBigQuerySchemaResolver.java
@@ -281,6 +281,62 @@ public class TestBigQuerySchemaResolver {
     Assertions.assertEquals(PRIMITIVE_TYPES_BQ_SCHEMA, resolver.getTableSchema(mockMetaClient, Collections.emptyList()));
   }
 
+  @Test
+  void convertSchema_blobField() {
+    HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
+        HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.INT)),
+        HoodieSchemaField.of("blob_data", HoodieSchema.createBlob())
+    ));
+
+    Field expectedBlobField = Field.newBuilder("blob_data", StandardSQLTypeName.STRUCT,
+        Field.newBuilder("type", StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(),
+        Field.newBuilder("data", StandardSQLTypeName.BYTES).setMode(Field.Mode.NULLABLE).build(),
+        Field.newBuilder("reference", StandardSQLTypeName.STRUCT,
+            Field.newBuilder("external_path", StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build(),
+            Field.newBuilder("offset", StandardSQLTypeName.INT64).setMode(Field.Mode.NULLABLE).build(),
+            Field.newBuilder("length", StandardSQLTypeName.INT64).setMode(Field.Mode.NULLABLE).build(),
+            Field.newBuilder("managed", StandardSQLTypeName.BOOL).setMode(Field.Mode.REQUIRED).build())
+            .setMode(Field.Mode.NULLABLE).build())
+        .setMode(Field.Mode.REQUIRED).build();
+
+    Schema result = SCHEMA_RESOLVER.convertSchema(input);
+    Assertions.assertEquals(2, result.getFields().size());
+    Assertions.assertEquals(expectedBlobField, result.getFields().get(1));
+  }
+
+  @Test
+  void convertSchema_nullableBlobField() {
+    HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
+        HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.INT)),
+        HoodieSchemaField.of("blob_data", HoodieSchema.createNullable(HoodieSchema.createBlob()))
+    ));
+
+    Schema result = SCHEMA_RESOLVER.convertSchema(input);
+    Field blobField = result.getFields().get(1);
+    Assertions.assertEquals(Field.Mode.NULLABLE, blobField.getMode());
+    Assertions.assertEquals(StandardSQLTypeName.STRUCT, blobField.getType().getStandardType());
+  }
+
+  @Test
+  void convertSchema_nestedBlobField() {
+    HoodieSchema inner = HoodieSchema.createRecord("media", null, null, false, Arrays.asList(
+        HoodieSchemaField.of("title", HoodieSchema.create(HoodieSchemaType.STRING)),
+        HoodieSchemaField.of("content", HoodieSchema.createBlob())
+    ));
+    HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
+        HoodieSchemaField.of("id", HoodieSchema.create(HoodieSchemaType.INT)),
+        HoodieSchemaField.of("media", inner)
+    ));
+
+    Schema result = SCHEMA_RESOLVER.convertSchema(input);
+    Field mediaField = result.getFields().get(1);
+    Assertions.assertEquals(StandardSQLTypeName.STRUCT, mediaField.getType().getStandardType());
+    // Verify the nested "content" sub-field is a STRUCT (BLOB)
+    Field contentField = mediaField.getSubFields().stream()
+        .filter(f -> f.getName().equals("content")).findFirst().get();
+    Assertions.assertEquals(StandardSQLTypeName.STRUCT, contentField.getType().getStandardType());
+  }
+
   @Test
   void convertSchema_vectorField() {
     HoodieSchema input = HoodieSchema.createRecord("testRecord", null, null, false, Arrays.asList(
diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
index 64933d962d22..9ea507f64d14 100644
--- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
+++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
@@ -249,6 +249,7 @@ public class HiveSchemaUtil {
         HoodieSchema valueType = nonNullType.getValueType();
         return createHiveMap(convertField(keyType, supportTimestamp, doFormat), convertField(valueType, supportTimestamp, doFormat), doFormat);
       case RECORD:
+      case BLOB:
         return createHiveStruct(nonNullType.getFields(), supportTimestamp, doFormat);
       default:
         throw new UnsupportedOperationException("Cannot convert type " + nonNullType.getType());
diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java
index 53698fc056be..1a0e6cecf86a 100644
--- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java
+++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestHiveSchemaUtil.java
@@ -149,7 +149,13 @@ public class TestHiveSchemaUtil {
             HoodieSchemaField.of("time_micros_field", HoodieSchema.createTimeMicros()),
             HoodieSchemaField.of("decimal_field", HoodieSchema.createDecimal(10, 2)),
             HoodieSchemaField.of("uuid_field", HoodieSchema.create(HoodieSchemaType.UUID)),
-            HoodieSchemaField.of("vector_field", HoodieSchema.createVector(128))
+            HoodieSchemaField.of("vector_field", HoodieSchema.createVector(128)),
+            HoodieSchemaField.of("blob_field", HoodieSchema.createBlob()),
+            HoodieSchemaField.of("nullable_blob_field", HoodieSchema.createNullable(HoodieSchema.createBlob())),
+            HoodieSchemaField.of("nested_blob_field", HoodieSchema.createRecord("media", null, null, false, Arrays.asList(
+                HoodieSchemaField.of("title", HoodieSchema.create(HoodieSchemaType.STRING)),
+                HoodieSchemaField.of("content", HoodieSchema.createBlob())
+            )))
         )
     );
 
@@ -174,6 +180,12 @@ public class TestHiveSchemaUtil {
     expected.put("`decimal_field`", "DECIMAL(10 , 2)");
     expected.put("`uuid_field`", "binary");
     expected.put("`vector_field`", "binary");
+    expected.put("`blob_field`",
+        "STRUCT< `type` : string, `data` : binary, `reference` : STRUCT< `external_path` : string, `offset` : bigint, `length` : bigint, `managed` : boolean>>");
+    expected.put("`nullable_blob_field`",
+        "STRUCT< `type` : string, `data` : binary, `reference` : STRUCT< `external_path` : string, `offset` : bigint, `length` : bigint, `managed` : boolean>>");
+    expected.put("`nested_blob_field`",
+        "STRUCT< `title` : string, `content` : STRUCT< `type` : string, `data` : binary, `reference` : STRUCT< `external_path` : string, `offset` : bigint, `length` : bigint, `managed` : boolean>>>");
     assertEquals(expected, actual);
   }
 }