You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vb...@apache.org on 2026/04/21 02:41:55 UTC
(hudi) branch master updated: feat(vector): Add Spark SQL DDL CREATE TABLE support for VECTOR type (#18488)
This is an automated email from the ASF dual-hosted git repository.
bvaradar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a83473689eae feat(vector): Add Spark SQL DDL CREATE TABLE support for VECTOR type (#18488)
a83473689eae is described below
commit a83473689eaee9b63211bd07bcbfb87b50304c08
Author: voonhous <vo...@gmail.com>
AuthorDate: Tue Apr 21 10:41:49 2026 +0800
feat(vector): Add Spark SQL DDL CREATE TABLE support for VECTOR type (#18488)
* feat(vector): Add Spark SQL DDL CREATE TABLE support for VECTOR type
- Enable VECTOR(dim[, elementType]) syntax in Spark SQL DDL so users can create tables with vector columns directly via SQL instead of only through the DataFrame API.
- Changes:
- Extend ANTLR grammar to accept identifier params in primitiveDataType
- Add VECTOR case in visitPrimitiveDataType (supports FLOAT, DOUBLE, INT8)
- Add VECTOR metadata attachment in addMetadataForType
- Add DDL tests for VECTOR columns in TestCreateTable
* Fix tests and address comments
* Improve VECTOR DDL test coverage with targeted tests and routing
- Relax isHoodieCommand VECTOR check from " vector(" to " vector" in all 4 extended parser files.
- The stricter " vector(" variant only routes SQL containing VECTOR type declarations with parentheses (e.g. VECTOR(128)), which means VECTOR without parens is delegated to Spark's native parser and never reaches our Hudi code path.
- Relaxing to " vector" routes all VECTOR-related SQL through our parser, enabling us to exercise the "vector with empty params" branch of the `case ("vector", _ :: _)` pattern - previously reported as partial coverage because the empty-list side of the `_ :: _` check was never hit.
- This is also consistent with the existing BLOB routing pattern " blob".
Add two targeted tests:
1. test create table with INT8 VECTOR column - isolated INT8 test that independently exercises the `case INT8 => ByteType` branch
2. test create table with VECTOR without dimension fails - routes VECTOR alone through the Hudi parser to cover the empty-list branch
---
.../spark/sql/hudi/ddl/TestCreateTable.scala | 167 +++++++++++++++++++++
.../src/main/antlr4/imports/SqlBase.g4 | 2 +-
.../HoodieSpark3_3ExtendedSqlAstBuilder.scala | 24 ++-
.../parser/HoodieSpark3_3ExtendedSqlParser.scala | 3 +-
.../src/main/antlr4/imports/SqlBase.g4 | 2 +-
.../HoodieSpark3_4ExtendedSqlAstBuilder.scala | 24 ++-
.../parser/HoodieSpark3_4ExtendedSqlParser.scala | 3 +-
.../src/main/antlr4/imports/SqlBase.g4 | 2 +-
.../HoodieSpark3_5ExtendedSqlAstBuilder.scala | 24 ++-
.../parser/HoodieSpark3_5ExtendedSqlParser.scala | 3 +-
.../src/main/antlr4/imports/SqlBase.g4 | 2 +-
.../HoodieSpark4_0ExtendedSqlAstBuilder.scala | 24 ++-
.../parser/HoodieSpark4_0ExtendedSqlParser.scala | 3 +-
13 files changed, 267 insertions(+), 16 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
index 5cd5dbb0d54b..1200ec34f3ab 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
@@ -2120,4 +2120,171 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
assertTrue(blobPathField.dataType.isInstanceOf[StringType])
}
}
+
+ test("test create table with VECTOR column") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |CREATE TABLE $tableName (
+ | id BIGINT,
+ | embedding VECTOR(128) COMMENT 'document embedding'
+ |) USING hudi
+ |LOCATION '${tmp.getCanonicalPath}'
+ |TBLPROPERTIES (
+ | primaryKey = 'id'
+ |)
+ """.stripMargin)
+
+ val schema = spark.table(tableName).schema
+ val embeddingField = schema.find(_.name == "embedding").get
+ assertTrue(embeddingField.metadata.contains(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals("VECTOR(128)", embeddingField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals("document embedding", embeddingField.metadata.getString("comment"))
+ assertEquals(ArrayType(FloatType, containsNull = false), embeddingField.dataType)
+ }
+ }
+
+ test("test create table with VECTOR column with element type") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |CREATE TABLE $tableName (
+ | id BIGINT,
+ | embedding VECTOR(64, DOUBLE)
+ |) USING hudi
+ |LOCATION '${tmp.getCanonicalPath}'
+ |TBLPROPERTIES (
+ | primaryKey = 'id'
+ |)
+ """.stripMargin)
+
+ val schema = spark.table(tableName).schema
+ val embeddingField = schema.find(_.name == "embedding").get
+ assertTrue(embeddingField.metadata.contains(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals("VECTOR(64, DOUBLE)", embeddingField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals(ArrayType(DoubleType, containsNull = false), embeddingField.dataType)
+ }
+ }
+
+ test("test create table with multiple VECTOR columns") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |CREATE TABLE $tableName (
+ | id BIGINT,
+ | float_vec VECTOR(128),
+ | float_vec_explicit VECTOR(128, FLOAT),
+ | double_vec VECTOR(64, DOUBLE),
+ | int8_vec VECTOR(256, INT8) NOT NULL
+ |) USING hudi
+ |LOCATION '${tmp.getCanonicalPath}'
+ |TBLPROPERTIES (
+ | primaryKey = 'id'
+ |)
+ """.stripMargin)
+
+ val schema = spark.table(tableName).schema
+
+ val floatVecField = schema.find(_.name == "float_vec").get
+ assertEquals("VECTOR(128)", floatVecField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals(ArrayType(FloatType, containsNull = false), floatVecField.dataType)
+ assertTrue(floatVecField.nullable)
+
+ // VECTOR(128, FLOAT) should be normalized to the canonical form "VECTOR(128)"
+ val floatVecExplicitField = schema.find(_.name == "float_vec_explicit").get
+ assertEquals("VECTOR(128)", floatVecExplicitField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals(ArrayType(FloatType, containsNull = false), floatVecExplicitField.dataType)
+
+ val doubleVecField = schema.find(_.name == "double_vec").get
+ assertEquals("VECTOR(64, DOUBLE)", doubleVecField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals(ArrayType(DoubleType, containsNull = false), doubleVecField.dataType)
+
+ val int8VecField = schema.find(_.name == "int8_vec").get
+ assertEquals("VECTOR(256, INT8)", int8VecField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals(ArrayType(ByteType, containsNull = false), int8VecField.dataType)
+ assertFalse(int8VecField.nullable)
+ }
+ }
+
+ test("test create table with INT8 VECTOR column") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |CREATE TABLE $tableName (
+ | id BIGINT,
+ | embedding VECTOR(256, INT8)
+ |) USING hudi
+ |LOCATION '${tmp.getCanonicalPath}'
+ |TBLPROPERTIES (
+ | primaryKey = 'id'
+ |)
+ """.stripMargin)
+
+ val schema = spark.table(tableName).schema
+ val embeddingField = schema.find(_.name == "embedding").get
+ assertEquals("VECTOR(256, INT8)", embeddingField.metadata.getString(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals(ArrayType(ByteType, containsNull = false), embeddingField.dataType)
+ }
+ }
+
+ test("test create table with VECTOR without dimension fails") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ checkExceptionContain(
+ s"""
+ |CREATE TABLE $tableName (
+ | id BIGINT,
+ | v VECTOR
+ |) USING hudi
+ |LOCATION '${tmp.getCanonicalPath}'
+ |TBLPROPERTIES (
+ | primaryKey = 'id'
+ |)
+ """.stripMargin)("vector is not supported")
+ }
+ }
+
+ test("test create table with invalid VECTOR type surfaces ParseException") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ // Unsupported element type
+ checkExceptionContain(
+ s"""
+ |CREATE TABLE $tableName (
+ | id BIGINT,
+ | embedding VECTOR(128, BOOLEAN)
+ |) USING hudi
+ |LOCATION '${tmp.getCanonicalPath}'
+ |TBLPROPERTIES (
+ | primaryKey = 'id'
+ |)
+ """.stripMargin)("Invalid VECTOR type")
+ }
+ }
+
+ test("test create table with VECTOR column case insensitive") {
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ |CREATE TABLE $tableName (
+ | id BIGINT,
+ | embedding vector(128)
+ |) USING hudi
+ |LOCATION '${tmp.getCanonicalPath}'
+ |TBLPROPERTIES (
+ | primaryKey = 'id'
+ |)
+ """.stripMargin)
+
+ val schema = spark.table(tableName).schema
+ val embeddingField = schema.find(_.name == "embedding").get
+ assertTrue(embeddingField.metadata.contains(HoodieSchema.TYPE_METADATA_FIELD))
+ assertEquals(ArrayType(FloatType, containsNull = false), embeddingField.dataType)
+ }
+ }
}
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/antlr4/imports/SqlBase.g4 b/hudi-spark-datasource/hudi-spark3.3.x/src/main/antlr4/imports/SqlBase.g4
index d7f87b4e5aa5..a5e31dadb2c4 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/antlr4/imports/SqlBase.g4
+++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/antlr4/imports/SqlBase.g4
@@ -942,7 +942,7 @@ dataType
| INTERVAL from=(YEAR | MONTH) (TO to=MONTH)? #yearMonthIntervalDataType
| INTERVAL from=(DAY | HOUR | MINUTE | SECOND)
(TO to=(HOUR | MINUTE | SECOND))? #dayTimeIntervalDataType
- | identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')? #primitiveDataType
+ | typeName=identifier ('(' (INTEGER_VALUE | identifier) (',' (INTEGER_VALUE | identifier))* ')')? #primitiveDataType
;
qualifiedColTypeWithPositionList
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala
index ec357ccb7b1a..cec7a8378e1b 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala
+++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlAstBuilder.scala
@@ -2587,7 +2587,7 @@ class HoodieSpark3_3ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
* Resolve/create a primitive type.
*/
override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType = withOrigin(ctx) {
- val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT)
+ val dataType = ctx.typeName.getText.toLowerCase(Locale.ROOT)
(dataType, ctx.INTEGER_VALUE().asScala.toList) match {
case ("boolean", Nil) => BooleanType
case ("tinyint" | "byte", Nil) => ByteType
@@ -2606,6 +2606,21 @@ class HoodieSpark3_3ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
case ("varchar", length :: Nil) => VarcharType(length.getText.toInt)
case ("binary", Nil) => BinaryType
case ("blob", Nil) => BlobType()
+ case ("vector", _ :: _) =>
+ // Delegate validation to HoodieSchema.parseTypeDescriptor which handles dimension
+ // range checks, element type validation, and canonical normalization.
+ val vectorSchema = try {
+ HoodieSchema.parseTypeDescriptor(ctx.getText).asInstanceOf[HoodieSchema.Vector]
+ } catch {
+ case e: IllegalArgumentException =>
+ throw new ParseException(s"Invalid VECTOR type: ${e.getMessage}", ctx)
+ }
+ val sparkElemType = vectorSchema.getVectorElementType match {
+ case HoodieSchema.Vector.VectorElementType.FLOAT => FloatType
+ case HoodieSchema.Vector.VectorElementType.DOUBLE => DoubleType
+ case HoodieSchema.Vector.VectorElementType.INT8 => ByteType
+ }
+ ArrayType(sparkElemType, containsNull = false)
case ("decimal" | "dec" | "numeric", Nil) => DecimalType.USER_DEFAULT
case ("decimal" | "dec" | "numeric", precision :: Nil) =>
DecimalType(precision.getText.toInt, 0)
@@ -2702,8 +2717,13 @@ class HoodieSpark3_3ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
private def addMetadataForType(dataType: HoodieSqlBaseParser.DataTypeContext, builder: MetadataBuilder): Unit = {
val typeText = dataType.getText
- if (typeText.equalsIgnoreCase(HoodieSchemaType.BLOB.name())) {
+ val upperTypeText = typeText.toUpperCase(Locale.ROOT)
+ if (upperTypeText == HoodieSchemaType.BLOB.name()) {
builder.putString(HoodieSchema.TYPE_METADATA_FIELD, HoodieSchemaType.BLOB.name())
+ } else if (upperTypeText.startsWith("VECTOR(")) {
+ // Normalize to canonical form (e.g. "VECTOR(128,FLOAT)" -> "VECTOR(128)")
+ val vectorSchema = HoodieSchema.parseTypeDescriptor(typeText).asInstanceOf[HoodieSchema.Vector]
+ builder.putString(HoodieSchema.TYPE_METADATA_FIELD, vectorSchema.toTypeDescriptor)
}
}
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala
index e10df36d2faf..688847a78cab 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala
+++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_3ExtendedSqlParser.scala
@@ -129,7 +129,8 @@ class HoodieSpark3_3ExtendedSqlParser(session: SparkSession, delegate: ParserInt
normalized.contains("drop index") ||
normalized.contains("show indexes") ||
normalized.contains("refresh index") ||
- normalized.contains(" blob")
+ normalized.contains(" blob") ||
+ normalized.contains(" vector")
}
}
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/antlr4/imports/SqlBase.g4 b/hudi-spark-datasource/hudi-spark3.4.x/src/main/antlr4/imports/SqlBase.g4
index d7f87b4e5aa5..a5e31dadb2c4 100644
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/antlr4/imports/SqlBase.g4
+++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/antlr4/imports/SqlBase.g4
@@ -942,7 +942,7 @@ dataType
| INTERVAL from=(YEAR | MONTH) (TO to=MONTH)? #yearMonthIntervalDataType
| INTERVAL from=(DAY | HOUR | MINUTE | SECOND)
(TO to=(HOUR | MINUTE | SECOND))? #dayTimeIntervalDataType
- | identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')? #primitiveDataType
+ | typeName=identifier ('(' (INTEGER_VALUE | identifier) (',' (INTEGER_VALUE | identifier))* ')')? #primitiveDataType
;
qualifiedColTypeWithPositionList
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala
index fb995babe70d..ba2852952bb5 100644
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala
+++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlAstBuilder.scala
@@ -2590,7 +2590,7 @@ class HoodieSpark3_4ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
* Resolve/create a primitive type.
*/
override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType = withOrigin(ctx) {
- val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT)
+ val dataType = ctx.typeName.getText.toLowerCase(Locale.ROOT)
(dataType, ctx.INTEGER_VALUE().asScala.toList) match {
case ("boolean", Nil) => BooleanType
case ("tinyint" | "byte", Nil) => ByteType
@@ -2609,6 +2609,21 @@ class HoodieSpark3_4ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
case ("varchar", length :: Nil) => VarcharType(length.getText.toInt)
case ("binary", Nil) => BinaryType
case ("blob", Nil) => BlobType()
+ case ("vector", _ :: _) =>
+ // Delegate validation to HoodieSchema.parseTypeDescriptor which handles dimension
+ // range checks, element type validation, and canonical normalization.
+ val vectorSchema = try {
+ HoodieSchema.parseTypeDescriptor(ctx.getText).asInstanceOf[HoodieSchema.Vector]
+ } catch {
+ case e: IllegalArgumentException =>
+ throw new ParseException(s"Invalid VECTOR type: ${e.getMessage}", ctx)
+ }
+ val sparkElemType = vectorSchema.getVectorElementType match {
+ case HoodieSchema.Vector.VectorElementType.FLOAT => FloatType
+ case HoodieSchema.Vector.VectorElementType.DOUBLE => DoubleType
+ case HoodieSchema.Vector.VectorElementType.INT8 => ByteType
+ }
+ ArrayType(sparkElemType, containsNull = false)
case ("decimal" | "dec" | "numeric", Nil) => DecimalType.USER_DEFAULT
case ("decimal" | "dec" | "numeric", precision :: Nil) =>
DecimalType(precision.getText.toInt, 0)
@@ -2705,8 +2720,13 @@ class HoodieSpark3_4ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
private def addMetadataForType(dataType: HoodieSqlBaseParser.DataTypeContext, builder: MetadataBuilder): Unit = {
val typeText = dataType.getText
- if (typeText.equalsIgnoreCase(HoodieSchemaType.BLOB.name())) {
+ val upperTypeText = typeText.toUpperCase(Locale.ROOT)
+ if (upperTypeText == HoodieSchemaType.BLOB.name()) {
builder.putString(HoodieSchema.TYPE_METADATA_FIELD, HoodieSchemaType.BLOB.name())
+ } else if (upperTypeText.startsWith("VECTOR(")) {
+ // Normalize to canonical form (e.g. "VECTOR(128,FLOAT)" -> "VECTOR(128)")
+ val vectorSchema = HoodieSchema.parseTypeDescriptor(typeText).asInstanceOf[HoodieSchema.Vector]
+ builder.putString(HoodieSchema.TYPE_METADATA_FIELD, vectorSchema.toTypeDescriptor)
}
}
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlParser.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlParser.scala
index 267e582bf2a1..687dc2b01b14 100644
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlParser.scala
+++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_4ExtendedSqlParser.scala
@@ -129,7 +129,8 @@ class HoodieSpark3_4ExtendedSqlParser(session: SparkSession, delegate: ParserInt
normalized.contains("drop index") ||
normalized.contains("show indexes") ||
normalized.contains("refresh index") ||
- normalized.contains(" blob")
+ normalized.contains(" blob") ||
+ normalized.contains(" vector")
}
}
diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/antlr4/imports/SqlBase.g4 b/hudi-spark-datasource/hudi-spark3.5.x/src/main/antlr4/imports/SqlBase.g4
index d7f87b4e5aa5..a5e31dadb2c4 100644
--- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/antlr4/imports/SqlBase.g4
+++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/antlr4/imports/SqlBase.g4
@@ -942,7 +942,7 @@ dataType
| INTERVAL from=(YEAR | MONTH) (TO to=MONTH)? #yearMonthIntervalDataType
| INTERVAL from=(DAY | HOUR | MINUTE | SECOND)
(TO to=(HOUR | MINUTE | SECOND))? #dayTimeIntervalDataType
- | identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')? #primitiveDataType
+ | typeName=identifier ('(' (INTEGER_VALUE | identifier) (',' (INTEGER_VALUE | identifier))* ')')? #primitiveDataType
;
qualifiedColTypeWithPositionList
diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlAstBuilder.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlAstBuilder.scala
index 653a19984490..cafccd0ea419 100644
--- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlAstBuilder.scala
+++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlAstBuilder.scala
@@ -2591,7 +2591,7 @@ class HoodieSpark3_5ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
* Resolve/create a primitive type.
*/
override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType = withOrigin(ctx) {
- val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT)
+ val dataType = ctx.typeName.getText.toLowerCase(Locale.ROOT)
(dataType, ctx.INTEGER_VALUE().asScala.toList) match {
case ("boolean", Nil) => BooleanType
case ("tinyint" | "byte", Nil) => ByteType
@@ -2610,6 +2610,21 @@ class HoodieSpark3_5ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
case ("varchar", length :: Nil) => VarcharType(length.getText.toInt)
case ("binary", Nil) => BinaryType
case ("blob", Nil) => BlobType()
+ case ("vector", _ :: _) =>
+ // Delegate validation to HoodieSchema.parseTypeDescriptor which handles dimension
+ // range checks, element type validation, and canonical normalization.
+ val vectorSchema = try {
+ HoodieSchema.parseTypeDescriptor(ctx.getText).asInstanceOf[HoodieSchema.Vector]
+ } catch {
+ case e: IllegalArgumentException =>
+ throw new ParseException(s"Invalid VECTOR type: ${e.getMessage}", ctx)
+ }
+ val sparkElemType = vectorSchema.getVectorElementType match {
+ case HoodieSchema.Vector.VectorElementType.FLOAT => FloatType
+ case HoodieSchema.Vector.VectorElementType.DOUBLE => DoubleType
+ case HoodieSchema.Vector.VectorElementType.INT8 => ByteType
+ }
+ ArrayType(sparkElemType, containsNull = false)
case ("decimal" | "dec" | "numeric", Nil) => DecimalType.USER_DEFAULT
case ("decimal" | "dec" | "numeric", precision :: Nil) =>
DecimalType(precision.getText.toInt, 0)
@@ -2706,8 +2721,13 @@ class HoodieSpark3_5ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
private def addMetadataForType(dataType: HoodieSqlBaseParser.DataTypeContext, builder: MetadataBuilder): Unit = {
val typeText = dataType.getText
- if (typeText.equalsIgnoreCase(HoodieSchemaType.BLOB.name())) {
+ val upperTypeText = typeText.toUpperCase(Locale.ROOT)
+ if (upperTypeText == HoodieSchemaType.BLOB.name()) {
builder.putString(HoodieSchema.TYPE_METADATA_FIELD, HoodieSchemaType.BLOB.name())
+ } else if (upperTypeText.startsWith("VECTOR(")) {
+ // Normalize to canonical form (e.g. "VECTOR(128,FLOAT)" -> "VECTOR(128)")
+ val vectorSchema = HoodieSchema.parseTypeDescriptor(typeText).asInstanceOf[HoodieSchema.Vector]
+ builder.putString(HoodieSchema.TYPE_METADATA_FIELD, vectorSchema.toTypeDescriptor)
}
}
diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlParser.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlParser.scala
index 3c3bc06736cb..ef31536985e5 100644
--- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlParser.scala
+++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_5ExtendedSqlParser.scala
@@ -129,7 +129,8 @@ class HoodieSpark3_5ExtendedSqlParser(session: SparkSession, delegate: ParserInt
normalized.contains("drop index") ||
normalized.contains("show indexes") ||
normalized.contains("refresh index") ||
- normalized.contains(" blob")
+ normalized.contains(" blob") ||
+ normalized.contains(" vector")
}
}
diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/antlr4/imports/SqlBase.g4 b/hudi-spark-datasource/hudi-spark4.0.x/src/main/antlr4/imports/SqlBase.g4
index d7f87b4e5aa5..a5e31dadb2c4 100644
--- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/antlr4/imports/SqlBase.g4
+++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/antlr4/imports/SqlBase.g4
@@ -942,7 +942,7 @@ dataType
| INTERVAL from=(YEAR | MONTH) (TO to=MONTH)? #yearMonthIntervalDataType
| INTERVAL from=(DAY | HOUR | MINUTE | SECOND)
(TO to=(HOUR | MINUTE | SECOND))? #dayTimeIntervalDataType
- | identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')? #primitiveDataType
+ | typeName=identifier ('(' (INTEGER_VALUE | identifier) (',' (INTEGER_VALUE | identifier))* ')')? #primitiveDataType
;
qualifiedColTypeWithPositionList
diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlAstBuilder.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlAstBuilder.scala
index d8d3faa0dfca..20a9ae4af0b6 100644
--- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlAstBuilder.scala
+++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlAstBuilder.scala
@@ -2593,7 +2593,7 @@ class HoodieSpark4_0ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
* Resolve/create a primitive type.
*/
override def visitPrimitiveDataType(ctx: PrimitiveDataTypeContext): DataType = withOrigin(ctx) {
- val dataType = ctx.identifier.getText.toLowerCase(Locale.ROOT)
+ val dataType = ctx.typeName.getText.toLowerCase(Locale.ROOT)
(dataType, ctx.INTEGER_VALUE().asScala.toList) match {
case ("boolean", Nil) => BooleanType
case ("tinyint" | "byte", Nil) => ByteType
@@ -2612,6 +2612,21 @@ class HoodieSpark4_0ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
case ("varchar", length :: Nil) => VarcharType(length.getText.toInt)
case ("binary", Nil) => BinaryType
case ("blob", Nil) => BlobType()
+ case ("vector", _ :: _) =>
+ // Delegate validation to HoodieSchema.parseTypeDescriptor which handles dimension
+ // range checks, element type validation, and canonical normalization.
+ val vectorSchema = try {
+ HoodieSchema.parseTypeDescriptor(ctx.getText).asInstanceOf[HoodieSchema.Vector]
+ } catch {
+ case e: IllegalArgumentException =>
+ throw new ParseException(s"Invalid VECTOR type: ${e.getMessage}", ctx)
+ }
+ val sparkElemType = vectorSchema.getVectorElementType match {
+ case HoodieSchema.Vector.VectorElementType.FLOAT => FloatType
+ case HoodieSchema.Vector.VectorElementType.DOUBLE => DoubleType
+ case HoodieSchema.Vector.VectorElementType.INT8 => ByteType
+ }
+ ArrayType(sparkElemType, containsNull = false)
case ("decimal" | "dec" | "numeric", Nil) => DecimalType.USER_DEFAULT
case ("decimal" | "dec" | "numeric", precision :: Nil) =>
DecimalType(precision.getText.toInt, 0)
@@ -2708,8 +2723,13 @@ class HoodieSpark4_0ExtendedSqlAstBuilder(conf: SQLConf, delegate: ParserInterfa
private def addMetadataForType(dataType: HoodieSqlBaseParser.DataTypeContext, builder: MetadataBuilder): Unit = {
val typeText = dataType.getText
- if (typeText.equalsIgnoreCase(HoodieSchemaType.BLOB.name())) {
+ val upperTypeText = typeText.toUpperCase(Locale.ROOT)
+ if (upperTypeText == HoodieSchemaType.BLOB.name()) {
builder.putString(HoodieSchema.TYPE_METADATA_FIELD, HoodieSchemaType.BLOB.name())
+ } else if (upperTypeText.startsWith("VECTOR(")) {
+ // Normalize to canonical form (e.g. "VECTOR(128,FLOAT)" -> "VECTOR(128)")
+ val vectorSchema = HoodieSchema.parseTypeDescriptor(typeText).asInstanceOf[HoodieSchema.Vector]
+ builder.putString(HoodieSchema.TYPE_METADATA_FIELD, vectorSchema.toTypeDescriptor)
}
}
diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlParser.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlParser.scala
index f44002ab0265..ccddd6e6d3f1 100644
--- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlParser.scala
+++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/parser/HoodieSpark4_0ExtendedSqlParser.scala
@@ -138,7 +138,8 @@ class HoodieSpark4_0ExtendedSqlParser(session: SparkSession, delegate: ParserInt
normalized.contains("drop index") ||
normalized.contains("show indexes") ||
normalized.contains("refresh index") ||
- normalized.contains(" blob")
+ normalized.contains(" blob") ||
+ normalized.contains(" vector")
}
override def parseRoutineParam(sqlText: String): StructType = throw new UnsupportedOperationException()