You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2026/08/23 04:42:19 UTC
(shardingsphere) branch master updated: Support parsing Doris VARCHAR without field length and add Doris function parser IT cases (#39573)
This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new be8bf02e51c Support parsing Doris VARCHAR without field length and add Doris function parser IT cases (#39573)
be8bf02e51c is described below
commit be8bf02e51cde14923461685bbd217a38d31807c
Author: Liang Zhang <zh...@apache.org>
AuthorDate: Sun Aug 23 12:42:12 2026 +0800
Support parsing Doris VARCHAR without field length and add Doris function parser IT cases (#39573)
Fixes #31484
---
.../src/main/antlr4/imports/doris/BaseRule.g4 | 4 +-
.../src/main/resources/case/ddl/create-table.xml | 16 ++
.../resources/case/dml/select-special-function.xml | 207 +++++++++++++++++++++
.../resources/sql/supported/ddl/create-table.xml | 1 +
.../sql/supported/dml/select-special-function.xml | 6 +
5 files changed, 233 insertions(+), 1 deletion(-)
diff --git a/parser/sql/engine/dialect/doris/src/main/antlr4/imports/doris/BaseRule.g4 b/parser/sql/engine/dialect/doris/src/main/antlr4/imports/doris/BaseRule.g4
index 894e6643120..ee059c847b8 100644
--- a/parser/sql/engine/dialect/doris/src/main/antlr4/imports/doris/BaseRule.g4
+++ b/parser/sql/engine/dialect/doris/src/main/antlr4/imports/doris/BaseRule.g4
@@ -1425,7 +1425,9 @@ dataType
| (dataTypeName = NCHAR | dataTypeName = NATIONAL_CHAR) fieldLength? BINARY?
| dataTypeName = (SIGNED | SIGNED_INT | SIGNED_INTEGER)
| dataTypeName = BINARY fieldLength?
- | (dataTypeName = CHAR_VARYING | dataTypeName = CHARACTER_VARYING | dataTypeName = VARCHAR) fieldLength charsetWithOptBinary?
+ // DORIS CHANGED BEGIN
+ | (dataTypeName = CHAR_VARYING | dataTypeName = CHARACTER_VARYING | dataTypeName = VARCHAR) fieldLength? charsetWithOptBinary?
+ // DORIS CHANGED END
| (dataTypeName = NATIONAL VARCHAR | dataTypeName = NVARCHAR | dataTypeName = NCHAR VARCHAR | dataTypeName = NATIONAL_CHAR_VARYING | dataTypeName = NCHAR VARYING) fieldLength BINARY?
| dataTypeName = VARBINARY fieldLength?
| dataTypeName = YEAR fieldLength? fieldOptions?
diff --git a/test/it/parser/src/main/resources/case/ddl/create-table.xml b/test/it/parser/src/main/resources/case/ddl/create-table.xml
index b47579d1775..e05f0646808 100644
--- a/test/it/parser/src/main/resources/case/ddl/create-table.xml
+++ b/test/it/parser/src/main/resources/case/ddl/create-table.xml
@@ -2656,6 +2656,22 @@
</column-definition>
</create-table>
+ <create-table sql-case-id="create_table_with_varchar_without_length_doris" db-types="Doris">
+ <table name="tags" start-index="13" stop-index="16" />
+ <column-definition type="VARCHAR" start-index="19" stop-index="29">
+ <column name="tag" start-index="19" stop-index="21" />
+ </column-definition>
+ <column-definition type="DATETIME" start-index="32" stop-index="44">
+ <column name="date" start-index="32" stop-index="35" />
+ </column-definition>
+ <column-definition type="BITMAP" agg-type="BITMAP_UNION" start-index="47" stop-index="73">
+ <column name="user_id" start-index="47" stop-index="53" />
+ </column-definition>
+ <create-table-option start-index="76" stop-index="86">
+ <engine name="OLAP" start-index="83" stop-index="86" />
+ </create-table-option>
+ </create-table>
+
<create-table sql-case-id="create_table_inverted_index_doris" db-types="Doris">
<table name="table_hash" start-index="13" stop-index="33">
<owner name="example_db" start-index="13" stop-index="22" />
diff --git a/test/it/parser/src/main/resources/case/dml/select-special-function.xml b/test/it/parser/src/main/resources/case/dml/select-special-function.xml
index 69962aced24..5ae3d45321d 100644
--- a/test/it/parser/src/main/resources/case/dml/select-special-function.xml
+++ b/test/it/parser/src/main/resources/case/dml/select-special-function.xml
@@ -6436,4 +6436,211 @@
<simple-table name="t_order" start-index="37" stop-index="43" />
</from>
</select>
+
+ <select sql-case-id="select_ucase" db-types="MySQL,Doris">
+ <projections start-index="7" stop-index="20">
+ <expression-projection start-index="7" stop-index="20" text="UCASE('hello')">
+ <expr>
+ <function function-name="UCASE" text="UCASE('hello')" start-index="7" stop-index="20">
+ <parameter>
+ <literal-expression value="hello" start-index="13" stop-index="19" />
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ </select>
+
+ <select sql-case-id="select_starts_with_doris" db-types="Doris">
+ <projections start-index="7" stop-index="41">
+ <expression-projection start-index="7" stop-index="41" text="STARTS_WITH("hello world", "hello")">
+ <expr>
+ <function function-name="STARTS_WITH" text="STARTS_WITH("hello world", "hello")" start-index="7" stop-index="41">
+ <parameter>
+ <literal-expression value="hello world" start-index="19" stop-index="31" />
+ </parameter>
+ <parameter>
+ <literal-expression value="hello" start-index="34" stop-index="40" />
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ </select>
+
+ <select sql-case-id="select_conv_with_number" db-types="MySQL,Doris">
+ <projections start-index="7" stop-index="21">
+ <expression-projection start-index="7" stop-index="21" text="CONV(15, 10, 2)">
+ <expr>
+ <function function-name="CONV" text="CONV(15, 10, 2)" start-index="7" stop-index="21">
+ <parameter>
+ <literal-expression value="15" start-index="12" stop-index="13" />
+ </parameter>
+ <parameter>
+ <literal-expression value="10" start-index="16" stop-index="17" />
+ </parameter>
+ <parameter>
+ <literal-expression value="2" start-index="20" stop-index="20" />
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ </select>
+
+ <select sql-case-id="select_conv_with_string" db-types="MySQL,Doris">
+ <projections start-index="7" stop-index="24">
+ <expression-projection start-index="7" stop-index="24" text="CONV('ff', 16, 10)">
+ <expr>
+ <function function-name="CONV" text="CONV('ff', 16, 10)" start-index="7" stop-index="24">
+ <parameter>
+ <literal-expression value="ff" start-index="12" stop-index="15" />
+ </parameter>
+ <parameter>
+ <literal-expression value="16" start-index="18" stop-index="19" />
+ </parameter>
+ <parameter>
+ <literal-expression value="10" start-index="22" stop-index="23" />
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ </select>
+
+ <select sql-case-id="select_bitmap_intersect_doris" db-types="Doris">
+ <projections start-index="7" stop-index="36">
+ <column-projection name="tag" start-index="7" stop-index="9" />
+ <expression-projection start-index="12" stop-index="36" text="BITMAP_INTERSECT(user_id)">
+ <expr>
+ <function function-name="BITMAP_INTERSECT" text="BITMAP_INTERSECT(user_id)" start-index="12" stop-index="36">
+ <parameter>
+ <column name="user_id" start-index="29" stop-index="35" />
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ <from start-index="43" stop-index="166">
+ <subquery-table alias="a" start-index="43" stop-index="166">
+ <subquery start-index="43" stop-index="164">
+ <select>
+ <projections start-index="51" stop-index="90">
+ <column-projection name="tag" start-index="51" stop-index="53" />
+ <column-projection name="date" start-index="56" stop-index="59" />
+ <expression-projection alias="user_id" start-index="62" stop-index="90" text="BITMAP_UNION(user_id)">
+ <expr>
+ <function function-name="BITMAP_UNION" text="BITMAP_UNION(user_id)" start-index="62" stop-index="82">
+ <parameter>
+ <column name="user_id" start-index="75" stop-index="81" />
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ <from>
+ <simple-table name="table" start-index="97" stop-index="101" />
+ </from>
+ <where start-index="103" stop-index="144">
+ <expr>
+ <in-expression start-index="109" stop-index="144">
+ <not>false</not>
+ <left>
+ <column name="date" start-index="109" stop-index="112" />
+ </left>
+ <right>
+ <list-expression start-index="117" stop-index="144">
+ <items>
+ <literal-expression value="2020-05-18" start-index="118" stop-index="129" />
+ </items>
+ <items>
+ <literal-expression value="2020-05-19" start-index="132" stop-index="143" />
+ </items>
+ </list-expression>
+ </right>
+ </in-expression>
+ </expr>
+ </where>
+ <group-by>
+ <column-item name="tag" order-direction="ASC" start-index="155" stop-index="157" />
+ <column-item name="date" order-direction="ASC" start-index="160" stop-index="163" />
+ </group-by>
+ </select>
+ </subquery>
+ </subquery-table>
+ </from>
+ <group-by>
+ <column-item name="tag" order-direction="ASC" start-index="177" stop-index="179" />
+ </group-by>
+ </select>
+
+ <select sql-case-id="select_bitmap_to_string_with_bitmap_intersect_doris" db-types="Doris">
+ <projections start-index="7" stop-index="54">
+ <column-projection name="tag" start-index="7" stop-index="9" />
+ <expression-projection start-index="12" stop-index="54" text="BITMAP_TO_STRING(BITMAP_INTERSECT(user_id))">
+ <expr>
+ <function function-name="BITMAP_TO_STRING" text="BITMAP_TO_STRING(BITMAP_INTERSECT(user_id))" start-index="12" stop-index="54">
+ <parameter>
+ <function function-name="BITMAP_INTERSECT" text="BITMAP_INTERSECT(user_id)" start-index="29" stop-index="53">
+ <parameter>
+ <column name="user_id" start-index="46" stop-index="52" />
+ </parameter>
+ </function>
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ <from start-index="61" stop-index="184">
+ <subquery-table alias="a" start-index="61" stop-index="184">
+ <subquery start-index="61" stop-index="182">
+ <select>
+ <projections start-index="69" stop-index="108">
+ <column-projection name="tag" start-index="69" stop-index="71" />
+ <column-projection name="date" start-index="74" stop-index="77" />
+ <expression-projection alias="user_id" start-index="80" stop-index="108" text="BITMAP_UNION(user_id)">
+ <expr>
+ <function function-name="BITMAP_UNION" text="BITMAP_UNION(user_id)" start-index="80" stop-index="100">
+ <parameter>
+ <column name="user_id" start-index="93" stop-index="99" />
+ </parameter>
+ </function>
+ </expr>
+ </expression-projection>
+ </projections>
+ <from>
+ <simple-table name="table" start-index="115" stop-index="119" />
+ </from>
+ <where start-index="121" stop-index="162">
+ <expr>
+ <in-expression start-index="127" stop-index="162">
+ <not>false</not>
+ <left>
+ <column name="date" start-index="127" stop-index="130" />
+ </left>
+ <right>
+ <list-expression start-index="135" stop-index="162">
+ <items>
+ <literal-expression value="2020-05-18" start-index="136" stop-index="147" />
+ </items>
+ <items>
+ <literal-expression value="2020-05-19" start-index="150" stop-index="161" />
+ </items>
+ </list-expression>
+ </right>
+ </in-expression>
+ </expr>
+ </where>
+ <group-by>
+ <column-item name="tag" order-direction="ASC" start-index="173" stop-index="175" />
+ <column-item name="date" order-direction="ASC" start-index="178" stop-index="181" />
+ </group-by>
+ </select>
+ </subquery>
+ </subquery-table>
+ </from>
+ <group-by>
+ <column-item name="tag" order-direction="ASC" start-index="195" stop-index="197" />
+ </group-by>
+ </select>
</sql-parser-test-cases>
diff --git a/test/it/parser/src/main/resources/sql/supported/ddl/create-table.xml b/test/it/parser/src/main/resources/sql/supported/ddl/create-table.xml
index 83350cdd587..74958afafa7 100644
--- a/test/it/parser/src/main/resources/sql/supported/ddl/create-table.xml
+++ b/test/it/parser/src/main/resources/sql/supported/ddl/create-table.xml
@@ -202,6 +202,7 @@
<sql-case id="create_table_hash_buckets_auto_doris" value="CREATE TABLE t_auto2 (k1 INT) DISTRIBUTED BY HASH(k1) BUCKETS AUTO" db-types="Doris" />
<sql-case id="create_table_rollup_with_duplicate_key_doris" value="CREATE TABLE t_r (k1 INT, v1 INT SUM) AGGREGATE KEY(k1) DISTRIBUTED BY HASH(k1) BUCKETS 1 ROLLUP (r1(k1, v1) DUPLICATE KEY(k1) PROPERTIES("replication_num" = "1"))" db-types="Doris" />
<sql-case id="create_table_with_hll_as_identifier_doris" value="CREATE TABLE hll (hll INT) DISTRIBUTED BY HASH(hll) BUCKETS 1" db-types="Doris" />
+ <sql-case id="create_table_with_varchar_without_length_doris" value="CREATE TABLE tags (tag VARCHAR, date DATETIME, user_id BITMAP BITMAP_UNION) ENGINE=OLAP AGGREGATE KEY(tag, date) DISTRIBUTED BY HASH(tag) BUCKETS 8" db-types="Doris" />
<sql-case id="create_table_with_select_without_query" value="CREATE TABLE t_order_new AS SELECT * FROM t_order" db-types="Oracle"/>
<sql-case id="create_table_as_select_with_pivot" value="CREATE TABLE pivot_table AS SELECT * FROM (SELECT EXTRACT(YEAR FROM order_date) year, order_mode, order_total FROM orders) PIVOT (SUM(order_total) FOR order_mode IN ('direct' AS Store, 'online' AS Internet));" db-types="Oracle"/>
<sql-case id="create_table_as_select_with_pivot_and_aggregate_alias" value="CREATE TABLE pivot_alias_table AS SELECT * FROM (SELECT quarter, quantity_sold FROM sales_view) PIVOT (SUM(quantity_sold) AS sumq FOR quarter IN ('01' AS Q1, '02' AS Q2))" db-types="Oracle" />
diff --git a/test/it/parser/src/main/resources/sql/supported/dml/select-special-function.xml b/test/it/parser/src/main/resources/sql/supported/dml/select-special-function.xml
index 60d00a4c6e5..a6004084f62 100644
--- a/test/it/parser/src/main/resources/sql/supported/dml/select-special-function.xml
+++ b/test/it/parser/src/main/resources/sql/supported/dml/select-special-function.xml
@@ -355,4 +355,10 @@
<sql-case id="select_array_cum_sum" value="SELECT ARRAY_CUM_SUM([1, 2, 3])" db-types="Doris" />
<sql-case id="select_array_cum_sum_empty_array" value="SELECT ARRAY_CUM_SUM([])" db-types="Doris" />
<sql-case id="select_array_cum_sum_from_table" value="SELECT ARRAY_CUM_SUM([1, 2, 3]) FROM t_order" db-types="Doris" />
+ <sql-case id="select_ucase" value="SELECT UCASE('hello')" db-types="MySQL,Doris" />
+ <sql-case id="select_starts_with_doris" value="SELECT STARTS_WITH("hello world", "hello")" db-types="Doris" />
+ <sql-case id="select_conv_with_number" value="SELECT CONV(15, 10, 2)" db-types="MySQL,Doris" />
+ <sql-case id="select_conv_with_string" value="SELECT CONV('ff', 16, 10)" db-types="MySQL,Doris" />
+ <sql-case id="select_bitmap_intersect_doris" value="SELECT tag, BITMAP_INTERSECT(user_id) FROM (SELECT tag, date, BITMAP_UNION(user_id) user_id FROM table WHERE date IN ('2020-05-18', '2020-05-19') GROUP BY tag, date) a GROUP BY tag" db-types="Doris" />
+ <sql-case id="select_bitmap_to_string_with_bitmap_intersect_doris" value="SELECT tag, BITMAP_TO_STRING(BITMAP_INTERSECT(user_id)) FROM (SELECT tag, date, BITMAP_UNION(user_id) user_id FROM table WHERE date IN ('2020-05-18', '2020-05-19') GROUP BY tag, date) a GROUP BY tag" db-types="Doris" />
</sql-cases>