You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by du...@apache.org on 2025/07/15 06:16:21 UTC

(shardingsphere) branch master updated: fix:Support Hive Loading files into tables statement parse (#36001)

This is an automated email from the ASF dual-hosted git repository.

duanzhengqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git


The following commit(s) were added to refs/heads/master by this push:
     new c9c3f4fad3a fix:Support Hive Loading files into tables statement parse (#36001)
c9c3f4fad3a is described below

commit c9c3f4fad3adabcb55257661ec27fd3df5854091
Author: Claire <cl...@163.com>
AuthorDate: Tue Jul 15 14:16:15 2025 +0800

    fix:Support Hive Loading files into tables statement parse (#36001)
    
    * support Hive loading files into tables statement
    
    * add javadoc
    
    * update RELEASE-NOTES
    
    * update RELEASE-NOTES
    
    * update pom.xml
    
    * update pom.xml
    
    * update again
    
    * update parser/sql/statement/type/pom.xml
---
 RELEASE-NOTES.md                                   |  1 +
 parser/sql/dialect/hive/pom.xml                    |  8 +---
 .../src/main/antlr4/imports/hive/DMLStatement.g4   | 24 ++++++++++++
 .../src/main/antlr4/imports/hive/HiveKeyword.g4    | 15 ++++++++
 .../sql/parser/autogen/HiveStatement.g4            |  1 +
 .../statement/type/HiveDMLStatementVisitor.java    | 13 +++++++
 parser/sql/statement/type/{ => hive}/pom.xml       | 18 ++++-----
 .../statement/hive/dml/HiveLoadDataStatement.java  | 45 ++++++++++++++++++++++
 parser/sql/statement/type/pom.xml                  |  1 +
 .../src/main/resources/case/dml/load-data.xml      | 42 ++++++++++++++++++++
 .../main/resources/sql/supported/dml/load-data.xml | 14 +++++++
 11 files changed, 166 insertions(+), 16 deletions(-)

diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 8810de81458..fcfe0ee9cec 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -56,6 +56,7 @@
 1. Proxy: Implement write method for PostgreSQL bool binary data type - [#35831](https://github.com/apache/shardingsphere/pull/35831)
 1. Encrypt: Use EncryptDerivedColumnSuffix to enhance encrypt table subquery rewrite logic - [#34829](https://github.com/apache/shardingsphere/pull/34829)
 1. Encrypt: Add quotes to encrypt rewrite derived columns - [#34950](https://github.com/apache/shardingsphere/pull/34950)
+1. SQL Parser: Support Hive Loading files into tables statement parse - [#36001](https://github.com/apache/shardingsphere/pull/36001)
 1. Encrypt: Add support for NOT LIKE operator in encryption - [#35984](https://github.com/apache/shardingsphere/pull/35984)
 
 ### Bug Fixes
diff --git a/parser/sql/dialect/hive/pom.xml b/parser/sql/dialect/hive/pom.xml
index 16afc55d785..f9f41a66c1d 100644
--- a/parser/sql/dialect/hive/pom.xml
+++ b/parser/sql/dialect/hive/pom.xml
@@ -38,7 +38,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.shardingsphere</groupId>
-            <artifactId>shardingsphere-parser-sql-statement-mysql</artifactId>
+            <artifactId>shardingsphere-parser-sql-statement-hive</artifactId>
             <version>${project.version}</version>
         </dependency>
         
@@ -48,11 +48,5 @@
             <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.shardingsphere</groupId>
-            <artifactId>shardingsphere-test-it-parser</artifactId>
-            <version>${project.version}</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>
diff --git a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4 b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
index 73d90c9bd12..d19d565b256 100644
--- a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
+++ b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
@@ -322,3 +322,27 @@ tableIdentOptWild
 tableAliasRefList
     : tableIdentOptWild (COMMA_ tableIdentOptWild)*
     ;
+
+loadStatement
+    : loadDataStatement
+    ;
+
+loadDataStatement
+    : LOAD DATA LOCAL? INPATH string_ overWrite? inputFormatClause? INTO TABLE tableName partitionSpec?
+    ;
+
+overWrite
+    : OVERWRITE
+    ;
+
+inputFormatClause
+    : INPUTFORMAT string_ SERDE string_
+    ;
+
+partitionSpec
+    : PARTITION LP_ partitionKeyValue (COMMA_ partitionKeyValue)* RP_
+    ;
+
+partitionKeyValue
+    : identifier EQ_ (string_ | numberLiterals)
+    ;
diff --git a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4 b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4
index 900e886e485..8850c9765b7 100644
--- a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4
+++ b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4
@@ -3158,3 +3158,18 @@ DBPROPERTIES
     : D B P R O P E R T I E S
     ;
 
+INPATH
+    : I N P A T H
+    ;
+
+INPUTFORMAT
+    : I N P U T F O R M A T
+    ;
+
+OVERWRITE
+    : O V E R W R I T E
+    ;
+
+SERDE
+    : S E R D E
+    ;
diff --git a/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4 b/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4
index c862c1a6993..36d0b41a0d9 100644
--- a/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4
+++ b/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4
@@ -25,6 +25,7 @@ execute
     | insert
     | update
     | delete
+    | loadStatement
     | createDatabase
     | dropDatabase
     ) (SEMI_ EOF? | EOF)
diff --git a/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java b/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java
index 8247069ebdd..ee62be0320f 100644
--- a/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java
+++ b/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java
@@ -187,6 +187,9 @@ import org.apache.shardingsphere.sql.parser.statement.core.value.literal.impl.Nu
 import org.apache.shardingsphere.sql.parser.statement.core.value.literal.impl.OtherLiteralValue;
 import org.apache.shardingsphere.sql.parser.statement.core.value.literal.impl.StringLiteralValue;
 import org.apache.shardingsphere.sql.parser.statement.core.value.parametermarker.ParameterMarkerValue;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.LoadDataStatementContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.LoadStatementContext;
+import org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveLoadDataStatement;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
@@ -1406,4 +1409,14 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
         getParameterMarkerSegments().add(result);
         return result;
     }
+    
+    @Override
+    public ASTNode visitLoadStatement(final LoadStatementContext ctx) {
+        return visit(ctx.loadDataStatement());
+    }
+    
+    @Override
+    public ASTNode visitLoadDataStatement(final LoadDataStatementContext ctx) {
+        return new HiveLoadDataStatement(getDatabaseType(), (SimpleTableSegment) visit(ctx.tableName()));
+    }
 }
diff --git a/parser/sql/statement/type/pom.xml b/parser/sql/statement/type/hive/pom.xml
similarity index 76%
copy from parser/sql/statement/type/pom.xml
copy to parser/sql/statement/type/hive/pom.xml
index 2d12dd59ea9..814eb3b9783 100644
--- a/parser/sql/statement/type/pom.xml
+++ b/parser/sql/statement/type/hive/pom.xml
@@ -20,17 +20,17 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>org.apache.shardingsphere</groupId>
-        <artifactId>shardingsphere-parser-sql-statement</artifactId>
+        <artifactId>shardingsphere-parser-sql-statement-type</artifactId>
         <version>5.5.3-SNAPSHOT</version>
     </parent>
-    <artifactId>shardingsphere-parser-sql-statement-type</artifactId>
-    <packaging>pom</packaging>
+    <artifactId>shardingsphere-parser-sql-statement-hive</artifactId>
     <name>${project.artifactId}</name>
     
-    <modules>
-        <module>postgresql</module>
-        <module>mysql</module>
-        <module>sqlserver</module>
-        <module>oracle</module>
-    </modules>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.shardingsphere</groupId>
+            <artifactId>shardingsphere-parser-sql-statement-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+    </dependencies>
 </project>
diff --git a/parser/sql/statement/type/hive/src/main/java/org/apache/shardingsphere/sql/parser/statement/hive/dml/HiveLoadDataStatement.java b/parser/sql/statement/type/hive/src/main/java/org/apache/shardingsphere/sql/parser/statement/hive/dml/HiveLoadDataStatement.java
new file mode 100644
index 00000000000..86c3d4e9083
--- /dev/null
+++ b/parser/sql/statement/type/hive/src/main/java/org/apache/shardingsphere/sql/parser/statement/hive/dml/HiveLoadDataStatement.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.sql.parser.statement.hive.dml;
+
+import lombok.Getter;
+
+import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.statement.attribute.SQLStatementAttributes;
+import org.apache.shardingsphere.sql.parser.statement.core.statement.attribute.type.TableSQLStatementAttribute;
+import org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml.DMLStatement;
+
+/**
+ * Load data statement for Hive.
+ */
+@Getter
+public final class HiveLoadDataStatement extends DMLStatement {
+    
+    private final SimpleTableSegment table;
+    
+    public HiveLoadDataStatement(final DatabaseType databaseType, final SimpleTableSegment table) {
+        super(databaseType);
+        this.table = table;
+    }
+    
+    @Override
+    public SQLStatementAttributes getAttributes() {
+        return new SQLStatementAttributes(new TableSQLStatementAttribute(table));
+    }
+}
diff --git a/parser/sql/statement/type/pom.xml b/parser/sql/statement/type/pom.xml
index 2d12dd59ea9..ee18e426080 100644
--- a/parser/sql/statement/type/pom.xml
+++ b/parser/sql/statement/type/pom.xml
@@ -32,5 +32,6 @@
         <module>mysql</module>
         <module>sqlserver</module>
         <module>oracle</module>
+        <module>hive</module>
     </modules>
 </project>
diff --git a/test/it/parser/src/main/resources/case/dml/load-data.xml b/test/it/parser/src/main/resources/case/dml/load-data.xml
index d92da550974..9922c571c50 100644
--- a/test/it/parser/src/main/resources/case/dml/load-data.xml
+++ b/test/it/parser/src/main/resources/case/dml/load-data.xml
@@ -40,4 +40,46 @@
     <load-data sql-case-id="load_data_into_table_with_at_02">
         <table name="t1" start-index="39" stop-index="40"/>
     </load-data>
+    <load-data sql-case-id="load_data_hive_hdfs_to_table">
+        <table name="t_order" start-index="50" stop-index="56" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_local_to_table">
+        <table name="t_order" start-index="56" stop-index="62" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_hdfs_overwrite_table">
+        <table name="t_order" start-index="60" stop-index="66" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_local_overwrite_table">
+        <table name="t_order" start-index="66" stop-index="72" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_hdfs_to_partition">
+        <table name="log_table" start-index="47" stop-index="55" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_local_to_partition">
+        <table name="log_table" start-index="60" stop-index="68" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_hdfs_overwrite_partition">
+        <table name="stats" start-index="60" stop-index="64" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_local_overwrite_partition">
+        <table name="stats" start-index="73" stop-index="77" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_hdfs_with_inputformat_serde">
+        <table name="orc_table" start-index="163" stop-index="171" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_local_with_inputformat_serde">
+        <table name="parquet_table" start-index="205" stop-index="217" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_hdfs_partition_with_inputformat_serde">
+        <table name="orc_partitioned_table" start-index="175" stop-index="195" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_local_partition_with_inputformat_serde">
+        <table name="parquet_partitioned_table" start-index="217" stop-index="241" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_hdfs_overwrite_partition_with_inputformat_serde">
+        <table name="orc_partitioned_table" start-index="186" stop-index="206" />
+    </load-data>
+    <load-data sql-case-id="load_data_hive_local_overwrite_partition_with_inputformat_serde">
+        <table name="parquet_partitioned_table" start-index="225" stop-index="249" />
+    </load-data>
 </sql-parser-test-cases>
diff --git a/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml b/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml
index a919ecee531..4b320a3b74e 100644
--- a/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml
+++ b/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml
@@ -24,4 +24,18 @@
     <sql-case id="load_data_into_table_with_ignore_lines" value="LOAD DATA INFILE '/tmp/test.txt' INTO TABLE t_order IGNORE 1 LINES" db-types="MySQL" />
     <sql-case id="load_data_into_table_with_at_01" value="LOAD DATA INFILE 'file.txt' INTO TABLE t1 (column1, @var1) SET column2 = @var1/100;" db-types="MySQL"/>
     <sql-case id="load_data_into_table_with_at_02" value="LOAD DATA INFILE 'file.txt' INTO TABLE t1 (column1, @dummy, column2, @dummy, column3);" db-types="MySQL"/>
+    <sql-case id="load_data_hive_hdfs_to_table" value="LOAD DATA INPATH '/user/hive/data.txt' INTO TABLE t_order" db-types="Hive" />
+    <sql-case id="load_data_hive_local_to_table" value="LOAD DATA LOCAL INPATH '/home/user/data.txt' INTO TABLE t_order" db-types="Hive" />
+    <sql-case id="load_data_hive_hdfs_overwrite_table" value="LOAD DATA INPATH '/user/hive/data.txt' OVERWRITE INTO TABLE t_order" db-types="Hive" />
+    <sql-case id="load_data_hive_local_overwrite_table" value="LOAD DATA LOCAL INPATH '/home/user/data.txt' OVERWRITE INTO TABLE t_order" db-types="Hive" />
+    <sql-case id="load_data_hive_hdfs_to_partition" value="LOAD DATA INPATH '/user/hive/logs/' INTO TABLE log_table PARTITION (dt='2025-07-11')" db-types="Hive" />
+    <sql-case id="load_data_hive_local_to_partition" value="LOAD DATA LOCAL INPATH '/home/user/region_logs/' INTO TABLE log_table PARTITION (dt='2025-07-11')" db-types="Hive" />
+    <sql-case id="load_data_hive_hdfs_overwrite_partition" value="LOAD DATA INPATH '/user/hive/updates/' OVERWRITE INTO TABLE stats PARTITION (category='sales')" db-types="Hive" />
+    <sql-case id="load_data_hive_local_overwrite_partition" value="LOAD DATA LOCAL INPATH '/home/user/partition_data/' OVERWRITE INTO TABLE stats PARTITION (category='sales')" db-types="Hive" />
+    <sql-case id="load_data_hive_hdfs_with_inputformat_serde" value="LOAD DATA INPATH '/user/hive/orc_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' INTO TABLE orc_table" db-types="Hive" />
+    <sql-case id="load_data_hive_local_with_inputformat_serde" value="LOAD DATA LOCAL INPATH '/home/user/parquet_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' INTO TABLE parquet_table" db-types="Hive" />
+    <sql-case id="load_data_hive_hdfs_partition_with_inputformat_serde" value="LOAD DATA INPATH '/user/hive/partitioned_orc_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' INTO TABLE orc_partitioned_table PARTITION (dt='2025-07-12')" db-types="Hive" />
+    <sql-case id="load_data_hive_local_partition_with_inputformat_serde" value="LOAD DATA LOCAL INPATH '/home/user/partitioned_parquet_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' INTO TABLE parquet_partitioned_table PARTITION (region='north')" db-types="Hive" />
+    <sql-case id="load_data_hive_hdfs_overwrite_partition_with_inputformat_serde" value="LOAD DATA INPATH '/user/hive/overwrite_orc_updates/' OVERWRITE INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' INTO TABLE orc_partitioned_table PARTITION (dt='2025-07-12')" db-types="Hive" />
+    <sql-case id="load_data_hive_local_overwrite_partition_with_inputformat_serde" value="LOAD DATA LOCAL INPATH '/home/user/overwrite_parquet_data/' OVERWRITE INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' INTO TABLE parquet_partitioned_table PARTITION (region='south')" db-types="Hive" />
 </sql-cases>