You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by du...@apache.org on 2025/07/15 06:16:21 UTC
(shardingsphere) branch master updated: fix:Support Hive Loading files into tables statement parse (#36001)
This is an automated email from the ASF dual-hosted git repository.
duanzhengqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new c9c3f4fad3a fix:Support Hive Loading files into tables statement parse (#36001)
c9c3f4fad3a is described below
commit c9c3f4fad3adabcb55257661ec27fd3df5854091
Author: Claire <cl...@163.com>
AuthorDate: Tue Jul 15 14:16:15 2025 +0800
fix:Support Hive Loading files into tables statement parse (#36001)
* support Hive loading files into tables statement
* add javadoc
* update RELEASE-NOTES
* update RELEASE-NOTES
* update pom.xml
* update pom.xml
* update again
* update parser/sql/statement/type/pom.xml
---
RELEASE-NOTES.md | 1 +
parser/sql/dialect/hive/pom.xml | 8 +---
.../src/main/antlr4/imports/hive/DMLStatement.g4 | 24 ++++++++++++
.../src/main/antlr4/imports/hive/HiveKeyword.g4 | 15 ++++++++
.../sql/parser/autogen/HiveStatement.g4 | 1 +
.../statement/type/HiveDMLStatementVisitor.java | 13 +++++++
parser/sql/statement/type/{ => hive}/pom.xml | 18 ++++-----
.../statement/hive/dml/HiveLoadDataStatement.java | 45 ++++++++++++++++++++++
parser/sql/statement/type/pom.xml | 1 +
.../src/main/resources/case/dml/load-data.xml | 42 ++++++++++++++++++++
.../main/resources/sql/supported/dml/load-data.xml | 14 +++++++
11 files changed, 166 insertions(+), 16 deletions(-)
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 8810de81458..fcfe0ee9cec 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -56,6 +56,7 @@
1. Proxy: Implement write method for PostgreSQL bool binary data type - [#35831](https://github.com/apache/shardingsphere/pull/35831)
1. Encrypt: Use EncryptDerivedColumnSuffix to enhance encrypt table subquery rewrite logic - [#34829](https://github.com/apache/shardingsphere/pull/34829)
1. Encrypt: Add quotes to encrypt rewrite derived columns - [#34950](https://github.com/apache/shardingsphere/pull/34950)
+1. SQL Parser: Support Hive Loading files into tables statement parse - [#36001](https://github.com/apache/shardingsphere/pull/36001)
1. Encrypt: Add support for NOT LIKE operator in encryption - [#35984](https://github.com/apache/shardingsphere/pull/35984)
### Bug Fixes
diff --git a/parser/sql/dialect/hive/pom.xml b/parser/sql/dialect/hive/pom.xml
index 16afc55d785..f9f41a66c1d 100644
--- a/parser/sql/dialect/hive/pom.xml
+++ b/parser/sql/dialect/hive/pom.xml
@@ -38,7 +38,7 @@
</dependency>
<dependency>
<groupId>org.apache.shardingsphere</groupId>
- <artifactId>shardingsphere-parser-sql-statement-mysql</artifactId>
+ <artifactId>shardingsphere-parser-sql-statement-hive</artifactId>
<version>${project.version}</version>
</dependency>
@@ -48,11 +48,5 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.shardingsphere</groupId>
- <artifactId>shardingsphere-test-it-parser</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
diff --git a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4 b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
index 73d90c9bd12..d19d565b256 100644
--- a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
+++ b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
@@ -322,3 +322,27 @@ tableIdentOptWild
tableAliasRefList
: tableIdentOptWild (COMMA_ tableIdentOptWild)*
;
+
+loadStatement
+ : loadDataStatement
+ ;
+
+loadDataStatement
+ : LOAD DATA LOCAL? INPATH string_ overWrite? inputFormatClause? INTO TABLE tableName partitionSpec?
+ ;
+
+overWrite
+ : OVERWRITE
+ ;
+
+inputFormatClause
+ : INPUTFORMAT string_ SERDE string_
+ ;
+
+partitionSpec
+ : PARTITION LP_ partitionKeyValue (COMMA_ partitionKeyValue)* RP_
+ ;
+
+partitionKeyValue
+ : identifier EQ_ (string_ | numberLiterals)
+ ;
diff --git a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4 b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4
index 900e886e485..8850c9765b7 100644
--- a/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4
+++ b/parser/sql/dialect/hive/src/main/antlr4/imports/hive/HiveKeyword.g4
@@ -3158,3 +3158,18 @@ DBPROPERTIES
: D B P R O P E R T I E S
;
+INPATH
+ : I N P A T H
+ ;
+
+INPUTFORMAT
+ : I N P U T F O R M A T
+ ;
+
+OVERWRITE
+ : O V E R W R I T E
+ ;
+
+SERDE
+ : S E R D E
+ ;
diff --git a/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4 b/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4
index c862c1a6993..36d0b41a0d9 100644
--- a/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4
+++ b/parser/sql/dialect/hive/src/main/antlr4/org/apache/shardingsphere/sql/parser/autogen/HiveStatement.g4
@@ -25,6 +25,7 @@ execute
| insert
| update
| delete
+ | loadStatement
| createDatabase
| dropDatabase
) (SEMI_ EOF? | EOF)
diff --git a/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java b/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java
index 8247069ebdd..ee62be0320f 100644
--- a/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java
+++ b/parser/sql/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/hive/visitor/statement/type/HiveDMLStatementVisitor.java
@@ -187,6 +187,9 @@ import org.apache.shardingsphere.sql.parser.statement.core.value.literal.impl.Nu
import org.apache.shardingsphere.sql.parser.statement.core.value.literal.impl.OtherLiteralValue;
import org.apache.shardingsphere.sql.parser.statement.core.value.literal.impl.StringLiteralValue;
import org.apache.shardingsphere.sql.parser.statement.core.value.parametermarker.ParameterMarkerValue;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.LoadDataStatementContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.LoadStatementContext;
+import org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveLoadDataStatement;
import java.math.BigDecimal;
import java.util.ArrayList;
@@ -1406,4 +1409,14 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
getParameterMarkerSegments().add(result);
return result;
}
+
+ @Override
+ public ASTNode visitLoadStatement(final LoadStatementContext ctx) {
+ return visit(ctx.loadDataStatement());
+ }
+
+ @Override
+ public ASTNode visitLoadDataStatement(final LoadDataStatementContext ctx) {
+ return new HiveLoadDataStatement(getDatabaseType(), (SimpleTableSegment) visit(ctx.tableName()));
+ }
}
diff --git a/parser/sql/statement/type/pom.xml b/parser/sql/statement/type/hive/pom.xml
similarity index 76%
copy from parser/sql/statement/type/pom.xml
copy to parser/sql/statement/type/hive/pom.xml
index 2d12dd59ea9..814eb3b9783 100644
--- a/parser/sql/statement/type/pom.xml
+++ b/parser/sql/statement/type/hive/pom.xml
@@ -20,17 +20,17 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.shardingsphere</groupId>
- <artifactId>shardingsphere-parser-sql-statement</artifactId>
+ <artifactId>shardingsphere-parser-sql-statement-type</artifactId>
<version>5.5.3-SNAPSHOT</version>
</parent>
- <artifactId>shardingsphere-parser-sql-statement-type</artifactId>
- <packaging>pom</packaging>
+ <artifactId>shardingsphere-parser-sql-statement-hive</artifactId>
<name>${project.artifactId}</name>
- <modules>
- <module>postgresql</module>
- <module>mysql</module>
- <module>sqlserver</module>
- <module>oracle</module>
- </modules>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.shardingsphere</groupId>
+ <artifactId>shardingsphere-parser-sql-statement-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
</project>
diff --git a/parser/sql/statement/type/hive/src/main/java/org/apache/shardingsphere/sql/parser/statement/hive/dml/HiveLoadDataStatement.java b/parser/sql/statement/type/hive/src/main/java/org/apache/shardingsphere/sql/parser/statement/hive/dml/HiveLoadDataStatement.java
new file mode 100644
index 00000000000..86c3d4e9083
--- /dev/null
+++ b/parser/sql/statement/type/hive/src/main/java/org/apache/shardingsphere/sql/parser/statement/hive/dml/HiveLoadDataStatement.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.sql.parser.statement.hive.dml;
+
+import lombok.Getter;
+
+import org.apache.shardingsphere.infra.database.core.type.DatabaseType;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.statement.attribute.SQLStatementAttributes;
+import org.apache.shardingsphere.sql.parser.statement.core.statement.attribute.type.TableSQLStatementAttribute;
+import org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml.DMLStatement;
+
+/**
+ * Load data statement for Hive.
+ */
+@Getter
+public final class HiveLoadDataStatement extends DMLStatement {
+
+ private final SimpleTableSegment table;
+
+ public HiveLoadDataStatement(final DatabaseType databaseType, final SimpleTableSegment table) {
+ super(databaseType);
+ this.table = table;
+ }
+
+ @Override
+ public SQLStatementAttributes getAttributes() {
+ return new SQLStatementAttributes(new TableSQLStatementAttribute(table));
+ }
+}
diff --git a/parser/sql/statement/type/pom.xml b/parser/sql/statement/type/pom.xml
index 2d12dd59ea9..ee18e426080 100644
--- a/parser/sql/statement/type/pom.xml
+++ b/parser/sql/statement/type/pom.xml
@@ -32,5 +32,6 @@
<module>mysql</module>
<module>sqlserver</module>
<module>oracle</module>
+ <module>hive</module>
</modules>
</project>
diff --git a/test/it/parser/src/main/resources/case/dml/load-data.xml b/test/it/parser/src/main/resources/case/dml/load-data.xml
index d92da550974..9922c571c50 100644
--- a/test/it/parser/src/main/resources/case/dml/load-data.xml
+++ b/test/it/parser/src/main/resources/case/dml/load-data.xml
@@ -40,4 +40,46 @@
<load-data sql-case-id="load_data_into_table_with_at_02">
<table name="t1" start-index="39" stop-index="40"/>
</load-data>
+ <load-data sql-case-id="load_data_hive_hdfs_to_table">
+ <table name="t_order" start-index="50" stop-index="56" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_local_to_table">
+ <table name="t_order" start-index="56" stop-index="62" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_hdfs_overwrite_table">
+ <table name="t_order" start-index="60" stop-index="66" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_local_overwrite_table">
+ <table name="t_order" start-index="66" stop-index="72" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_hdfs_to_partition">
+ <table name="log_table" start-index="47" stop-index="55" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_local_to_partition">
+ <table name="log_table" start-index="60" stop-index="68" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_hdfs_overwrite_partition">
+ <table name="stats" start-index="60" stop-index="64" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_local_overwrite_partition">
+ <table name="stats" start-index="73" stop-index="77" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_hdfs_with_inputformat_serde">
+ <table name="orc_table" start-index="163" stop-index="171" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_local_with_inputformat_serde">
+ <table name="parquet_table" start-index="205" stop-index="217" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_hdfs_partition_with_inputformat_serde">
+ <table name="orc_partitioned_table" start-index="175" stop-index="195" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_local_partition_with_inputformat_serde">
+ <table name="parquet_partitioned_table" start-index="217" stop-index="241" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_hdfs_overwrite_partition_with_inputformat_serde">
+ <table name="orc_partitioned_table" start-index="186" stop-index="206" />
+ </load-data>
+ <load-data sql-case-id="load_data_hive_local_overwrite_partition_with_inputformat_serde">
+ <table name="parquet_partitioned_table" start-index="225" stop-index="249" />
+ </load-data>
</sql-parser-test-cases>
diff --git a/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml b/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml
index a919ecee531..4b320a3b74e 100644
--- a/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml
+++ b/test/it/parser/src/main/resources/sql/supported/dml/load-data.xml
@@ -24,4 +24,18 @@
<sql-case id="load_data_into_table_with_ignore_lines" value="LOAD DATA INFILE '/tmp/test.txt' INTO TABLE t_order IGNORE 1 LINES" db-types="MySQL" />
<sql-case id="load_data_into_table_with_at_01" value="LOAD DATA INFILE 'file.txt' INTO TABLE t1 (column1, @var1) SET column2 = @var1/100;" db-types="MySQL"/>
<sql-case id="load_data_into_table_with_at_02" value="LOAD DATA INFILE 'file.txt' INTO TABLE t1 (column1, @dummy, column2, @dummy, column3);" db-types="MySQL"/>
+ <sql-case id="load_data_hive_hdfs_to_table" value="LOAD DATA INPATH '/user/hive/data.txt' INTO TABLE t_order" db-types="Hive" />
+ <sql-case id="load_data_hive_local_to_table" value="LOAD DATA LOCAL INPATH '/home/user/data.txt' INTO TABLE t_order" db-types="Hive" />
+ <sql-case id="load_data_hive_hdfs_overwrite_table" value="LOAD DATA INPATH '/user/hive/data.txt' OVERWRITE INTO TABLE t_order" db-types="Hive" />
+ <sql-case id="load_data_hive_local_overwrite_table" value="LOAD DATA LOCAL INPATH '/home/user/data.txt' OVERWRITE INTO TABLE t_order" db-types="Hive" />
+ <sql-case id="load_data_hive_hdfs_to_partition" value="LOAD DATA INPATH '/user/hive/logs/' INTO TABLE log_table PARTITION (dt='2025-07-11')" db-types="Hive" />
+ <sql-case id="load_data_hive_local_to_partition" value="LOAD DATA LOCAL INPATH '/home/user/region_logs/' INTO TABLE log_table PARTITION (dt='2025-07-11')" db-types="Hive" />
+ <sql-case id="load_data_hive_hdfs_overwrite_partition" value="LOAD DATA INPATH '/user/hive/updates/' OVERWRITE INTO TABLE stats PARTITION (category='sales')" db-types="Hive" />
+ <sql-case id="load_data_hive_local_overwrite_partition" value="LOAD DATA LOCAL INPATH '/home/user/partition_data/' OVERWRITE INTO TABLE stats PARTITION (category='sales')" db-types="Hive" />
+ <sql-case id="load_data_hive_hdfs_with_inputformat_serde" value="LOAD DATA INPATH '/user/hive/orc_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' INTO TABLE orc_table" db-types="Hive" />
+ <sql-case id="load_data_hive_local_with_inputformat_serde" value="LOAD DATA LOCAL INPATH '/home/user/parquet_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' INTO TABLE parquet_table" db-types="Hive" />
+ <sql-case id="load_data_hive_hdfs_partition_with_inputformat_serde" value="LOAD DATA INPATH '/user/hive/partitioned_orc_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' INTO TABLE orc_partitioned_table PARTITION (dt='2025-07-12')" db-types="Hive" />
+ <sql-case id="load_data_hive_local_partition_with_inputformat_serde" value="LOAD DATA LOCAL INPATH '/home/user/partitioned_parquet_data/' INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' INTO TABLE parquet_partitioned_table PARTITION (region='north')" db-types="Hive" />
+ <sql-case id="load_data_hive_hdfs_overwrite_partition_with_inputformat_serde" value="LOAD DATA INPATH '/user/hive/overwrite_orc_updates/' OVERWRITE INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' INTO TABLE orc_partitioned_table PARTITION (dt='2025-07-12')" db-types="Hive" />
+ <sql-case id="load_data_hive_local_overwrite_partition_with_inputformat_serde" value="LOAD DATA LOCAL INPATH '/home/user/overwrite_parquet_data/' OVERWRITE INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' INTO TABLE parquet_partitioned_table PARTITION (region='south')" db-types="Hive" />
</sql-cases>