You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by ch...@apache.org on 2026/07/30 01:03:35 UTC
(shardingsphere) branch master updated: Support Hive parser and binder for insert select columns and transform expressions (#39269)
This is an automated email from the ASF dual-hosted git repository.
FlyingZC pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new ae20897af7c Support Hive parser and binder for insert select columns and transform expressions (#39269)
ae20897af7c is described below
commit ae20897af7c684f47cf561b8d09031c05b509f1f
Author: Zhengqiang Duan <du...@apache.org>
AuthorDate: Thu Jul 30 09:03:27 2026 +0800
Support Hive parser and binder for insert select columns and transform expressions (#39269)
* Support Hive parser for insert select columns and transform expressions
* Support Hive binder for transform expressions
---
RELEASE-NOTES.md | 1 +
...AggregationDistinctProjectionSegmentBinder.java | 5 +-
.../dml/from/type/SimpleTableSegmentBinder.java | 2 +
.../dml/projection/ProjectionsSegmentBinder.java | 8 +--
.../statement/dml/InsertStatementBinder.java | 2 +-
.../statement/dml/SelectStatementBinder.java | 18 ++++-
...egationDistinctProjectionSegmentBinderTest.java | 68 ++++++++++++++++++
.../from/type/SimpleTableSegmentBinderTest.java | 2 +
.../statement/dml/InsertStatementBinderTest.java | 2 +
.../statement/dml/SelectStatementBinderTest.java | 18 +++++
.../src/main/antlr4/imports/hive/DMLStatement.g4 | 19 +++--
.../statement/type/HiveDMLStatementVisitor.java | 82 +++++++++++++++++-----
.../statement/HiveStatementVisitorTest.java | 50 +++++++++++++
.../segment/generic/table/SimpleTableSegment.java | 3 +
.../core/statement/type/dml/InsertStatement.java | 5 +-
.../core/statement/type/dml/SelectStatement.java | 9 ++-
.../statement/type/dml/SelectStatementTest.java | 42 +++++++++++
.../src/main/resources/case/dml/select-hive.xml | 12 +++-
18 files changed, 311 insertions(+), 37 deletions(-)
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 340f7e89274..e4d3f243035 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -71,6 +71,7 @@
1. SQL Parser: Support oracle insert into returning into item statement parse - [#38902](https://github.com/apache/shardingsphere/pull/38902)
1. SQL Parser: Enhance Hive SQL parser to support CTE, lateral view, table sampling, transform and set operations - [#39008](https://github.com/apache/shardingsphere/pull/39008)
1. SQL Parser: Add Hive CTAS and window function parsing - [#39188](https://github.com/apache/shardingsphere/pull/39188)
+1. SQL Parser: Support Hive parser and binder for insert select columns and transform expressions - [#39269](https://github.com/apache/shardingsphere/pull/39269)
1. SQL Parser: Support SQLServer table variable declaration parse - [#38904](https://github.com/apache/shardingsphere/pull/38904)
1. SQL Parser: Support Oracle procedure parser and binder - [#39231](https://github.com/apache/shardingsphere/pull/39231)
1. SQL Parser: Support function table alias column parsing for PostgreSQL and openGauss - [#39268](https://github.com/apache/shardingsphere/pull/39268)
diff --git a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/expression/type/AggregationDistinctProjectionSegmentBinder.java b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/expression/type/AggregationDistinctProjectionSegmentBinder.java
index 7c1abe7d807..92f5e8e4b1a 100644
--- a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/expression/type/AggregationDistinctProjectionSegmentBinder.java
+++ b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/expression/type/AggregationDistinctProjectionSegmentBinder.java
@@ -46,9 +46,10 @@ public final class AggregationDistinctProjectionSegmentBinder {
public static AggregationDistinctProjectionSegment bind(final AggregationDistinctProjectionSegment segment, final SegmentType parentSegmentType,
final SQLStatementBinderContext binderContext, final Multimap<CaseInsensitiveString, TableSegmentBinderContext> tableBinderContexts,
final Multimap<CaseInsensitiveString, TableSegmentBinderContext> outerTableBinderContexts) {
- AggregationDistinctProjectionSegment result =
- new AggregationDistinctProjectionSegment(segment.getStartIndex(), segment.getStopIndex(), segment.getType(), segment.getExpression(), segment.getDistinctInnerExpression());
+ AggregationDistinctProjectionSegment result = new AggregationDistinctProjectionSegment(
+ segment.getStartIndex(), segment.getStopIndex(), segment.getType(), segment.getExpression(), segment.getDistinctInnerExpression(), segment.getSeparator().orElse(null));
segment.getParameters().forEach(each -> result.getParameters().add(ExpressionSegmentBinder.bind(each, parentSegmentType, binderContext, tableBinderContexts, outerTableBinderContexts)));
+ segment.getWindow().ifPresent(optional -> result.setWindow(WindowItemSegmentBinder.bind(optional, parentSegmentType, binderContext, tableBinderContexts, outerTableBinderContexts)));
return result;
}
}
diff --git a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinder.java b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinder.java
index cb2b4d97cf6..9b3dc4c0f9b 100644
--- a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinder.java
+++ b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinder.java
@@ -111,6 +111,7 @@ public final class SimpleTableSegmentBinder {
SimpleTableSegment result = new SimpleTableSegment(tableNameSegment);
segment.getOwner().ifPresent(result::setOwner);
segment.getAliasSegment().ifPresent(result::setAlias);
+ result.setTableSampled(segment.isTableSampled());
segment.getTableSampleExpression().map(optional -> ExpressionSegmentBinder.bind(optional, SegmentType.JOIN_ON, binderContext, tableBinderContexts, LinkedHashMultimap.create()))
.ifPresent(result::setTableSampleExpression);
tableBinderContext.flatMap(context -> segment.getPivot()
@@ -238,6 +239,7 @@ public final class SimpleTableSegmentBinder {
SimpleTableSegment result = new SimpleTableSegment(tableNameSegment);
fromTableOwner.ifPresent(result::setOwner);
result.setAlias(segment.getAliasSegment().orElseGet(() -> new AliasSegment(segment.getTableName().getStartIndex(), segment.getTableName().getStopIndex(), tableName)));
+ result.setTableSampled(segment.isTableSampled());
segment.getTableSampleExpression().map(optional -> ExpressionSegmentBinder.bind(optional, SegmentType.JOIN_ON, binderContext, tableBinderContexts, LinkedHashMultimap.create()))
.ifPresent(result::setTableSampleExpression);
return result;
diff --git a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/projection/ProjectionsSegmentBinder.java b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/projection/ProjectionsSegmentBinder.java
index 0b342718949..7de9e257766 100644
--- a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/projection/ProjectionsSegmentBinder.java
+++ b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/projection/ProjectionsSegmentBinder.java
@@ -24,6 +24,7 @@ import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.shardingsphere.infra.binder.engine.segment.SegmentType;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.ExpressionSegmentBinder;
+import org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.type.AggregationDistinctProjectionSegmentBinder;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.type.AggregationProjectionSegmentBinder;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.from.context.TableSegmentBinderContext;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.from.context.type.SimpleTableSegmentBinderContext;
@@ -130,11 +131,8 @@ public final class ProjectionsSegmentBinder {
final SQLStatementBinderContext binderContext,
final Multimap<CaseInsensitiveString, TableSegmentBinderContext> tableBinderContexts,
final Multimap<CaseInsensitiveString, TableSegmentBinderContext> outerTableBinderContexts) {
- AggregationDistinctProjectionSegment result = new AggregationDistinctProjectionSegment(aggregationDistinctSegment.getStartIndex(), aggregationDistinctSegment.getStopIndex(),
- aggregationDistinctSegment.getType(), aggregationDistinctSegment.getExpression(), aggregationDistinctSegment.getDistinctInnerExpression(),
- aggregationDistinctSegment.getSeparator().orElse(null));
- aggregationDistinctSegment.getParameters()
- .forEach(each -> result.getParameters().add(ExpressionSegmentBinder.bind(each, SegmentType.PROJECTION, binderContext, tableBinderContexts, outerTableBinderContexts)));
+ AggregationDistinctProjectionSegment result =
+ AggregationDistinctProjectionSegmentBinder.bind(aggregationDistinctSegment, SegmentType.PROJECTION, binderContext, tableBinderContexts, outerTableBinderContexts);
aggregationDistinctSegment.getAliasSegment().ifPresent(result::setAlias);
return result;
}
diff --git a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinder.java b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinder.java
index 9bf403082fb..a66c731d5f0 100644
--- a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinder.java
+++ b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinder.java
@@ -101,7 +101,7 @@ public final class InsertStatementBinder implements SQLStatementBinder<InsertSta
.multiTableInsertInto(sqlStatement.getMultiTableInsertInto().orElse(null)).multiTableConditionalInto(sqlStatement.getMultiTableConditionalInto().orElse(null))
.where(sqlStatement.getWhere().orElse(null)).exec(sqlStatement.getExec().orElse(null)).withTableHint(sqlStatement.getWithTableHint().orElse(null))
.rowSetFunction(sqlStatement.getRowSetFunction().orElse(null)).ignore(sqlStatement.isIgnore()).replace(sqlStatement.isReplace())
- .values(new LinkedList<>(boundValues)).build();
+ .overwrite(sqlStatement.isOverwrite()).values(new LinkedList<>(boundValues)).build();
SQLStatementCopyUtils.copyAttributes(sqlStatement, result);
return result;
}
diff --git a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinder.java b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinder.java
index 324c27e7fd7..2db9faa44be 100644
--- a/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinder.java
+++ b/infra/binder/core/src/main/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinder.java
@@ -24,6 +24,7 @@ import lombok.RequiredArgsConstructor;
import org.apache.shardingsphere.infra.binder.engine.segment.SegmentType;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.combine.CombineSegmentBinder;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.ExpressionSegmentBinder;
+import org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.type.FunctionExpressionSegmentBinder;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.type.WindowItemSegmentBinder;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.from.TableSegmentBinder;
import org.apache.shardingsphere.infra.binder.engine.segment.dml.from.context.TableSegmentBinderContext;
@@ -43,6 +44,7 @@ import org.apache.shardingsphere.infra.binder.engine.statement.SQLStatementCopyU
import org.apache.shardingsphere.sql.parser.statement.core.enums.TableSourceType;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.column.ColumnSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.combine.CombineSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.FunctionSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ProjectionSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ProjectionsSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.order.GroupBySegment;
@@ -96,8 +98,9 @@ public final class SelectStatementBinder implements SQLStatementBinder<SelectSta
HavingSegment boundHaving =
sqlStatement.getHaving().map(optional -> HavingSegmentBinder.bind(optional, binderContext, currentTableBinderContexts, tableBinderContexts, outerTableBinderContexts)).orElse(null);
WindowSegment boundWindow = sqlStatement.getWindow().map(optional -> bindWindowSegment(optional, binderContext, tableBinderContexts, outerTableBinderContexts)).orElse(null);
+ Collection<FunctionSegment> boundTransformSegments = bindTransformSegments(sqlStatement, binderContext, tableBinderContexts, outerTableBinderContexts);
return copy(sqlStatement, boundWith, boundFrom.orElse(null), boundProjections, boundWhere,
- boundHierarchicalQuery, boundCombine, boundLock, boundGroupBy, boundOrderBy, boundHaving, boundModel, boundWindow);
+ boundHierarchicalQuery, boundCombine, boundLock, boundGroupBy, boundOrderBy, boundHaving, boundModel, boundWindow, boundTransformSegments);
}
private ModelSegment bindModelSegment(final ModelSegment segment, final SQLStatementBinderContext binderContext,
@@ -185,15 +188,26 @@ public final class SelectStatementBinder implements SQLStatementBinder<SelectSta
private SelectStatement copy(final SelectStatement sqlStatement, final WithSegment boundWith, final TableSegment boundFrom, final ProjectionsSegment boundProjections,
final WhereSegment boundWhere, final HierarchicalQuerySegment boundHierarchicalQuery, final CombineSegment boundCombine,
final LockSegment boundLock, final GroupBySegment boundGroupBy, final OrderBySegment boundOrderBy, final HavingSegment boundHaving,
- final ModelSegment boundModel, final WindowSegment boundWindow) {
+ final ModelSegment boundModel, final WindowSegment boundWindow, final Collection<FunctionSegment> boundTransformSegments) {
SelectStatement result = SelectStatement.builder().databaseType(sqlStatement.getDatabaseType()).with(boundWith).from(boundFrom).projections(boundProjections)
.where(boundWhere).hierarchicalQuery(boundHierarchicalQuery).combine(boundCombine).lock(boundLock).groupBy(boundGroupBy).orderBy(boundOrderBy).having(boundHaving)
.limit(sqlStatement.getLimit().orElse(null)).window(boundWindow).model(boundModel)
.subqueryType(sqlStatement.getSubqueryType().orElse(null)).build();
+ result.getTransformSegments().addAll(boundTransformSegments);
SQLStatementCopyUtils.copyAttributes(sqlStatement, result);
return result;
}
+ private Collection<FunctionSegment> bindTransformSegments(final SelectStatement sqlStatement, final SQLStatementBinderContext binderContext,
+ final Multimap<CaseInsensitiveString, TableSegmentBinderContext> tableBinderContexts,
+ final Multimap<CaseInsensitiveString, TableSegmentBinderContext> outerTableBinderContexts) {
+ Collection<FunctionSegment> result = new LinkedList<>();
+ for (FunctionSegment each : sqlStatement.getTransformSegments()) {
+ result.add(FunctionExpressionSegmentBinder.bind(each, SegmentType.PROJECTION, binderContext, tableBinderContexts, outerTableBinderContexts));
+ }
+ return result;
+ }
+
private WindowSegment bindWindowSegment(final WindowSegment segment, final SQLStatementBinderContext binderContext,
final Multimap<CaseInsensitiveString, TableSegmentBinderContext> tableBinderContexts,
final Multimap<CaseInsensitiveString, TableSegmentBinderContext> outerTableBinderContexts) {
diff --git a/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/expression/type/AggregationDistinctProjectionSegmentBinderTest.java b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/expression/type/AggregationDistinctProjectionSegmentBinderTest.java
new file mode 100644
index 00000000000..6bae73d833b
--- /dev/null
+++ b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/expression/type/AggregationDistinctProjectionSegmentBinderTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.type;
+
+import com.cedarsoftware.util.CaseInsensitiveMap.CaseInsensitiveString;
+import com.google.common.collect.LinkedHashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.shardingsphere.infra.binder.engine.segment.SegmentType;
+import org.apache.shardingsphere.infra.binder.engine.segment.dml.expression.ExpressionSegmentBinder;
+import org.apache.shardingsphere.infra.binder.engine.segment.dml.from.context.TableSegmentBinderContext;
+import org.apache.shardingsphere.infra.binder.engine.statement.SQLStatementBinderContext;
+import org.apache.shardingsphere.sql.parser.statement.core.enums.AggregationType;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.ExpressionSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.AggregationDistinctProjectionSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.WindowItemSegment;
+import org.apache.shardingsphere.test.infra.framework.extension.mock.AutoMockExtension;
+import org.apache.shardingsphere.test.infra.framework.extension.mock.StaticMockSettings;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(AutoMockExtension.class)
+@StaticMockSettings({ExpressionSegmentBinder.class, WindowItemSegmentBinder.class})
+class AggregationDistinctProjectionSegmentBinderTest {
+
+ @Test
+ void assertBind() {
+ AggregationDistinctProjectionSegment segment = new AggregationDistinctProjectionSegment(0, 72, AggregationType.COUNT,
+ "COUNT(DISTINCT product_id) OVER (PARTITION BY order_id)", "product_id", ",");
+ ExpressionSegment parameter = mock(ExpressionSegment.class);
+ segment.getParameters().add(parameter);
+ WindowItemSegment windowItemSegment = new WindowItemSegment(33, 72);
+ segment.setWindow(windowItemSegment);
+ ExpressionSegment expectedParameter = mock(ExpressionSegment.class);
+ WindowItemSegment expectedWindowItemSegment = mock(WindowItemSegment.class);
+ SQLStatementBinderContext binderContext = mock(SQLStatementBinderContext.class);
+ Multimap<CaseInsensitiveString, TableSegmentBinderContext> tableBinderContexts = LinkedHashMultimap.create();
+ Multimap<CaseInsensitiveString, TableSegmentBinderContext> outerTableBinderContexts = LinkedHashMultimap.create();
+ when(ExpressionSegmentBinder.bind(parameter, SegmentType.PROJECTION, binderContext, tableBinderContexts, outerTableBinderContexts)).thenReturn(expectedParameter);
+ when(WindowItemSegmentBinder.bind(windowItemSegment, SegmentType.PROJECTION, binderContext, tableBinderContexts, outerTableBinderContexts)).thenReturn(expectedWindowItemSegment);
+ AggregationDistinctProjectionSegment actual =
+ AggregationDistinctProjectionSegmentBinder.bind(segment, SegmentType.PROJECTION, binderContext, tableBinderContexts, outerTableBinderContexts);
+ assertThat(actual.getParameters().iterator().next(), is(expectedParameter));
+ assertThat(actual.getSeparator().orElse(null), is(","));
+ assertTrue(actual.getWindow().isPresent());
+ assertThat(actual.getWindow().get(), is(expectedWindowItemSegment));
+ }
+}
diff --git a/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinderTest.java b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinderTest.java
index 9abd5895883..7510164570e 100644
--- a/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinderTest.java
+++ b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/segment/dml/from/type/SimpleTableSegmentBinderTest.java
@@ -81,10 +81,12 @@ class SimpleTableSegmentBinderTest {
@Test
void assertBindTableSampleExpression() {
SimpleTableSegment simpleTableSegment = new SimpleTableSegment(new TableNameSegment(0, 6, new IdentifierValue("t_order")));
+ simpleTableSegment.setTableSampled(true);
simpleTableSegment.setTableSampleExpression(new ColumnSegment(31, 38, new IdentifierValue("order_id")));
Multimap<CaseInsensitiveString, TableSegmentBinderContext> tableBinderContexts = LinkedHashMultimap.create();
SimpleTableSegment actual = SimpleTableSegmentBinder.bind(simpleTableSegment, new SQLStatementBinderContext(
createMetaData(), "foo_db", new HintValueContext(), SelectStatement.builder().databaseType(databaseType).build()), tableBinderContexts);
+ assertTrue(actual.isTableSampled());
assertTrue(actual.getTableSampleExpression().isPresent());
assertTrue(actual.getTableSampleExpression().get() instanceof ColumnSegment);
ColumnSegment actualExpression = (ColumnSegment) actual.getTableSampleExpression().get();
diff --git a/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinderTest.java b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinderTest.java
index f3c46721454..fd210a37683 100644
--- a/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinderTest.java
+++ b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/InsertStatementBinderTest.java
@@ -72,9 +72,11 @@ class InsertStatementBinderTest {
new ColumnSegment(0, 0, new IdentifierValue("user_id")), new ColumnSegment(0, 0, new IdentifierValue("status")))))
.values(Collections.singletonList(new InsertValuesSegment(0, 0, Arrays.asList(new LiteralExpressionSegment(0, 0, 1),
new LiteralExpressionSegment(0, 0, 1), new LiteralExpressionSegment(0, 0, "OK")))))
+ .overwrite(true)
.build();
InsertStatement actual = new InsertStatementBinder().bind(insertStatement, new SQLStatementBinderContext(createMetaData(), "foo_db", new HintValueContext(), insertStatement));
assertThat(actual, not(insertStatement));
+ assertTrue(actual.isOverwrite());
assertTrue(actual.getTable().isPresent());
assertTrue(insertStatement.getTable().isPresent());
assertThat(actual.getTable().get().getTableName(), not(insertStatement.getTable().get().getTableName()));
diff --git a/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinderTest.java b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinderTest.java
index be98c2ba8aa..98f6c9c411c 100644
--- a/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinderTest.java
+++ b/infra/binder/core/src/test/java/org/apache/shardingsphere/infra/binder/engine/statement/dml/SelectStatementBinderTest.java
@@ -191,6 +191,24 @@ class SelectStatementBinderTest {
assertThat(actualColumnOrderByItem.getColumn().getColumnBoundInfo().getOriginalTable().getValue(), is("t_user"));
}
+ @Test
+ void assertBindTransformSegments() {
+ ProjectionsSegment projections = new ProjectionsSegment(0, 0);
+ projections.getProjections().add(new ColumnProjectionSegment(new ColumnSegment(0, 0, new IdentifierValue("user_id"))));
+ FunctionSegment transformSegment = new FunctionSegment(0, 0, "TRANSFORM", "TRANSFORM(user_name)");
+ transformSegment.getParameters().add(new ColumnSegment(0, 0, new IdentifierValue("user_name")));
+ SelectStatement selectStatement = SelectStatement.builder().databaseType(databaseType).projections(projections)
+ .from(new SimpleTableSegment(new TableNameSegment(0, 0, new IdentifierValue("t_user")))).build();
+ selectStatement.getTransformSegments().add(transformSegment);
+ SelectStatement actual = new SelectStatementBinder().bind(selectStatement,
+ new SQLStatementBinderContext(mockMetaData(), "foo_db", new HintValueContext(), selectStatement));
+ FunctionSegment actualTransformSegment = actual.getTransformSegments().iterator().next();
+ assertThat(actualTransformSegment, not(transformSegment));
+ ColumnSegment actualParameter = (ColumnSegment) actualTransformSegment.getParameters().iterator().next();
+ assertThat(actualParameter.getColumnBoundInfo().getOriginalTable().getValue(), is("t_user"));
+ assertThat(actualParameter.getColumnBoundInfo().getOriginalColumn().getValue(), is("user_name"));
+ }
+
@Test
void assertBindWithLikeSubquery() {
ProjectionsSegment subqueryProjections = new ProjectionsSegment(0, 0);
diff --git a/parser/sql/engine/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4 b/parser/sql/engine/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
index 9b04b59f470..561c25c4d34 100644
--- a/parser/sql/engine/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
+++ b/parser/sql/engine/dialect/hive/src/main/antlr4/imports/hive/DMLStatement.g4
@@ -20,7 +20,7 @@ grammar DMLStatement;
import BaseRule;
insert
- : INSERT insertSpecification INTO? tableName partitionNames? (insertValuesClause | setAssignmentsClause | insertSelectClause) onDuplicateKeyClause?
+ : withClause? INSERT insertSpecification INTO? tableName partitionNames? (insertSelectClause | insertValuesClause | setAssignmentsClause) onDuplicateKeyClause?
| insertDataIntoTablesFromQueries
| writingDataIntoFileSystem
| insertingValuesIntoTables
@@ -50,6 +50,10 @@ insertSelectClause
: valueReference? (LP_ fields? RP_)? select
;
+insertSelectColumns
+ : LP_ fields? RP_
+ ;
+
onDuplicateKeyClause
: (AS identifier)? ON DUPLICATE KEY UPDATE assignment (COMMA_ assignment)*
;
@@ -442,7 +446,7 @@ insertDataIntoTablesFromQueries
standardSyntax
: INSERT OVERWRITE TABLE tableName partitionClause? select
- | INSERT INTO TABLE tableName partitionSpec? select
+ | INSERT INTO TABLE tableName partitionSpec? insertSelectColumns? select
;
multipleInserts
@@ -450,7 +454,7 @@ multipleInserts
;
dynamicPartitionInserts
- : INSERT (OVERWRITE | INTO) TABLE tableName dynamicPartitionClause select
+ : INSERT (OVERWRITE | INTO) TABLE tableName dynamicPartitionClause insertSelectColumns? select
;
hiveMultipleInserts
@@ -459,7 +463,7 @@ hiveMultipleInserts
hiveInsertStatement
: INSERT OVERWRITE TABLE tableName partitionClause? select
- | INSERT INTO TABLE tableName partitionSpec? select
+ | INSERT INTO TABLE tableName partitionSpec? insertSelectColumns? select
;
dynamicPartitionClause
@@ -476,7 +480,12 @@ partitionClause
writingDataIntoFileSystem
: insertOverwriteStandardSyntax
- | fromClause insertOverwriteStandardSyntax+
+ | fromClause hiveFileSystemInsertStatement+
+ ;
+
+hiveFileSystemInsertStatement
+ : insertOverwriteStandardSyntax
+ | hiveInsertStatement
;
insertOverwriteStandardSyntax
diff --git a/parser/sql/engine/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/type/HiveDMLStatementVisitor.java b/parser/sql/engine/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/type/HiveDMLStatementVisitor.java
index 1fe6c5aed98..adbd929f686 100644
--- a/parser/sql/engine/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/type/HiveDMLStatementVisitor.java
+++ b/parser/sql/engine/dialect/hive/src/main/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/type/HiveDMLStatementVisitor.java
@@ -57,8 +57,11 @@ import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.FieldsCo
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.FromClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.FunctionCallContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.GroupByClauseContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.GroupByModifierContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.GroupConcatFunctionContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.GroupingSetContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.HavingClauseContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.HiveFileSystemInsertStatementContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.HiveInsertStatementContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.HiveSelectTailClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.HiveMultipleInsertsContext;
@@ -67,6 +70,7 @@ import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertCo
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertDataIntoTablesFromQueriesContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertIdentifierContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertOverwriteStandardSyntaxContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertSelectColumnsContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertSelectClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.InsertValuesClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.IntervalExpressionContext;
@@ -86,6 +90,7 @@ import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.LoadStat
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.LockClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.LockClauseListContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.MatchExpressionContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.MapReduceClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.MultipleInsertsContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.MultipleTablesClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.NaturalJoinTypeContext;
@@ -124,6 +129,7 @@ import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TableRef
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TableReferencesContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TableStatementContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TableValueConstructorContext;
+import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TransformClauseContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TransformMapReduceQuerySpecificationContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TransformOutputColumnContext;
import org.apache.shardingsphere.sql.parser.autogen.HiveStatementParser.TransformOutputContext;
@@ -418,6 +424,7 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
SelectStatement result = selectStatementBuilder.build();
addHiveSystemColumnVariableNames(result);
if (null != ctx.projections().transformClause()) {
+ result.getTransformSegments().add(createTransformSegment(ctx.projections().transformClause()));
addTransformOutputVariableNames(result, ctx.projections().transformClause().transformOutput());
}
return result;
@@ -432,6 +439,7 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
}
SelectStatement result = selectStatementBuilder.build();
addHiveSystemColumnVariableNames(result);
+ result.getTransformSegments().add(createMapReduceSegment(ctx.mapReduceClause()));
addTransformOutputVariableNames(result, ctx.mapReduceClause().transformOutput());
return result;
}
@@ -597,13 +605,21 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
AggregationDistinctProjectionSegment result = new AggregationDistinctProjectionSegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex(),
type, innerExpression, getDistinctExpression(ctx));
result.getParameters().addAll(getExpressions(ctx));
+ setAggregationWindow(ctx, result);
return result;
}
AggregationProjectionSegment result = new AggregationProjectionSegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex(), type, innerExpression);
result.getParameters().addAll(getExpressions(ctx));
+ setAggregationWindow(ctx, result);
return result;
}
+ private void setAggregationWindow(final AggregationFunctionContext ctx, final AggregationProjectionSegment result) {
+ if (null != ctx.overClause() && null != ctx.overClause().windowSpecification()) {
+ result.setWindow((WindowItemSegment) visit(ctx.overClause().windowSpecification()));
+ }
+ }
+
private Collection<ExpressionSegment> getExpressions(final AggregationFunctionContext ctx) {
if (null == ctx.expr()) {
return Collections.emptyList();
@@ -1066,7 +1082,7 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
.setAssignment(insertStatement.getSetAssignment().orElse(null))
.onDuplicateKeyColumns(null == ctx.onDuplicateKeyClause() ? insertStatement.getOnDuplicateKeyColumns().orElse(null) : (OnDuplicateKeyColumnsSegment) visit(ctx.onDuplicateKeyClause()))
.valueReference(insertStatement.getValueReference().orElse(null)).returning(insertStatement.getReturning().orElse(null))
- .output(insertStatement.getOutput().orElse(null)).with(insertStatement.getWith().orElse(null))
+ .output(insertStatement.getOutput().orElse(null)).with(null == ctx.withClause() ? insertStatement.getWith().orElse(null) : (WithSegment) visit(ctx.withClause()))
.multiTableInsertType(insertStatement.getMultiTableInsertType().orElse(null)).multiTableInsertInto(insertStatement.getMultiTableInsertInto().orElse(null))
.multiTableConditionalInto(insertStatement.getMultiTableConditionalInto().orElse(null)).where(insertStatement.getWhere().orElse(null))
.exec(insertStatement.getExec().orElse(null)).withTableHint(insertStatement.getWithTableHint().orElse(null))
@@ -1095,7 +1111,7 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
@Override
public ASTNode visitStandardSyntax(final StandardSyntaxContext ctx) {
- return createHiveInsertStatement(ctx.tableName(), ctx.select(), ctx.start.getStartIndex());
+ return createHiveInsertStatement(ctx.tableName(), ctx.insertSelectColumns(), ctx.select(), ctx.start.getStartIndex());
}
@Override
@@ -1109,7 +1125,7 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
@Override
public ASTNode visitDynamicPartitionInserts(final DynamicPartitionInsertsContext ctx) {
- return createHiveInsertStatement(ctx.tableName(), ctx.select(), ctx.start.getStartIndex());
+ return createHiveInsertStatement(ctx.tableName(), ctx.insertSelectColumns(), ctx.select(), ctx.start.getStartIndex());
}
@Override
@@ -1147,9 +1163,9 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
.ifPresent(selectStmt -> subquery.setSelect(createSelectStatementBuilder(selectStmt).from(sourceTable).build()));
}
- private InsertStatement createHiveInsertStatement(final TableNameContext tableName, final SelectContext select, final int startIndex) {
+ private InsertStatement createHiveInsertStatement(final TableNameContext tableName, final InsertSelectColumnsContext insertSelectColumns, final SelectContext select, final int startIndex) {
InsertStatement result = InsertStatement.builder().databaseType(getDatabaseType()).table((SimpleTableSegment) visit(tableName))
- .insertColumns(new InsertColumnsSegment(startIndex, startIndex, Collections.emptyList())).insertSelect(createInsertSelectSegment(select)).build();
+ .insertColumns(createInsertColumns(insertSelectColumns, startIndex)).insertSelect(createInsertSelectSegment(select)).build();
result.addParameterMarkers(getParameterMarkerSegments());
return result;
}
@@ -1157,7 +1173,7 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
@Override
public ASTNode visitHiveInsertStatement(final HiveInsertStatementContext ctx) {
return InsertStatement.builder().databaseType(getDatabaseType()).table((SimpleTableSegment) visit(ctx.tableName()))
- .insertColumns(new InsertColumnsSegment(ctx.start.getStartIndex(), ctx.start.getStartIndex(), Collections.emptyList()))
+ .insertColumns(createInsertColumns(ctx.insertSelectColumns(), ctx.start.getStartIndex()))
.insertSelect(null == ctx.select() ? null : createInsertSelectSegment(ctx.select())).build();
}
@@ -1173,26 +1189,22 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
@Override
public ASTNode visitWritingDataIntoFileSystem(final WritingDataIntoFileSystemContext ctx) {
- List<InsertOverwriteStandardSyntaxContext> statements = ctx.insertOverwriteStandardSyntax();
- if (1 == statements.size() && null == ctx.fromClause()) {
- return visit(statements.get(0));
+ if (null != ctx.insertOverwriteStandardSyntax()) {
+ return visit(ctx.insertOverwriteStandardSyntax());
}
- TableSegment sourceTable = null != ctx.fromClause() ? (TableSegment) visit(ctx.fromClause()) : null;
+ List<HiveFileSystemInsertStatementContext> statements = ctx.hiveFileSystemInsertStatement();
+ TableSegment sourceTable = (TableSegment) visit(ctx.fromClause());
if (1 == statements.size()) {
InsertStatement single = (InsertStatement) visit(statements.get(0));
- if (null != sourceTable) {
- single.getInsertSelect().ifPresent(subquery -> setFromForSelect(subquery, sourceTable));
- }
+ single.getInsertSelect().ifPresent(subquery -> setFromForSelect(subquery, sourceTable));
single.addParameterMarkers(getParameterMarkerSegments());
return single;
}
MultiTableInsertIntoSegment multiTableInsertInto = new MultiTableInsertIntoSegment(
ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex());
- for (InsertOverwriteStandardSyntaxContext each : statements) {
+ for (HiveFileSystemInsertStatementContext each : statements) {
InsertStatement insertStmt = (InsertStatement) visit(each);
- if (null != sourceTable) {
- insertStmt.getInsertSelect().ifPresent(subquery -> setFromForSelect(subquery, sourceTable));
- }
+ insertStmt.getInsertSelect().ifPresent(subquery -> setFromForSelect(subquery, sourceTable));
insertStmt.addParameterMarkers(getParameterMarkerSegments());
multiTableInsertInto.getInsertStatements().add(insertStmt);
}
@@ -1298,6 +1310,14 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
return result;
}
+ private InsertColumnsSegment createInsertColumns(final InsertSelectColumnsContext insertSelectColumns, final int defaultIndex) {
+ if (null == insertSelectColumns) {
+ return new InsertColumnsSegment(defaultIndex, defaultIndex, Collections.emptyList());
+ }
+ return new InsertColumnsSegment(insertSelectColumns.LP_().getSymbol().getStartIndex(), insertSelectColumns.RP_().getSymbol().getStopIndex(),
+ null == insertSelectColumns.fields() ? Collections.emptyList() : createInsertColumns(insertSelectColumns.fields()));
+ }
+
@Override
public ASTNode visitUpdate(final UpdateContext ctx) {
TableSegment tableSegment = (TableSegment) visit(ctx.tableReferences());
@@ -1526,6 +1546,22 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
return new ColumnProjectionSegment(new ColumnSegment(startIndex, stopIndex, new IdentifierValue(name)));
}
+ private FunctionSegment createTransformSegment(final TransformClauseContext ctx) {
+ FunctionSegment result = new FunctionSegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex(), ctx.TRANSFORM().getText(), getOriginalText(ctx));
+ for (ExprContext each : ctx.expr()) {
+ result.getParameters().add((ExpressionSegment) visit(each));
+ }
+ return result;
+ }
+
+ private FunctionSegment createMapReduceSegment(final MapReduceClauseContext ctx) {
+ FunctionSegment result = new FunctionSegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex(), ctx.getStart().getText(), getOriginalText(ctx));
+ for (ExprContext each : ctx.expr()) {
+ result.getParameters().add((ExpressionSegment) visit(each));
+ }
+ return result;
+ }
+
@Override
public ASTNode visitProjection(final ProjectionContext ctx) {
// FIXME :The stop index of project is the stop index of projection, instead of alias.
@@ -1804,7 +1840,16 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
for (OrderByItemContext each : ctx.orderByItem()) {
items.add((OrderByItemSegment) visit(each));
}
- return new GroupBySegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex(), items, null != ctx.groupByModifier() && null != ctx.groupByModifier().ROLLUP());
+ GroupByModifierContext groupByModifier = ctx.groupByModifier();
+ if (null != groupByModifier) {
+ for (GroupingSetContext each : groupByModifier.groupingSet()) {
+ for (OrderByItemContext orderByItem : each.orderByItem()) {
+ items.add((OrderByItemSegment) visit(orderByItem));
+ }
+ }
+ }
+ boolean withRollup = null != groupByModifier && null != groupByModifier.ROLLUP();
+ return new GroupBySegment(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex(), items, withRollup, null != groupByModifier);
}
@Override
@@ -1887,6 +1932,7 @@ public final class HiveDMLStatementVisitor extends HiveStatementVisitor implemen
private SelectStatement buildSelectStatement(final SelectStatement.SelectStatementBuilder builder, final SelectStatement source) {
SelectStatement result = builder.build();
result.getVariableNames().addAll(source.getVariableNames());
+ result.getTransformSegments().addAll(source.getTransformSegments());
return result;
}
diff --git a/parser/sql/engine/dialect/hive/src/test/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/HiveStatementVisitorTest.java b/parser/sql/engine/dialect/hive/src/test/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/HiveStatementVisitorTest.java
index 7a80ee0bccd..1893efc1d84 100644
--- a/parser/sql/engine/dialect/hive/src/test/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/HiveStatementVisitorTest.java
+++ b/parser/sql/engine/dialect/hive/src/test/java/org/apache/shardingsphere/sql/parser/engine/hive/visitor/statement/HiveStatementVisitorTest.java
@@ -24,7 +24,9 @@ import org.apache.shardingsphere.sql.parser.engine.core.ParseASTNode;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.assignment.ColumnAssignmentSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.CaseWhenExpression;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.FunctionSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.AggregationProjectionSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ExpressionProjectionSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.order.GroupBySegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.WindowItemSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment;
import org.apache.shardingsphere.sql.parser.statement.core.statement.type.ddl.table.CreateTableStatement;
@@ -34,6 +36,7 @@ import org.junit.jupiter.api.Test;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class HiveStatementVisitorTest {
@@ -44,6 +47,8 @@ class HiveStatementVisitorTest {
void assertVisitTransformOutputVariables() {
SelectStatement actual = parseSelectStatement("SELECT TRANSFORM(user_id) USING 'cat' AS (transformed_user_id INT) FROM t_user");
assertTrue(actual.getVariableNames().contains("transformed_user_id"));
+ assertThat(actual.getTransformSegments().size(), is(1));
+ assertThat(actual.getTransformSegments().iterator().next().getParameters().size(), is(1));
}
@Test
@@ -53,6 +58,33 @@ class HiveStatementVisitorTest {
assertTrue(actual.getVariableNames().contains("value"));
}
+ @Test
+ void assertVisitMapReduceInputExpressions() {
+ SelectStatement actual = parseSelectStatement("FROM t_user MAP user_id, user_name USING 'cat' AS transformed_user_id, transformed_user_name");
+ assertThat(actual.getTransformSegments().size(), is(1));
+ FunctionSegment transformSegment = actual.getTransformSegments().iterator().next();
+ assertThat(transformSegment.getFunctionName(), is("MAP"));
+ assertThat(transformSegment.getParameters().size(), is(2));
+ }
+
+ @Test
+ void assertVisitCubeGroupingExtension() {
+ SelectStatement actual = parseSelectStatement("SELECT status, merchant_id, grouping__id, count(*) FROM t_order GROUP BY status, merchant_id WITH CUBE");
+ GroupBySegment actualGroupBy = actual.getGroupBy().orElseThrow(AssertionError::new);
+ assertFalse(actualGroupBy.isWithRollup());
+ assertTrue(actualGroupBy.isContainsGroupingExtension());
+ assertThat(actualGroupBy.getGroupByItems().size(), is(2));
+ }
+
+ @Test
+ void assertVisitGroupingSetsItems() {
+ SelectStatement actual = parseSelectStatement(
+ "SELECT status, merchant_id, grouping__id, count(*) FROM t_order GROUP BY status, merchant_id GROUPING SETS ((status, merchant_id), (status))");
+ GroupBySegment actualGroupBy = actual.getGroupBy().orElseThrow(AssertionError::new);
+ assertTrue(actualGroupBy.isContainsGroupingExtension());
+ assertThat(actualGroupBy.getGroupByItems().size(), is(5));
+ }
+
@Test
void assertVisitHiveSystemColumnVariables() {
SelectStatement actual = parseSelectStatement("SELECT `INPUT__FILE__NAME` FROM t_user");
@@ -79,6 +111,24 @@ class HiveStatementVisitorTest {
assertThat(actualWindow.getOrderBySegment().getOrderByItems().size(), is(1));
}
+ @Test
+ void assertVisitAggregationWindowFunction() {
+ SelectStatement actual = parseSelectStatement(
+ "SELECT avg(quantity) OVER (PARTITION BY order_id ORDER BY item_id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS avg_quantity FROM t_order_item");
+ AggregationProjectionSegment actualProjection = (AggregationProjectionSegment) actual.getProjections().getProjections().iterator().next();
+ WindowItemSegment actualWindow = actualProjection.getWindow().get();
+ assertThat(actualWindow.getPartitionListSegments().size(), is(1));
+ assertThat(actualWindow.getOrderBySegment().getOrderByItems().size(), is(1));
+ }
+
+ @Test
+ void assertVisitDistinctAggregationWindowFunction() {
+ SelectStatement actual = parseSelectStatement(
+ "SELECT count(DISTINCT product_id) OVER (PARTITION BY order_id) AS distinct_product_count FROM t_order_item");
+ AggregationProjectionSegment actualProjection = (AggregationProjectionSegment) actual.getProjections().getProjections().iterator().next();
+ assertThat(actualProjection.getWindow().get().getPartitionListSegments().size(), is(1));
+ }
+
@Test
void assertVisitFirstValueWithParameter() {
SelectStatement actual = parseSelectStatement(
diff --git a/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/segment/generic/table/SimpleTableSegment.java b/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/segment/generic/table/SimpleTableSegment.java
index 97aba35d49d..3b6b9ce54ee 100644
--- a/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/segment/generic/table/SimpleTableSegment.java
+++ b/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/segment/generic/table/SimpleTableSegment.java
@@ -51,6 +51,9 @@ public final class SimpleTableSegment implements TableSegment, OwnerAvailable {
@Setter
private ExpressionSegment tableSampleExpression;
+ @Setter
+ private boolean tableSampled;
+
private final Collection<IndexHintSegment> indexHintSegments = new LinkedList<>();
@Setter
diff --git a/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/InsertStatement.java b/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/InsertStatement.java
index d0573be2b58..18adfa4e42a 100644
--- a/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/InsertStatement.java
+++ b/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/InsertStatement.java
@@ -88,6 +88,8 @@ public final class InsertStatement extends DMLStatement {
private final boolean replace;
+ private final boolean overwrite;
+
private final Collection<InsertValuesSegment> values;
private final Collection<ColumnSegment> derivedInsertColumns;
@@ -101,7 +103,7 @@ public final class InsertStatement extends DMLStatement {
final MultiTableInsertType multiTableInsertType, final MultiTableInsertIntoSegment multiTableInsertInto,
final MultiTableConditionalIntoSegment multiTableConditionalInto, final WhereSegment where, final ExecSegment exec,
final WithTableHintSegment withTableHint, final FunctionSegment rowSetFunction, final boolean ignore, final boolean replace,
- final Collection<InsertValuesSegment> values, final Collection<ColumnSegment> derivedInsertColumns) {
+ final boolean overwrite, final Collection<InsertValuesSegment> values, final Collection<ColumnSegment> derivedInsertColumns) {
super(databaseType);
this.table = table;
this.insertColumns = insertColumns;
@@ -121,6 +123,7 @@ public final class InsertStatement extends DMLStatement {
this.rowSetFunction = rowSetFunction;
this.ignore = ignore;
this.replace = replace;
+ this.overwrite = overwrite;
this.values = null == values ? new LinkedList<>() : values;
this.derivedInsertColumns = null == derivedInsertColumns ? new LinkedList<>() : derivedInsertColumns;
attributes = new SQLStatementAttributes(new WithSQLStatementAttribute(with));
diff --git a/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatement.java b/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatement.java
index 685bf241ddd..48ce2670dbb 100644
--- a/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatement.java
+++ b/parser/sql/statement/core/src/main/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatement.java
@@ -22,6 +22,7 @@ import lombok.Getter;
import org.apache.shardingsphere.database.connector.core.type.DatabaseType;
import org.apache.shardingsphere.sql.parser.statement.core.enums.SubqueryType;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.combine.CombineSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.FunctionSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.hint.WithTableHintSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ProjectionsSegment;
import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.outfile.OutfileSegment;
@@ -40,6 +41,8 @@ import org.apache.shardingsphere.sql.parser.statement.core.statement.attribute.S
import org.apache.shardingsphere.sql.parser.statement.core.statement.attribute.type.AllowNotUseDatabaseSQLStatementAttribute;
import org.apache.shardingsphere.sql.parser.statement.core.statement.attribute.type.WithSQLStatementAttribute;
+import java.util.Collection;
+import java.util.LinkedList;
import java.util.Optional;
/**
@@ -82,6 +85,8 @@ public final class SelectStatement extends DMLStatement {
private final WithTableHintSegment withTableHint;
+ private final Collection<FunctionSegment> transformSegments = new LinkedList<>();
+
private final SQLStatementAttributes attributes;
@Builder
@@ -199,9 +204,11 @@ public final class SelectStatement extends DMLStatement {
* @return select statement
*/
public SelectStatement withSubqueryType(final SubqueryType subqueryType) {
- return builder().databaseType(getDatabaseType()).projections(projections).from(from).where(where).hierarchicalQuery(hierarchicalQuery)
+ SelectStatement result = builder().databaseType(getDatabaseType()).projections(projections).from(from).where(where).hierarchicalQuery(hierarchicalQuery)
.groupBy(groupBy).having(having).orderBy(orderBy).combine(combine).with(with).subqueryType(subqueryType).limit(limit).lock(lock).window(window)
.into(into).model(model).outfile(outfile).withTableHint(withTableHint).build();
+ result.getTransformSegments().addAll(transformSegments);
+ return result;
}
/**
diff --git a/parser/sql/statement/core/src/test/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatementTest.java b/parser/sql/statement/core/src/test/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatementTest.java
new file mode 100644
index 00000000000..8021a88bbe6
--- /dev/null
+++ b/parser/sql/statement/core/src/test/java/org/apache/shardingsphere/sql/parser/statement/core/statement/type/dml/SelectStatementTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml;
+
+import org.apache.shardingsphere.database.connector.core.type.DatabaseType;
+import org.apache.shardingsphere.sql.parser.statement.core.enums.SubqueryType;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.FunctionSegment;
+import org.apache.shardingsphere.sql.parser.statement.core.segment.dml.item.ProjectionsSegment;
+import org.junit.jupiter.api.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+class SelectStatementTest {
+
+ @Test
+ void assertWithSubqueryType() {
+ SelectStatement selectStatement = SelectStatement.builder().databaseType(mock(DatabaseType.class)).projections(new ProjectionsSegment(0, 0)).build();
+ FunctionSegment transformSegment = new FunctionSegment(0, 0, "TRANSFORM", "TRANSFORM(user_id)");
+ selectStatement.getTransformSegments().add(transformSegment);
+ SelectStatement actual = selectStatement.withSubqueryType(SubqueryType.TABLE);
+ assertThat(actual.getSubqueryType().orElse(null), is(SubqueryType.TABLE));
+ assertThat(actual.getTransformSegments(), contains(transformSegment));
+ }
+}
diff --git a/test/it/parser/src/main/resources/case/dml/select-hive.xml b/test/it/parser/src/main/resources/case/dml/select-hive.xml
index 374d4c1a8b6..43a3d2d5862 100644
--- a/test/it/parser/src/main/resources/case/dml/select-hive.xml
+++ b/test/it/parser/src/main/resources/case/dml/select-hive.xml
@@ -1104,6 +1104,11 @@
<select sql-case-id="hive_select_window_frame">
<projections start-index="7" stop-index="98">
<aggregation-projection type="SUM" expression="(amount) OVER (PARTITION BY user_id ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)" start-index="7" stop-index="98">
+ <window-item start-index="24" stop-index="98">
+ <order-by start-index="46" stop-index="56">
+ <column-item name="id" order-direction="ASC" start-index="55" stop-index="56" />
+ </order-by>
+ </window-item>
<parameter>
<column name="amount" start-index="11" stop-index="16" />
</parameter>
@@ -1249,7 +1254,7 @@
<from>
<simple-table name="t_order" start-index="39" stop-index="45" />
</from>
- <group-by>
+ <group-by contains-grouping-extension="true">
<column-item name="status" start-index="56" stop-index="61" />
</group-by>
</select>
@@ -1263,9 +1268,12 @@
<from>
<simple-table name="t_order" start-index="42" stop-index="48" />
</from>
- <group-by>
+ <group-by contains-grouping-extension="true">
<column-item name="status" start-index="59" stop-index="64" />
<column-item name="merchant_id" start-index="67" stop-index="77" />
+ <column-item name="status" start-index="95" stop-index="100" />
+ <column-item name="merchant_id" start-index="103" stop-index="113" />
+ <column-item name="status" start-index="118" stop-index="123" />
</group-by>
</select>