You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2024/11/20 18:24:00 UTC
(beam) branch master updated: Bump zetasql version to 2024.11.1 (#32902)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 5d3088f7489 Bump zetasql version to 2024.11.1 (#32902)
5d3088f7489 is described below
commit 5d3088f74894538cf35e86c18727379d5e40fc78
Author: Yi Hu <ya...@google.com>
AuthorDate: Wed Nov 20 13:23:52 2024 -0500
Bump zetasql version to 2024.11.1 (#32902)
---
sdks/java/extensions/sql/zetasql/build.gradle | 2 +-
.../sql/zetasql/translation/AggregateScanConverter.java | 7 ++++---
.../sql/zetasql/translation/ExpressionConverter.java | 13 +++++++++++--
3 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/sdks/java/extensions/sql/zetasql/build.gradle b/sdks/java/extensions/sql/zetasql/build.gradle
index 8d6e2aac0bf..29a3f95402b 100644
--- a/sdks/java/extensions/sql/zetasql/build.gradle
+++ b/sdks/java/extensions/sql/zetasql/build.gradle
@@ -27,7 +27,7 @@ applyJavaNature(
description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL :: ZetaSQL"
ext.summary = "ZetaSQL to Calcite translator"
-def zetasql_version = "2022.04.1"
+def zetasql_version = "2024.11.1"
dependencies {
// TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency
diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java
index 2dfc7fe372f..412cd46001f 100644
--- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java
+++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java
@@ -29,6 +29,7 @@ import com.google.zetasql.resolvedast.ResolvedNode;
import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedAggregateFunctionCall;
import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedAggregateScan;
import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedComputedColumn;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedComputedColumnBase;
import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedExpr;
import java.util.ArrayList;
import java.util.Arrays;
@@ -94,7 +95,7 @@ class AggregateScanConverter extends RelConverter<ResolvedAggregateScan> {
aggregateCalls = new ArrayList<>();
// For aggregate calls, their input ref follow after GROUP BY input ref.
int columnRefoff = groupFieldsListSize;
- for (ResolvedComputedColumn computedColumn : zetaNode.getAggregateList()) {
+ for (ResolvedComputedColumnBase computedColumn : zetaNode.getAggregateList()) {
AggregateCall aggCall =
convertAggCall(computedColumn, columnRefoff, groupSet.size(), input);
aggregateCalls.add(aggCall);
@@ -144,7 +145,7 @@ class AggregateScanConverter extends RelConverter<ResolvedAggregateScan> {
// LogicalProject should also include columns used by aggregate functions. These columns should
// follow after GROUP BY columns.
// TODO: remove duplicate columns in projects.
- for (ResolvedComputedColumn resolvedComputedColumn : node.getAggregateList()) {
+ for (ResolvedComputedColumnBase resolvedComputedColumn : node.getAggregateList()) {
// Should create Calcite's RexInputRef from ResolvedColumn from ResolvedComputedColumn.
// TODO: handle aggregate function with more than one argument and handle OVER
// TODO: is there is general way for column reference tracking and deduplication for
@@ -180,7 +181,7 @@ class AggregateScanConverter extends RelConverter<ResolvedAggregateScan> {
}
private AggregateCall convertAggCall(
- ResolvedComputedColumn computedColumn, int columnRefOff, int groupCount, RelNode input) {
+ ResolvedComputedColumnBase computedColumn, int columnRefOff, int groupCount, RelNode input) {
ResolvedAggregateFunctionCall aggregateFunctionCall =
(ResolvedAggregateFunctionCall) computedColumn.getExpr();
diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java
index 14db554d6f0..0f32451504b 100644
--- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java
+++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java
@@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.zetasql.TVFRelation;
+import com.google.zetasql.TVFRelation.Column;
import com.google.zetasql.TableValuedFunction;
import com.google.zetasql.TableValuedFunction.FixedOutputSchemaTVF;
import com.google.zetasql.Type;
@@ -65,6 +66,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import org.apache.beam.repackaged.core.org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.extensions.sql.impl.QueryPlanner.QueryParameters;
import org.apache.beam.sdk.extensions.sql.impl.ZetaSqlUserDefinedSQLNativeTableValuedFunction;
@@ -495,9 +497,16 @@ public class ExpressionConverter {
new ZetaSqlUserDefinedSQLNativeTableValuedFunction(
new SqlIdentifier(tvf.getName(), SqlParserPos.ZERO),
opBinding -> {
+ TVFRelation rel = fixedOutputSchemaTVF.getOutputSchema();
+ // TODO(yathu) revert this workaround when ZetaSQL adds back this API.
+ List<Column> cols;
+ try {
+ cols = (List<Column>) FieldUtils.readField(rel, "columns", true);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
List<RelDataTypeField> relDataTypeFields =
- convertTVFRelationColumnsToRelDataTypeFields(
- fixedOutputSchemaTVF.getOutputSchema().getColumns());
+ convertTVFRelationColumnsToRelDataTypeFields(cols);
return new RelRecordType(relDataTypeFields);
},
null,