You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2019/07/31 23:01:00 UTC
[beam] 01/01: [BEAM-7832] Add ZetaSQL as a dialect in BeamSQL.
This is an automated email from the ASF dual-hosted git repository.
amaliujia pushed a commit to branch rw_beam_zetasql
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 3c367003b38da7050a3078734123a3de54fcc07e
Author: amaliujia <am...@gmail.com>
AuthorDate: Wed Jul 31 16:00:03 2019 -0700
[BEAM-7832] Add ZetaSQL as a dialect in BeamSQL.
---
build.gradle | 1 +
.../src/main/resources/beam/suppressions.xml | 4 +
sdks/java/extensions/sql/build.gradle | 33 +
.../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 7 +
.../extensions/sql/zetasql/BeamBuiltinMethods.java | 71 +
.../extensions/sql/zetasql/BeamCodegenUtils.java | 37 +
.../sdk/extensions/sql/zetasql/DateFunctions.java | 38 +
.../sdk/extensions/sql/zetasql/DateTimeUtils.java | 278 ++
.../sdk/extensions/sql/zetasql/QueryTrait.java | 76 +
.../sdk/extensions/sql/zetasql/SqlAnalyzer.java | 278 ++
.../zetasql/SqlCaseWithValueOperatorRewriter.java | 77 +
.../sql/zetasql/SqlCoalesceOperatorRewriter.java | 67 +
.../SqlExtractTimestampOperatorRewriter.java | 47 +
.../sql/zetasql/SqlIfNullOperatorRewriter.java | 51 +
.../sql/zetasql/SqlNullIfOperatorRewriter.java | 53 +
.../sql/zetasql/SqlOperatorRewriter.java | 35 +
.../sdk/extensions/sql/zetasql/SqlOperators.java | 197 +
.../sql/zetasql/SqlStdOperatorMappingTable.java | 359 ++
.../extensions/sql/zetasql/StringFunctions.java | 182 +
.../extensions/sql/zetasql/TableResolution.java | 91 +
.../sql/zetasql/TableResolutionContext.java | 73 +
.../sdk/extensions/sql/zetasql/TableResolver.java | 36 +
.../extensions/sql/zetasql/TableResolverImpl.java | 61 +
.../beam/sdk/extensions/sql/zetasql/TestInput.java | 255 ++
.../extensions/sql/zetasql/TimestampFunctions.java | 48 +
.../beam/sdk/extensions/sql/zetasql/TypeUtils.java | 166 +
.../sql/zetasql/ZetaSQLCastFunctionImpl.java | 116 +
.../extensions/sql/zetasql/ZetaSQLPlannerImpl.java | 185 +
.../sql/zetasql/ZetaSQLQueryPlanner.java | 143 +
.../sdk/extensions/sql/zetasql/ZetaSqlIdUtils.java | 84 +
.../sdk/extensions/sql/zetasql/package-info.java | 2 +
.../translation/AggregateScanConverter.java | 230 ++
.../translation/ArrayScanToJoinConverter.java | 101 +
.../translation/ArrayScanToUncollectConverter.java | 62 +
.../sql/zetasql/translation/ConversionContext.java | 65 +
.../zetasql/translation/ExpressionConverter.java | 1019 ++++++
.../zetasql/translation/FilterScanConverter.java | 52 +
.../sql/zetasql/translation/JoinScanConverter.java | 101 +
.../translation/JoinScanWithRefConverter.java | 95 +
.../LimitOffsetScanToLimitConverter.java | 63 +
.../LimitOffsetScanToOrderByLimitConverter.java | 101 +
.../OrderByScanUnsupportedConverter.java | 39 +
.../zetasql/translation/ProjectScanConverter.java | 49 +
.../translation/QueryStatementConverter.java | 114 +
.../sql/zetasql/translation/RelConverter.java | 74 +
.../translation/SetOperationScanConverter.java | 114 +
.../translation/SingleRowScanConverter.java | 41 +
.../zetasql/translation/TableScanConverter.java | 131 +
.../zetasql/translation/WithRefScanConverter.java | 53 +
.../sql/zetasql/translation/WithScanConverter.java | 55 +
.../sql/zetasql/translation/package-info.java | 2 +
.../JoinCompoundIdentifiersTestZetaSQL.java | 342 ++
.../sql/zetasql/ZetaSQLDialectSpecTestZetaSQL.java | 3800 ++++++++++++++++++++
53 files changed, 9854 insertions(+)
diff --git a/build.gradle b/build.gradle
index 24507fa..f23c680 100644
--- a/build.gradle
+++ b/build.gradle
@@ -157,6 +157,7 @@ task javaPostCommit() {
task sqlPostCommit() {
dependsOn ":sdks:java:extensions:sql:postCommit"
+ dependsOn ":sdks:java:extensions:sql:runZetaSQLTest"
dependsOn ":sdks:java:extensions:sql:jdbc:postCommit"
}
diff --git a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml
index a6b25a0..203d92b 100644
--- a/sdks/java/build-tools/src/main/resources/beam/suppressions.xml
+++ b/sdks/java/build-tools/src/main/resources/beam/suppressions.xml
@@ -80,9 +80,13 @@
<suppress id="ForbidNonVendoredGuava" files=".*kinesis.*KinesisIO\.java" />
<suppress id="ForbidNonVendoredGuava" files=".*kinesis.*KinesisProducerMock\.java" />
<suppress id="ForbidNonVendoredGuava" files=".*bigtable.*VendoredListenableFutureAdapter\.java" />
+ <suppress id="ForbidNonVendoredGuava" files=".*bigtable.*VendoredListenableFutureAdapter\.java" />
<suppress id="ForbidNonVendoredGuava" files=".*bigtable.*BigtableServiceImplTest\.java" />
<suppress id="ForbidNonVendoredGuava" files=".*sql.*BeamValuesRel\.java" />
<suppress id="ForbidNonVendoredGuava" files=".*sql.*BeamEnumerableConverterTest\.java" />
+ <suppress id="ForbidNonVendoredGuava" files=".*zetasql.*TableScanConverter\.java" />
+ <suppress id="ForbidNonVendoredGuava" files=".*zetasql.*ExpressionConverter\.java" />
+ <suppress id="ForbidNonVendoredGuava" files=".*zetasql.*ZetaSQLPlannerImpl\.java" />
<!-- Flink -->
<!-- Checkstyle does not correctly detect package files across multiple source directories. -->
diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle
index 0bbf0aa..5bcbf86 100644
--- a/sdks/java/extensions/sql/build.gradle
+++ b/sdks/java/extensions/sql/build.gradle
@@ -35,6 +35,8 @@ applyJavaNature(
include(dependency("org.apache.calcite:.*"))
include(dependency("org.apache.calcite.avatica:.*"))
include(dependency("org.codehaus.janino:.*"))
+ include(dependency("com.google.api.grpc:.*"))
+ include(dependency("com.google.zetasql:.*"))
}
// guava uses the com.google.common and com.google.thirdparty package namespaces
relocate("com.google.common", project.getJavaRelocatedPath("com.google.common")) {
@@ -44,9 +46,17 @@ applyJavaNature(
exclude "com.google.common.testing.**"
exclude "com.google.common.util.concurrent.testing.**"
}
+ relocate "com.google.api", getJavaRelocatedPath("com.google.api")
+ relocate "com.google.cloud", getJavaRelocatedPath("com.google.cloud")
+ relocate "com.google.logging", getJavaRelocatedPath("com.google.logging")
+ relocate "com.google.longrunning", getJavaRelocatedPath("com.google.longrunning")
+ relocate "com.google.rpc", getJavaRelocatedPath("com.google.rpc")
+ relocate "com.google.type", getJavaRelocatedPath("com.google.type")
+
relocate "com.google.thirdparty", project.getJavaRelocatedPath("com.google.thirdparty")
relocate "com.google.protobuf", getJavaRelocatedPath("com.google.protobuf")
+ relocate "com.google.zetasql", getJavaRelocatedPath("com.google.zetasql")
relocate "org.apache.calcite", getJavaRelocatedPath("org.apache.calcite")
// Looking up the compiler factory in Calcite depends on having a properties
@@ -82,6 +92,10 @@ dependencies {
compile "org.apache.calcite:calcite-core:$calcite_version"
compile "org.apache.calcite:calcite-linq4j:$calcite_version"
compile "org.apache.calcite.avatica:avatica-core:$avatica_version"
+ compile "com.google.api.grpc:proto-google-common-protos:1.12.0" // Interfaces with ZetaSQL use this
+ compile "com.google.zetasql:jni-channel:2019.07.1"
+ compile "com.google.zetasql:client:2019.07.1"
+ compile "com.google.zetasql:types:2019.07.1"
shadow project(path: ":sdks:java:core", configuration: "shadow")
shadow project(":sdks:java:extensions:join-library")
shadow library.java.slf4j_api
@@ -91,6 +105,8 @@ dependencies {
shadow library.java.jackson_databind
shadow library.java.jackson_dataformat_yaml
shadow library.java.joda_time
+ shadow library.java.protobuf_java
+ shadow library.java.protobuf_java_util
shadow "com.alibaba:fastjson:1.2.49"
shadow "com.jayway.jsonpath:json-path:2.4.0"
shadow project(path: ":runners:direct-java", configuration: "shadow")
@@ -108,6 +124,10 @@ dependencies {
permitUnusedDeclared "com.jayway.jsonpath:json-path:2.4.0"
permitUnusedDeclared library.java.jackson_dataformat_yaml
+ permitUnusedDeclared "com.google.api.grpc:proto-google-common-protos:1.12.0"
+ permitUnusedDeclared "com.google.zetasql:jni-channel:2019.07.1"
+ permitUnusedDeclared "com.google.protobuf:protobuf-java-util:3.6.0"
+
// Dependencies that are bundled in when we bundle Calcite
permitUsedUndeclared "org.codehaus.janino:janino:3.0.11"
permitUsedUndeclared "org.codehaus.janino:commons-compiler:3.0.11"
@@ -184,6 +204,19 @@ task runPojoExample(type: JavaExec) {
args = ["--runner=DirectRunner"]
}
+task runZetaSQLTest(type: Test) {
+ // Disable Gradle cache (it should not be used because the IT's won't run).
+ outputs.upToDateWhen { false }
+
+ include '**/*ZetaSQL.class'
+ classpath = project(":sdks:java:extensions:sql")
+ .sourceSets
+ .test
+ .runtimeClasspath
+ testClassesDirs = files(project(":sdks:java:extensions:sql").sourceSets.test.output.classesDirs)
+ useJUnit { }
+}
+
task integrationTest(type: Test) {
group = "Verification"
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
index bcfeaf7..acea7dd 100644
--- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/BeamSqlEnv.java
@@ -39,6 +39,7 @@ import org.apache.beam.sdk.extensions.sql.meta.provider.ReadOnlyTableProvider;
import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
import org.apache.beam.sdk.extensions.sql.meta.provider.UdfUdafProvider;
import org.apache.beam.sdk.extensions.sql.meta.store.InMemoryMetaStore;
+import org.apache.beam.sdk.extensions.sql.zetasql.ZetaSQLQueryPlanner;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Combine.CombineFn;
@@ -132,6 +133,8 @@ public class BeamSqlEnv {
public static class BeamSqlEnvBuilder {
private static final String CALCITE_PLANNER =
"org.apache.beam.sdk.extensions.sql.impl.CalciteQueryPlanner";
+ private static final String ZETASQL_PLANNER =
+ "org.apache.beam.sdk.extensions.sql.zetasql.ZetaSQLQueryPlanner";
private String queryPlannerClassName;
private TableProvider defaultTableProvider;
private String currentSchemaName;
@@ -309,6 +312,10 @@ public class BeamSqlEnv {
return new CalciteQueryPlanner(jdbcConnection, ruleSets);
}
+ if (queryPlannerClassName.equals(ZETASQL_PLANNER)) {
+ return new ZetaSQLQueryPlanner(jdbcConnection, ruleSets);
+ }
+
try {
return (QueryPlanner)
Class.forName(queryPlannerClassName)
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamBuiltinMethods.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamBuiltinMethods.java
new file mode 100644
index 0000000..9d16948
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamBuiltinMethods.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.lang.reflect.Method;
+import org.apache.calcite.linq4j.tree.Types;
+
+/** BeamBuiltinMethods. */
+public class BeamBuiltinMethods {
+ public static final Method STARTS_WITH_METHOD =
+ Types.lookupMethod(StringFunctions.class, "startsWith", String.class, String.class);
+
+ public static final Method ENDS_WITH_METHOD =
+ Types.lookupMethod(StringFunctions.class, "endsWith", String.class, String.class);
+
+ public static final Method LIKE_METHOD =
+ Types.lookupMethod(StringFunctions.class, "like", String.class, String.class);
+
+ public static final Method CONCAT_METHOD =
+ Types.lookupMethod(
+ StringFunctions.class,
+ "concat",
+ String.class,
+ String.class,
+ String.class,
+ String.class,
+ String.class);
+
+ public static final Method REPLACE_METHOD =
+ Types.lookupMethod(
+ StringFunctions.class, "replace", String.class, String.class, String.class);
+
+ public static final Method TRIM_METHOD =
+ Types.lookupMethod(StringFunctions.class, "trim", String.class, String.class);
+
+ public static final Method LTRIM_METHOD =
+ Types.lookupMethod(StringFunctions.class, "ltrim", String.class, String.class);
+
+ public static final Method RTRIM_METHOD =
+ Types.lookupMethod(StringFunctions.class, "rtrim", String.class, String.class);
+
+ public static final Method SUBSTR_METHOD =
+ Types.lookupMethod(StringFunctions.class, "substr", String.class, long.class, long.class);
+
+ public static final Method REVERSE_METHOD =
+ Types.lookupMethod(StringFunctions.class, "reverse", String.class);
+
+ public static final Method CHAR_LENGTH_METHOD =
+ Types.lookupMethod(StringFunctions.class, "charLength", String.class);
+
+ public static final Method TIMESTAMP_METHOD =
+ Types.lookupMethod(TimestampFunctions.class, "timestamp", String.class, String.class);
+
+ public static final Method DATE_METHOD =
+ Types.lookupMethod(DateFunctions.class, "date", Integer.class, Integer.class, Integer.class);
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamCodegenUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamCodegenUtils.java
new file mode 100644
index 0000000..bc2229e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamCodegenUtils.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.io.UnsupportedEncodingException;
+import org.joda.time.DateTime;
+
+/** BeamCodegenUtils. */
+public class BeamCodegenUtils {
+ // convert bytes to String in UTF8 encoding.
+ public static String toStringUTF8(byte[] bytes) {
+ try {
+ return new String(bytes, "UTF8");
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static String toStringTimestamp(long timestamp) {
+ return DateTimeUtils.formatTimestampWithTimeZone(new DateTime(timestamp));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateFunctions.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateFunctions.java
new file mode 100644
index 0000000..c75d033
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateFunctions.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.TimeZone;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/** DateFunctions. */
+public class DateFunctions {
+ public DateTime date(Integer year, Integer month, Integer day) {
+ return DateTimeUtils.parseDate(
+ String.join("-", year.toString(), month.toString(), day.toString()));
+ }
+
+ public DateTime date(DateTime ts) {
+ return date(ts, "UTC");
+ }
+
+ public DateTime date(DateTime ts, String timezone) {
+ return ts.withZoneRetainFields(DateTimeZone.forTimeZone(TimeZone.getTimeZone(timezone)));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateTimeUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateTimeUtils.java
new file mode 100644
index 0000000..c992d8f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/DateTimeUtils.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static com.google.zetasql.CivilTimeEncoder.decodePacked64TimeNanos;
+import static com.google.zetasql.CivilTimeEncoder.encodePacked64TimeNanos;
+
+import com.google.zetasql.Value;
+import io.grpc.Status;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Splitter;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.math.LongMath;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.util.DateString;
+import org.apache.calcite.util.TimeString;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.LocalTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+/** DateTimeUtils. */
+public class DateTimeUtils {
+ public static final Long MILLIS_PER_DAY = 86400000L;
+ private static final Long MICROS_PER_MILLI = 1000L;
+
+ @SuppressWarnings("unchecked")
+ private enum TimestampPatterns {
+ TIMESTAMP_PATTERN,
+ TIMESTAMP_PATTERN_SUBSECOND,
+ TIMESTAMP_PATTERN_T,
+ TIMESTAMP_PATTERN_SUBSECOND_T,
+ }
+
+ @SuppressWarnings("unchecked")
+ private static final ImmutableMap<Enum, DateTimeFormatter> TIMESTAMP_PATTERN_WITHOUT_TZ =
+ ImmutableMap.of(
+ TimestampPatterns.TIMESTAMP_PATTERN, DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"),
+ TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND,
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"),
+ TimestampPatterns.TIMESTAMP_PATTERN_T, DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss"),
+ TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND_T,
+ DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS"));
+
+ @SuppressWarnings("unchecked")
+ private static final ImmutableMap<Enum, DateTimeFormatter> TIMESTAMP_PATTERN_WITH_TZ =
+ ImmutableMap.of(
+ TimestampPatterns.TIMESTAMP_PATTERN, DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ssZZ"),
+ TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND,
+ DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZZ"),
+ TimestampPatterns.TIMESTAMP_PATTERN_T,
+ DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ssZZ"),
+ TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND_T,
+ DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZZ"));
+
+ public static DateTimeFormatter findDateTimePattern(String str) {
+ if (str.indexOf('+') == -1) {
+ return findDateTimePattern(str, TIMESTAMP_PATTERN_WITHOUT_TZ);
+ } else {
+ return findDateTimePattern(str, TIMESTAMP_PATTERN_WITH_TZ);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static DateTimeFormatter findDateTimePattern(
+ String str, ImmutableMap<Enum, DateTimeFormatter> patternMap) {
+ if (str.indexOf('.') == -1) {
+ if (str.indexOf('T') == -1) {
+ return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN);
+ } else {
+ return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_T);
+ }
+ } else {
+ if (str.indexOf('T') == -1) {
+ return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND);
+ } else {
+ return patternMap.get(TimestampPatterns.TIMESTAMP_PATTERN_SUBSECOND_T);
+ }
+ }
+ }
+
+ // https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#timestamp_differences
+ // 0001-01-01 00:00:00 to 9999-12-31 23:59:59.999999 UTC.
+ // -62135596800000000 to 253402300799999999
+ @SuppressWarnings("GoodTime")
+ public static final Long MIN_UNIX_MILLIS = -62135596800000L;
+
+ @SuppressWarnings("GoodTime")
+ public static final Long MAX_UNIX_MILLIS = 253402300799999L;
+
+ public static DateTime parseTimestampWithUTCTimeZone(String str) {
+ return findDateTimePattern(str).withZoneUTC().parseDateTime(str);
+ }
+
+ @SuppressWarnings("unused")
+ public static DateTime parseTimestampWithLocalTimeZone(String str) {
+ return findDateTimePattern(str).withZone(DateTimeZone.getDefault()).parseDateTime(str);
+ }
+
+ public static DateTime parseTimestampWithTimeZone(String str) {
+ // for example, accept "1990-10-20 13:24:01+0730"
+ if (str.indexOf('.') == -1) {
+ return DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ssZ").parseDateTime(str);
+ } else {
+ return DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSSZ").parseDateTime(str);
+ }
+ }
+
+ public static String formatTimestampWithTimeZone(DateTime dt) {
+ String resultWithoutZone;
+ if (dt.getMillisOfSecond() == 0) {
+ resultWithoutZone = dt.toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"));
+ } else {
+ resultWithoutZone = dt.toString(DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"));
+ }
+
+ // ZetaSQL expects a 2-digit timezone offset (-05) if the minute part is zero, and it expects
+ // a 4-digit timezone with a colon (-07:52) if the minute part is non-zero. None of the
+ // variations on z,Z,ZZ,.. do this for us so we have to do it manually here.
+ String zone = dt.toString(DateTimeFormat.forPattern("ZZ"));
+ List<String> zoneParts = Lists.newArrayList(Splitter.on(':').limit(2).split(zone));
+ if (zoneParts.size() == 2 && zoneParts.get(1).equals("00")) {
+ zone = zoneParts.get(0);
+ }
+
+ return resultWithoutZone + zone;
+ }
+
+ @SuppressWarnings("unused")
+ public static DateTime parseTimestampWithoutTimeZone(String str) {
+ return DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").parseDateTime(str);
+ }
+
+ public static DateTime parseDate(String str) {
+ return DateTimeFormat.forPattern("yyyy-MM-dd").withZoneUTC().parseDateTime(str);
+ }
+
+ public static DateTime parseTime(String str) {
+ // DateTimeFormat does not parse "08:10:10" for pattern "HH:mm:ss.SSS". In this case, '.' must
+ // appear.
+ if (str.indexOf('.') == -1) {
+ return DateTimeFormat.forPattern("HH:mm:ss").withZoneUTC().parseDateTime(str);
+ } else {
+ return DateTimeFormat.forPattern("HH:mm:ss.SSS").withZoneUTC().parseDateTime(str);
+ }
+ }
+
+ @SuppressWarnings(
+ "Value with nanoseconds will be truncated to milliseconds in decodePacked64TimeNanos.")
+ public static TimeString convertTimeValueToTimeString(Value value) {
+ LocalTime localTime = decodePacked64TimeNanos(value.getTimeValue());
+ return TimeString.fromMillisOfDay(localTime.getMillisOfDay());
+ }
+
+ // dates are represented as an int32 value, indicating the offset
+ // in days from the epoch 1970-01-01. ZetaSQL dates are not timezone aware,
+ // and do not correspond to any particular 24 hour period.
+ public static DateString convertDateValueToDateString(Value value) {
+ return DateString.fromDaysSinceEpoch(value.getDateValue());
+ }
+
+ public static Value parseDateToValue(String dateString) {
+ DateTime dateTime = parseDate(dateString);
+ return Value.createDateValue((int) (dateTime.getMillis() / MILLIS_PER_DAY));
+ }
+
+ public static Value parseTimeToValue(String timeString) {
+ DateTime dateTime = parseTime(timeString);
+ return Value.createTimeValue(
+ encodePacked64TimeNanos(LocalTime.fromMillisOfDay(dateTime.getMillisOfDay())));
+ }
+
+ public static Value parseTimestampWithTZToValue(String timestampString) {
+ DateTime dateTime = parseTimestampWithTimeZone(timestampString);
+ // convert from micros.
+ // TODO: how to handle overflow.
+ return Value.createTimestampValueFromUnixMicros(
+ LongMath.checkedMultiply(dateTime.getMillis(), MICROS_PER_MILLI));
+ }
+
+ private static void safeCheckSubMillisPrecision(long micros) {
+ long subMilliPrecision = micros % 1000L;
+ if (subMilliPrecision != 0) {
+ throw new IllegalArgumentException(
+ String.format(
+ "%s has sub-millisecond precision, which Beam ZetaSQL does"
+ + " not currently support.",
+ micros));
+ }
+ }
+
+ @SuppressWarnings("GoodTime")
+ public static long safeMicrosToMillis(long micros) {
+ safeCheckSubMillisPrecision(micros);
+ return micros / 1000L;
+ }
+
+ /**
+ * This function validates that Long representation of timestamp is compatible with ZetaSQL
+ * timestamp values range.
+ *
+ * <p>Invoked via reflection. @see SqlOperators
+ *
+ * @param ts Timestamp to validate.
+ * @return Unchanged timestamp sent for validation.
+ */
+ @SuppressWarnings("GoodTime")
+ public static Long validateTimestamp(Long ts) {
+ if (ts == null) {
+ return null;
+ }
+
+ if ((ts < MIN_UNIX_MILLIS) || (ts > MAX_UNIX_MILLIS)) {
+ throw Status.OUT_OF_RANGE
+ .withDescription("Timestamp is out of valid range.")
+ .asRuntimeException();
+ }
+
+ return ts;
+ }
+
+ /**
+ * This function validates that interval is compatible with ZetaSQL timestamp values range.
+ *
+ * <p>ZetaSQL validates that if we represent interval in milliseconds, it will fit into Long.
+ *
+ * <p>In case of SECOND or smaller time unit, it converts timestamp to microseconds, so we need to
+ * convert those to microsecond and verify that we do not cause overflow.
+ *
+ * <p>Invoked via reflection. @see SqlOperators
+ *
+ * @param arg Argument for the interval.
+ * @param unit Time unit used in this interval.
+ * @return Argument for the interval.
+ */
+ @SuppressWarnings("GoodTime")
+ public static Long validateTimeInterval(Long arg, TimeUnit unit) {
+ if (arg == null) {
+ return null;
+ }
+
+ // multiplier to convert to milli or microseconds.
+ long multiplier = unit.multiplier.longValue();
+ switch (unit) {
+ case SECOND:
+ case MILLISECOND:
+ multiplier *= 1000L; // Change multiplier from milliseconds to microseconds.
+ break;
+ default:
+ break;
+ }
+
+ if ((arg > Long.MAX_VALUE / multiplier) || (arg < Long.MIN_VALUE / multiplier)) {
+ throw Status.OUT_OF_RANGE
+ .withDescription("Interval is out of valid range")
+ .asRuntimeException();
+ }
+
+ return arg;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/QueryTrait.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/QueryTrait.java
new file mode 100644
index 0000000..428c360
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/QueryTrait.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
+
+import com.google.zetasql.Table;
+import com.google.zetasql.resolvedast.ResolvedColumn;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedOutputColumn;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedWithEntry;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.beam.sdk.extensions.sql.zetasql.TableResolution.SimpleTableWithPath;
+
+/** QueryTrait. */
+public class QueryTrait {
+ public Map<String, ResolvedWithEntry> withEntries = new HashMap<>();
+
+ public Map<ResolvedColumn, String> outputColumnMap = new HashMap<>();
+
+ public Map<Long, SimpleTableWithPath> resolvedTables = new HashMap<>();
+
+ // TODO: move query parameter map to QueryTrait.
+
+ public void addOutputColumnList(List<ResolvedOutputColumn> outputColumnList) {
+ outputColumnList.forEach(
+ column -> {
+ outputColumnMap.put(column.getColumn(), column.getName());
+ });
+ }
+
+ /** Store a table together with its full path for repeated resolutions. */
+ public void addResolvedTable(SimpleTableWithPath tableWithPath) {
+ // table ids are autoincremted in SimpleTable
+ resolvedTables.put(tableWithPath.getTable().getId(), tableWithPath);
+ }
+
+ /** True if the table was resolved using the Calcite schema. */
+ public boolean isTableResolved(Table table) {
+ return resolvedTables.containsKey(table.getId());
+ }
+
+ /** Returns a full table path (exlucding top-level schema) for a given ZetaSQL Table. */
+ public List<String> getTablePath(Table table) {
+ checkArgument(
+ isTableResolved(table),
+ "Attempting to get a path of an unresolved table. Resolve and add the table first: %s",
+ table.getFullName());
+ return resolvedTables.get(table.getId()).getPath();
+ }
+
+ public List<String> retrieveFieldNames(List<ResolvedColumn> resolvedColumnList) {
+ return resolvedColumnList.stream().map(this::resolveAlias).collect(Collectors.toList());
+ }
+
+ public String resolveAlias(ResolvedColumn resolvedColumn) {
+ return this.outputColumnMap.getOrDefault(resolvedColumn, resolvedColumn.getName());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java
new file mode 100644
index 0000000..d3b1ed3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_CREATE_FUNCTION_STMT;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_QUERY_STMT;
+import static org.apache.beam.sdk.extensions.sql.zetasql.SqlStdOperatorMappingTable.ZETASQL_BUILTIN_FUNCTION_WHITELIST;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TypeUtils.toZetaType;
+
+import com.google.zetasql.Analyzer;
+import com.google.zetasql.AnalyzerOptions;
+import com.google.zetasql.Function;
+import com.google.zetasql.SimpleCatalog;
+import com.google.zetasql.Value;
+import com.google.zetasql.ZetaSQLBuiltinFunctionOptions;
+import com.google.zetasql.ZetaSQLFunctions.FunctionEnums.Mode;
+import com.google.zetasql.ZetaSQLOptions.ErrorMessageMode;
+import com.google.zetasql.ZetaSQLOptions.LanguageFeature;
+import com.google.zetasql.ZetaSQLOptions.ProductMode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedCreateFunctionStmt;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedStatement;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.beam.sdk.extensions.sql.zetasql.TableResolution.SimpleTableWithPath;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.plan.Context;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.schema.SchemaPlus;
+
+/** Adapter for {@link Analyzer} to simplify the API for parsing the query and resolving the AST. */
+class SqlAnalyzer {
+ static final String PRE_DEFINED_WINDOW_FUNCTIONS = "pre_defined_window_functions";
+
+ private static final ImmutableList<String> FUNCTION_LIST =
+ ImmutableList.of(
+ // TODO: support optional function argument (for window_offset).
+ "CREATE FUNCTION TUMBLE(ts TIMESTAMP, window_size STRING) AS (1);",
+ "CREATE FUNCTION TUMBLE_START(window_size STRING) AS (1);",
+ "CREATE FUNCTION TUMBLE_END(window_size STRING) AS (1);",
+ "CREATE FUNCTION HOP(ts TIMESTAMP, emit_frequency STRING, window_size STRING) AS (1);",
+ "CREATE FUNCTION HOP_START(emit_frequency STRING, window_size STRING) AS (1);",
+ "CREATE FUNCTION HOP_END(emit_frequency STRING, window_size STRING) AS (1);",
+ "CREATE FUNCTION SESSION(ts TIMESTAMP, session_gap STRING) AS (1);",
+ "CREATE FUNCTION SESSION_START(session_gap STRING) AS (1);",
+ "CREATE FUNCTION SESSION_END(session_gap STRING) AS (1);");
+
+ private final Builder builder;
+
+ private SqlAnalyzer(Builder builder) {
+ this.builder = builder;
+ }
+
+ /** Static factory method to create the builder with query parameters. */
+ static Builder withQueryParams(Map<String, Value> params) {
+ return new Builder().withQueryParams(ImmutableMap.copyOf(params));
+ }
+
+ /**
+ * Accepts the SQL string, returns the resolved AST.
+ *
+ * <p>Initializes query parameters, populates the catalog based on Calcite's schema and table name
+ * resolution strategy set in the context.
+ */
+ ResolvedStatement analyze(String sql) {
+ AnalyzerOptions options = initAnalyzerOptions(builder.queryParams);
+ List<List<String>> tables = Analyzer.extractTableNamesFromStatement(sql);
+ SimpleCatalog catalog =
+ createPopulatedCatalog(builder.topLevelSchema.getName(), options, tables);
+
+ return Analyzer.analyzeStatement(sql, options, catalog);
+ }
+
+ private AnalyzerOptions initAnalyzerOptions(Map<String, Value> queryParams) {
+ AnalyzerOptions options = new AnalyzerOptions();
+ options.setErrorMessageMode(ErrorMessageMode.ERROR_MESSAGE_MULTI_LINE_WITH_CARET);
+ // +00:00 UTC offset
+ options.setDefaultTimezone("UTC");
+ options.getLanguageOptions().setProductMode(ProductMode.PRODUCT_EXTERNAL);
+ options
+ .getLanguageOptions()
+ .setEnabledLanguageFeatures(
+ new HashSet<>(
+ Arrays.asList(
+ LanguageFeature.FEATURE_DISALLOW_GROUP_BY_FLOAT,
+ LanguageFeature.FEATURE_V_1_2_CIVIL_TIME,
+ LanguageFeature.FEATURE_V_1_1_SELECT_STAR_EXCEPT_REPLACE)));
+
+ options
+ .getLanguageOptions()
+ .setSupportedStatementKinds(
+ ImmutableSet.of(RESOLVED_QUERY_STMT, RESOLVED_CREATE_FUNCTION_STMT));
+
+ for (Map.Entry<String, Value> entry : queryParams.entrySet()) {
+ options.addQueryParameter(entry.getKey(), entry.getValue().getType());
+ }
+
+ return options;
+ }
+
+ /**
+ * Creates a SimpleCatalog which represents the top-level schema, populates it with tables,
+ * built-in functions.
+ */
+ private SimpleCatalog createPopulatedCatalog(
+ String catalogName, AnalyzerOptions options, List<List<String>> tables) {
+
+ SimpleCatalog catalog = new SimpleCatalog(catalogName);
+ addBuiltinFunctionsToCatalog(catalog, options);
+
+ tables.forEach(table -> addTableToLeafCatalog(builder.queryTrait, catalog, table));
+
+ return catalog;
+ }
+
+ private void addBuiltinFunctionsToCatalog(SimpleCatalog catalog, AnalyzerOptions options) {
+
+ // Enable ZetaSQL builtin functions.
+ ZetaSQLBuiltinFunctionOptions zetasqlBuiltinFunctionOptions =
+ new ZetaSQLBuiltinFunctionOptions(options.getLanguageOptions());
+
+ ZETASQL_BUILTIN_FUNCTION_WHITELIST.forEach(
+ zetasqlBuiltinFunctionOptions::includeFunctionSignatureId);
+
+ catalog.addZetaSQLFunctions(zetasqlBuiltinFunctionOptions);
+
+ FUNCTION_LIST.stream()
+ .map(func -> (ResolvedCreateFunctionStmt) Analyzer.analyzeStatement(func, options, catalog))
+ .map(
+ resolvedFunc ->
+ new Function(
+ String.join(".", resolvedFunc.getNamePath()),
+ PRE_DEFINED_WINDOW_FUNCTIONS,
+ Mode.SCALAR,
+ ImmutableList.of(resolvedFunc.getSignature())))
+ .forEach(catalog::addFunction);
+ }
+
+ /**
+ * Assume last element in tablePath is a table name, and everything before is catalogs. So the
+ * logic is to create nested catalogs until the last level, then add a table at the last level.
+ *
+ * <p>Table schema is extracted from Calcite schema based on the table name resultion strategy,
+ * e.g. either by drilling down the schema.getSubschema() path or joining the table name with dots
+ * to construct a single compound identifier (e.g. Data Catalog use case).
+ */
+ private void addTableToLeafCatalog(
+ QueryTrait trait, SimpleCatalog topLevelCatalog, List<String> tablePath) {
+
+ SimpleCatalog leafCatalog = createNestedCatalogs(topLevelCatalog, tablePath);
+
+ org.apache.calcite.schema.Table calciteTable =
+ TableResolution.resolveCalciteTable(
+ builder.calciteContext, builder.topLevelSchema, tablePath);
+
+ if (calciteTable == null) {
+ throw new RuntimeException(
+ "Wasn't able to find resolve the path "
+ + tablePath
+ + " in "
+ + builder.topLevelSchema.getName());
+ }
+
+ RelDataType rowType = calciteTable.getRowType(builder.typeFactory);
+
+ SimpleTableWithPath tableWithPath =
+ SimpleTableWithPath.of(builder.topLevelSchema.getName(), tablePath);
+ trait.addResolvedTable(tableWithPath);
+
+ addFieldsToTable(tableWithPath, rowType);
+ leafCatalog.addSimpleTable(tableWithPath.getTable());
+ }
+
+ private void addFieldsToTable(SimpleTableWithPath tableWithPath, RelDataType rowType) {
+ for (RelDataTypeField field : rowType.getFieldList()) {
+ tableWithPath.getTable().addSimpleColumn(field.getName(), toZetaType(field.getType()));
+ }
+ }
+
+ /** For table path like a.b.c we assume c is the table and a.b are the nested catalogs/schemas. */
+ private SimpleCatalog createNestedCatalogs(SimpleCatalog catalog, List<String> tablePath) {
+ SimpleCatalog currentCatalog = catalog;
+ for (int i = 0; i < tablePath.size() - 1; i++) {
+ String nextCatalogName = tablePath.get(i);
+
+ Optional<SimpleCatalog> existing = tryGetExisting(currentCatalog, nextCatalogName);
+
+ currentCatalog =
+ existing.isPresent() ? existing.get() : addNewCatalog(currentCatalog, nextCatalogName);
+ }
+ return currentCatalog;
+ }
+
+ private Optional<SimpleCatalog> tryGetExisting(
+ SimpleCatalog currentCatalog, String nextCatalogName) {
+ return currentCatalog.getCatalogList().stream()
+ .filter(c -> nextCatalogName.equals(c.getFullName()))
+ .findFirst();
+ }
+
+ private SimpleCatalog addNewCatalog(SimpleCatalog currentCatalog, String nextCatalogName) {
+ SimpleCatalog nextCatalog = new SimpleCatalog(nextCatalogName);
+ currentCatalog.addSimpleCatalog(nextCatalog);
+ return nextCatalog;
+ }
+
+ /** Builder for SqlAnalyzer. */
+ static class Builder {
+
+ private Map<String, Value> queryParams;
+ private QueryTrait queryTrait;
+ private Context calciteContext;
+ private SchemaPlus topLevelSchema;
+ private JavaTypeFactory typeFactory;
+
+ private Builder() {}
+
+ /** Query parameters. */
+ Builder withQueryParams(Map<String, Value> params) {
+ this.queryParams = ImmutableMap.copyOf(params);
+ return this;
+ }
+
+ /** QueryTrait, has parsing-time configuration for rel conversion. */
+ Builder withQueryTrait(QueryTrait trait) {
+ this.queryTrait = trait;
+ return this;
+ }
+
+ /** Current top-level schema. */
+ Builder withTopLevelSchema(SchemaPlus schema) {
+ this.topLevelSchema = schema;
+ return this;
+ }
+
+ /** Calcite parsing context, can have name resolution and other configuration. */
+ Builder withCalciteContext(Context context) {
+ this.calciteContext = context;
+ return this;
+ }
+
+ /**
+ * Current type factory.
+ *
+ * <p>Used to convert field types in schemas.
+ */
+ Builder withTypeFactory(JavaTypeFactory typeFactory) {
+ this.typeFactory = typeFactory;
+ return this;
+ }
+
+ /** Returns the parsed and resolved query. */
+ ResolvedStatement analyze(String sql) {
+ return new SqlAnalyzer(this).analyze(sql);
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlCaseWithValueOperatorRewriter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlCaseWithValueOperatorRewriter.java
new file mode 100644
index 0000000..8bf7057
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlCaseWithValueOperatorRewriter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+
+/**
+ * Rewrites $case_with_value calls as $case_no_value calls.
+ *
+ * <p>Turns:
+ *
+ * <pre><code>CASE x
+ * WHEN w1 THEN t1
+ * WHEN w2 THEN t2
+ * ELSE e
+ * END</code></pre>
+ *
+ * <p>into:
+ *
+ * <pre><code>CASE
+ * WHEN x == w1 THEN t1
+ * WHEN x == w2 THEN t2
+ * ELSE expr
+ * END</code></pre>
+ *
+ * <p>Note that the ELSE statement is actually optional, but we don't need to worry about that here
+ * because the ZetaSQL analyzer populates the ELSE argument as a NULL literal if it's not specified.
+ */
+public class SqlCaseWithValueOperatorRewriter implements SqlOperatorRewriter {
+ @Override
+ public RexNode apply(RexBuilder rexBuilder, List<RexNode> operands) {
+ Preconditions.checkArgument(
+ operands.size() % 2 == 0 && !operands.isEmpty(),
+ "$case_with_value should have an even number of arguments greater than 0 in function call"
+ + " (The value operand, the else operand, and paired when/then operands).");
+ SqlOperator op = SqlStdOperatorTable.CASE;
+
+ List<RexNode> newOperands = new ArrayList<>();
+ RexNode value = operands.get(0);
+
+ for (int i = 1; i < operands.size() - 2; i += 2) {
+ RexNode when = operands.get(i);
+ RexNode then = operands.get(i + 1);
+ newOperands.add(
+ rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, ImmutableList.of(value, when)));
+ newOperands.add(then);
+ }
+
+ RexNode elseOperand = Iterables.getLast(operands);
+ newOperands.add(elseOperand);
+
+ return rexBuilder.makeCall(op, newOperands);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlCoalesceOperatorRewriter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlCoalesceOperatorRewriter.java
new file mode 100644
index 0000000..39198d0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlCoalesceOperatorRewriter.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.Util;
+
+/**
+ * Rewrites COALESCE calls as CASE ($case_no_value) calls.
+ *
+ * <p>Turns <code>COALESCE(a, b, c)</code> into:
+ *
+ * <pre><code>CASE
+ * WHEN a IS NOT NULL THEN a
+ * WHEN b IS NOT NULL THEN b
+ * ELSE c
+ * END</code></pre>
+ *
+ * <p>There is also a special case for the single-argument case: <code>COALESCE(a)</code> becomes
+ * just <code>a</code>.
+ */
+public class SqlCoalesceOperatorRewriter implements SqlOperatorRewriter {
+ @Override
+ public RexNode apply(RexBuilder rexBuilder, List<RexNode> operands) {
+ Preconditions.checkArgument(
+ operands.size() >= 1, "COALESCE should have at least one argument in function call.");
+
+ // No need for a case operator if there's only one operand
+ if (operands.size() == 1) {
+ return operands.get(0);
+ }
+
+ SqlOperator op = SqlStdOperatorTable.CASE;
+
+ List<RexNode> newOperands = new ArrayList<>();
+ for (RexNode operand : Util.skipLast(operands)) {
+ newOperands.add(
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NOT_NULL, ImmutableList.of(operand)));
+ newOperands.add(operand);
+ }
+ newOperands.add(Util.last(operands));
+
+ return rexBuilder.makeCall(op, newOperands);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlExtractTimestampOperatorRewriter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlExtractTimestampOperatorRewriter.java
new file mode 100644
index 0000000..a3e9c55
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlExtractTimestampOperatorRewriter.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+
+/** Rewrites EXTRACT calls by swapping first two arguments to fit for calcite SqlExtractOperator. */
+public class SqlExtractTimestampOperatorRewriter implements SqlOperatorRewriter {
+ @Override
+ public RexNode apply(RexBuilder rexBuilder, List<RexNode> operands) {
+ Preconditions.checkArgument(
+ operands.size() == 2,
+ "EXTRACT should have two arguments in function call. AT TIME ZONE not supported.");
+
+ SqlOperator op =
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get("$extract");
+
+ ImmutableList.Builder<RexNode> newOperands =
+ ImmutableList.<RexNode>builder().add(operands.get(1)).add(operands.get(0));
+
+ for (int i = 2; i < operands.size(); ++i) {
+ newOperands.add(operands.get(i));
+ }
+
+ return rexBuilder.makeCall(op, newOperands.build());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlIfNullOperatorRewriter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlIfNullOperatorRewriter.java
new file mode 100644
index 0000000..69478bb
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlIfNullOperatorRewriter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+
+/**
+ * Rewrites IFNULL calls as CASE ($case_no_value) calls.
+ *
+ * <p>Turns <code>IFNULL(expr, null_result)</code> into: <code><pre>CASE
+ * WHEN expr IS NULL THEN null_result
+ * ELSE expr
+ * END</pre></code>
+ */
+public class SqlIfNullOperatorRewriter implements SqlOperatorRewriter {
+ @Override
+ public RexNode apply(RexBuilder rexBuilder, List<RexNode> operands) {
+ Preconditions.checkArgument(
+ operands.size() == 2, "IFNULL should have two arguments in function call.");
+
+ SqlOperator op = SqlStdOperatorTable.CASE;
+ List<RexNode> newOperands =
+ ImmutableList.of(
+ rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, ImmutableList.of(operands.get(0))),
+ operands.get(1),
+ operands.get(0));
+
+ return rexBuilder.makeCall(op, newOperands);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlNullIfOperatorRewriter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlNullIfOperatorRewriter.java
new file mode 100644
index 0000000..69184ac
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlNullIfOperatorRewriter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+
+/**
+ * Rewrites NULLIF calls as CASE ($case_no_value) calls.
+ *
+ * <p>Turns <code>NULLIF(expression, expression_to_match)</code> into: <code><pre>CASE
+ * WHEN expression == expression_to_match THEN NULL
+ * ELSE expression
+ * END</pre></code>
+ */
+public class SqlNullIfOperatorRewriter implements SqlOperatorRewriter {
+ @Override
+ public RexNode apply(RexBuilder rexBuilder, List<RexNode> operands) {
+ Preconditions.checkArgument(
+ operands.size() == 2, "NULLIF should have two arguments in function call.");
+
+ SqlOperator op =
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get("$case_no_value");
+ List<RexNode> newOperands =
+ ImmutableList.of(
+ rexBuilder.makeCall(
+ SqlStdOperatorTable.EQUALS, ImmutableList.of(operands.get(0), operands.get(1))),
+ rexBuilder.makeNullLiteral(operands.get(1).getType()),
+ operands.get(0));
+
+ return rexBuilder.makeCall(op, newOperands);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlOperatorRewriter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlOperatorRewriter.java
new file mode 100644
index 0000000..f4245bd
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlOperatorRewriter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.List;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+
+/** Interface for rewriting calls a specific ZetaSQL operator. */
+public interface SqlOperatorRewriter {
+ /**
+ * Create and return a new {@link RexNode} that represents a call to this operator with the
+ * specified operands.
+ *
+ * @param rexBuilder A {@link RexBuilder} instance to use for creating new {@link RexNode}s
+ * @param operands The original list of {@link RexNode} operands passed to this operator call
+ * @return The created RexNode
+ */
+ RexNode apply(RexBuilder rexBuilder, List<RexNode> operands);
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlOperators.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlOperators.java
new file mode 100644
index 0000000..99bfb13
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlOperators.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.impl.ScalarFunctionImpl;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRelDataTypeSystem;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
+import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeFactoryImpl;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.FunctionParameter;
+import org.apache.calcite.schema.ScalarFunction;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.FamilyOperandTypeChecker;
+import org.apache.calcite.sql.type.InferTypes;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeFactoryImpl;
+import org.apache.calcite.sql.type.SqlTypeFamily;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+import org.apache.calcite.util.Util;
+
+/**
+ * A separate SqlOperators table for those functions that do not exist or not compatible with
+ * Calcite. Most of functions within this class is copied from Calcite.
+ */
+public class SqlOperators {
+ public static final RelDataType TIMESTAMP_WITH_NULLABILITY =
+ createSqlType(SqlTypeName.TIMESTAMP, true);
+ public static final RelDataType OTHER = createSqlType(SqlTypeName.OTHER, false);
+ public static final RelDataType TIMESTAMP = createSqlType(SqlTypeName.TIMESTAMP, false);
+ public static final RelDataType BIGINT = createSqlType(SqlTypeName.BIGINT, false);
+
+ public static SqlUserDefinedFunction createUdfOperator(
+ String name,
+ Class<?> methodClass,
+ String methodName,
+ SqlReturnTypeInference returnTypeInference,
+ List<RelDataType> paramTypes) {
+ return new SqlUserDefinedFunction(
+ new SqlIdentifier(name, SqlParserPos.ZERO),
+ returnTypeInference,
+ null,
+ null,
+ paramTypes,
+ ScalarFunctionImpl.create(methodClass, methodName));
+ }
+
+ // Helper function to create SqlUserDefinedFunction based on a function name and a method.
+ // SqlUserDefinedFunction will be able to pass through Calcite codegen and get proper function
+ // called.
+ public static SqlUserDefinedFunction createUdfOperator(String name, Method method) {
+ Function function = ScalarFunctionImpl.create(method);
+ final RelDataTypeFactory typeFactory = createTypeFactory();
+
+ List<RelDataType> argTypes = new ArrayList<>();
+ List<SqlTypeFamily> typeFamilies = new ArrayList<>();
+ for (FunctionParameter o : function.getParameters()) {
+ final RelDataType type = o.getType(typeFactory);
+ argTypes.add(type);
+ typeFamilies.add(Util.first(type.getSqlTypeName().getFamily(), SqlTypeFamily.ANY));
+ }
+
+ final FamilyOperandTypeChecker typeChecker =
+ OperandTypes.family(typeFamilies, i -> function.getParameters().get(i).isOptional());
+
+ final List<RelDataType> paramTypes = toSql(typeFactory, argTypes);
+
+ return new SqlUserDefinedFunction(
+ new SqlIdentifier(name, SqlParserPos.ZERO),
+ infer((ScalarFunction) function),
+ InferTypes.explicit(argTypes),
+ typeChecker,
+ paramTypes,
+ function);
+ }
+
+ private static RelDataType createSqlType(SqlTypeName typeName, boolean withNullability) {
+ final RelDataTypeFactory typeFactory = createTypeFactory();
+ RelDataType type = typeFactory.createSqlType(typeName);
+ if (withNullability) {
+ type = typeFactory.createTypeWithNullability(type, true);
+ }
+ return type;
+ }
+
+ private static RelDataTypeFactory createTypeFactory() {
+ return new SqlTypeFactoryImpl(BeamRelDataTypeSystem.INSTANCE);
+ }
+
+ private static SqlReturnTypeInference infer(final ScalarFunction function) {
+ return opBinding -> {
+ final RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
+ final RelDataType type;
+ if (function instanceof ScalarFunctionImpl) {
+ type = ((ScalarFunctionImpl) function).getReturnType(typeFactory, opBinding);
+ } else {
+ type = function.getReturnType(typeFactory);
+ }
+ return toSql(typeFactory, type);
+ };
+ }
+
+ private static List<RelDataType> toSql(
+ final RelDataTypeFactory typeFactory, List<RelDataType> types) {
+ return Lists.transform(types, type -> toSql(typeFactory, type));
+ }
+
+ private static RelDataType toSql(RelDataTypeFactory typeFactory, RelDataType type) {
+ if (type instanceof RelDataTypeFactoryImpl.JavaType
+ && ((RelDataTypeFactoryImpl.JavaType) type).getJavaClass() == Object.class) {
+ return typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.ANY), true);
+ }
+ return JavaTypeFactoryImpl.toSql(typeFactory, type);
+ }
+
+ private static final RelDataType BIGINT_WITH_NULLABILITY =
+ createSqlType(SqlTypeName.BIGINT, true);
+
+ public static final SqlOperator START_WITHS =
+ createUdfOperator("STARTS_WITH", BeamBuiltinMethods.STARTS_WITH_METHOD);
+
+ public static final SqlOperator CONCAT =
+ createUdfOperator("CONCAT", BeamBuiltinMethods.CONCAT_METHOD);
+
+ public static final SqlOperator REPLACE =
+ createUdfOperator("REPLACE", BeamBuiltinMethods.REPLACE_METHOD);
+
+ public static final SqlOperator TRIM = createUdfOperator("TRIM", BeamBuiltinMethods.TRIM_METHOD);
+
+ public static final SqlOperator LTRIM =
+ createUdfOperator("LTRIM", BeamBuiltinMethods.LTRIM_METHOD);
+
+ public static final SqlOperator RTRIM =
+ createUdfOperator("RTRIM", BeamBuiltinMethods.RTRIM_METHOD);
+
+ public static final SqlOperator SUBSTR =
+ createUdfOperator("SUBSTR", BeamBuiltinMethods.SUBSTR_METHOD);
+
+ public static final SqlOperator REVERSE =
+ createUdfOperator("REVERSE", BeamBuiltinMethods.REVERSE_METHOD);
+
+ public static final SqlOperator CHAR_LENGTH =
+ createUdfOperator("CHAR_LENGTH", BeamBuiltinMethods.CHAR_LENGTH_METHOD);
+
+ public static final SqlOperator ENDS_WITH =
+ createUdfOperator("ENDS_WITH", BeamBuiltinMethods.ENDS_WITH_METHOD);
+
+ public static final SqlOperator LIKE = createUdfOperator("LIKE", BeamBuiltinMethods.LIKE_METHOD);
+
+ public static final SqlOperator VALIDATE_TIMESTAMP =
+ createUdfOperator(
+ "validateTimestamp",
+ DateTimeUtils.class,
+ "validateTimestamp",
+ x -> TIMESTAMP_WITH_NULLABILITY,
+ ImmutableList.of(TIMESTAMP));
+
+ public static final SqlOperator VALIDATE_TIME_INTERVAL =
+ createUdfOperator(
+ "validateIntervalArgument",
+ DateTimeUtils.class,
+ "validateTimeInterval",
+ x -> BIGINT_WITH_NULLABILITY,
+ ImmutableList.of(BIGINT, OTHER));
+
+ public static final SqlOperator TIMESTAMP_OP =
+ createUdfOperator("TIMESTAMP", BeamBuiltinMethods.TIMESTAMP_METHOD);
+
+ public static final SqlOperator DATE_OP =
+ createUdfOperator("DATE", BeamBuiltinMethods.DATE_METHOD);
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlStdOperatorMappingTable.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlStdOperatorMappingTable.java
new file mode 100644
index 0000000..8ecf652
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlStdOperatorMappingTable.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import com.google.zetasql.ZetaSQLFunction.FunctionSignatureId;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+
+/** SqlStdOperatorMappingTable. */
+@Internal
+public class SqlStdOperatorMappingTable {
+ static final List<FunctionSignatureId> ZETASQL_BUILTIN_FUNCTION_WHITELIST =
+ Arrays.asList(
+ FunctionSignatureId.FN_AND,
+ FunctionSignatureId.FN_OR,
+ FunctionSignatureId.FN_NOT,
+ FunctionSignatureId.FN_MULTIPLY_DOUBLE,
+ FunctionSignatureId.FN_MULTIPLY_INT64,
+ FunctionSignatureId.FN_MULTIPLY_NUMERIC,
+ FunctionSignatureId.FN_DIVIDE_DOUBLE,
+ FunctionSignatureId.FN_DIVIDE_NUMERIC,
+ FunctionSignatureId.FN_ADD_DOUBLE,
+ FunctionSignatureId.FN_ADD_INT64,
+ FunctionSignatureId.FN_ADD_NUMERIC,
+ FunctionSignatureId.FN_SUBTRACT_DOUBLE,
+ FunctionSignatureId.FN_SUBTRACT_INT64,
+ FunctionSignatureId.FN_SUBTRACT_NUMERIC,
+ FunctionSignatureId.FN_UNARY_MINUS_INT64,
+ FunctionSignatureId.FN_UNARY_MINUS_DOUBLE,
+ FunctionSignatureId.FN_UNARY_MINUS_NUMERIC,
+ FunctionSignatureId.FN_GREATER,
+ FunctionSignatureId.FN_GREATER_OR_EQUAL,
+ FunctionSignatureId.FN_LESS,
+ FunctionSignatureId.FN_LESS_OR_EQUAL,
+ FunctionSignatureId.FN_EQUAL,
+ FunctionSignatureId.FN_NOT_EQUAL,
+ FunctionSignatureId.FN_IS_NULL,
+ FunctionSignatureId.FN_IS_TRUE,
+ FunctionSignatureId.FN_IS_FALSE,
+ FunctionSignatureId.FN_STARTS_WITH_STRING,
+ FunctionSignatureId.FN_SUBSTR_STRING,
+ FunctionSignatureId.FN_TRIM_STRING,
+ FunctionSignatureId.FN_LTRIM_STRING,
+ FunctionSignatureId.FN_RTRIM_STRING,
+ FunctionSignatureId.FN_REPLACE_STRING,
+ FunctionSignatureId.FN_CONCAT_STRING,
+ FunctionSignatureId.FN_COUNT_STAR,
+ FunctionSignatureId.FN_COUNT,
+ FunctionSignatureId.FN_MAX,
+ FunctionSignatureId.FN_MIN,
+ FunctionSignatureId.FN_AVG_DOUBLE,
+ FunctionSignatureId.FN_AVG_INT64,
+ FunctionSignatureId.FN_AVG_NUMERIC,
+ FunctionSignatureId.FN_SUM_DOUBLE,
+ FunctionSignatureId.FN_SUM_INT64,
+ FunctionSignatureId.FN_SUM_NUMERIC,
+ FunctionSignatureId.FN_MOD_INT64,
+ FunctionSignatureId.FN_MOD_NUMERIC,
+ FunctionSignatureId.FN_CASE_NO_VALUE,
+ FunctionSignatureId.FN_CASE_WITH_VALUE,
+ FunctionSignatureId.FN_TIMESTAMP_ADD,
+ // FunctionSignatureId.FN_TIMESTAMP_SUB,
+ FunctionSignatureId.FN_FLOOR_DOUBLE,
+ FunctionSignatureId.FN_FLOOR_NUMERIC,
+ FunctionSignatureId.FN_CEIL_DOUBLE,
+ FunctionSignatureId.FN_CEIL_NUMERIC,
+ FunctionSignatureId.FN_REVERSE_STRING,
+ FunctionSignatureId.FN_CHAR_LENGTH_STRING,
+ FunctionSignatureId.FN_ENDS_WITH_STRING,
+ FunctionSignatureId.FN_STRING_LIKE,
+ FunctionSignatureId.FN_COALESCE,
+ FunctionSignatureId.FN_IF,
+ FunctionSignatureId.FN_IFNULL,
+ FunctionSignatureId.FN_NULLIF,
+ FunctionSignatureId.FN_EXTRACT_FROM_DATE,
+ FunctionSignatureId.FN_EXTRACT_FROM_DATETIME,
+ FunctionSignatureId.FN_EXTRACT_FROM_TIME,
+ FunctionSignatureId.FN_EXTRACT_FROM_TIMESTAMP,
+ FunctionSignatureId.FN_TIMESTAMP_FROM_STRING,
+ FunctionSignatureId.FN_TIMESTAMP_FROM_DATE,
+ // FunctionSignatureId.FN_TIMESTAMP_FROM_DATETIME
+ FunctionSignatureId.FN_DATE_FROM_YEAR_MONTH_DAY
+ // FunctionSignatureId.FN_DATE_FROM_TIMESTAMP
+ );
+
+ // todo: Some of operators defined here are later overridden in ZetaSQLPlannerImpl.
+ // We should remove them from this table and add generic way to provide custom
+ // implementation. (Ex.: timestamp_add)
+ public static final ImmutableMap<String, SqlOperator> ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR =
+ ImmutableMap.<String, SqlOperator>builder()
+ // grouped window function
+ .put("TUMBLE", SqlStdOperatorTable.TUMBLE)
+ .put("HOP", SqlStdOperatorTable.HOP)
+ .put("SESSION", SqlStdOperatorTable.SESSION)
+
+ // built-in logical operator
+ .put("$and", SqlStdOperatorTable.AND)
+ .put("$or", SqlStdOperatorTable.OR)
+ .put("$not", SqlStdOperatorTable.NOT)
+
+ // built-in comparison operator
+ .put("$equal", SqlStdOperatorTable.EQUALS)
+ .put("$not_equal", SqlStdOperatorTable.NOT_EQUALS)
+ .put("$greater", SqlStdOperatorTable.GREATER_THAN)
+ .put("$greater_or_equal", SqlStdOperatorTable.GREATER_THAN_OR_EQUAL)
+ .put("$less", SqlStdOperatorTable.LESS_THAN)
+ .put("$less_or_equal", SqlStdOperatorTable.LESS_THAN_OR_EQUAL)
+ .put("$like", SqlOperators.LIKE)
+ // .put("$in", SqlStdOperatorTable.IN)
+ // .put("$between", SqlStdOperatorTable.BETWEEN)
+ .put("$is_null", SqlStdOperatorTable.IS_NULL)
+ .put("$is_true", SqlStdOperatorTable.IS_TRUE)
+ .put("$is_false", SqlStdOperatorTable.IS_FALSE)
+
+ // +, -, *, /
+ .put("$add", SqlStdOperatorTable.PLUS)
+ .put("$subtract", SqlStdOperatorTable.MINUS)
+ .put("$multiply", SqlStdOperatorTable.MULTIPLY)
+ .put("$unary_minus", SqlStdOperatorTable.UNARY_MINUS)
+ .put("$divide", SqlStdOperatorTable.DIVIDE)
+
+ // built-in string function
+ .put("concat", SqlOperators.CONCAT)
+ // .put("lower", SqlStdOperatorTable.LOWER)
+ // .put("upper", SqlStdOperatorTable.UPPER)
+ .put("substr", SqlOperators.SUBSTR)
+ .put("trim", SqlOperators.TRIM)
+ .put("replace", SqlOperators.REPLACE)
+ .put("char_length", SqlOperators.CHAR_LENGTH)
+
+ // string function UDFs
+ // .put("strpos", )
+ // .put("length", )
+ // tells Calcite codegen that starts_with function is a udf.
+ .put("starts_with", SqlOperators.START_WITHS)
+ .put("ends_with", SqlOperators.ENDS_WITH)
+ .put("ltrim", SqlOperators.LTRIM)
+ .put("rtrim", SqlOperators.RTRIM)
+ // .put("regexp_match",)
+ // .put("regexp_extract",)
+ // .put("regexp_replace",)
+ // .put("regexp_extract_all",)
+ // .put("byte_length",)
+ // .put("format",)
+ // .put("split",)
+ // .put("regexp_contains", )
+ // .put("normalize",)
+ // .put("to_base32",)
+ // .put("to_base64",)
+ // .put("to_hex",)
+ // .put("from_base64",)
+ // .put("from_base32",)
+ // .put("from_hex",)
+ // .put("to_code_points")
+ // .put("code_points_to_string")
+ // .put("lpad", )
+ // .put("rpad", )
+ // .put("repeat", )
+ .put("reverse", SqlOperators.REVERSE)
+
+ // built-in aggregate function
+ .put("$count_star", SqlStdOperatorTable.COUNT)
+ // TODO: add support to all aggregate functions.
+ .put("max", SqlStdOperatorTable.MAX)
+ .put("min", SqlStdOperatorTable.MIN)
+ .put("avg", SqlStdOperatorTable.AVG)
+ .put("sum", SqlStdOperatorTable.SUM)
+ // .put("any_value", SqlStdOperatorTable.ANY_VALUE)
+ .put("count", SqlStdOperatorTable.COUNT)
+
+ // aggregate UDF
+ // .put("array_agg", )
+ // .put("array_concat_agg")
+ // .put("string_agg")
+ // .put("bit_and")
+ // .put("bit_or")
+ // .put("bit_xor")
+ // .put("logical_and")
+ // .put("logical_or")
+
+ // built-in statistical aggregate function
+ // .put("covar_pop", SqlStdOperatorTable.COVAR_POP)
+ // .put("covar_samp", SqlStdOperatorTable.COVAR_SAMP)
+ // .put("stddev_pop", SqlStdOperatorTable.STDDEV_POP)
+ // .put("stddev_samp", SqlStdOperatorTable.STDDEV_SAMP)
+ // .put("var_pop", SqlStdOperatorTable.VAR_POP)
+ // .put("var_samp", SqlStdOperatorTable.VAR_SAMP)
+
+ // statistical aggregate UDF
+ // .put("corr", )
+
+ // built-in approximate aggregate function
+ // .put("approx_count_distinct", SqlStdOperatorTable.APPROX_COUNT_DISTINCT)
+
+ // approximate aggregate UDF
+ // .put("approx_quantiles", )
+ // .put("approx_top_sum")
+
+ // HLL++ UDF
+ // hll_count.merge
+ // hll_count.extract
+ // hll_count.init
+ // hll_count.merge_partial
+
+ // CAST
+ // CAST operator does not go through lookup table.
+ // .put("cast", SqlStdOperatorTable.CAST)
+
+ // built-in math functions
+ // .put("math", SqlStdOperatorTable.ABS)
+ // .put("sign", SqlStdOperatorTable.SIGN)
+ // .put("round", SqlStdOperatorTable.ROUND)
+ .put("ceil", SqlStdOperatorTable.CEIL)
+ .put("floor", SqlStdOperatorTable.FLOOR)
+ .put("mod", SqlStdOperatorTable.MOD)
+ // .put("sqrt", SqlStdOperatorTable.SQRT)
+ // .put("exp", SqlStdOperatorTable.EXP)
+ // .put("ln and log", SqlStdOperatorTable.LN)
+ // .put("log10", SqlStdOperatorTable.LOG10)
+ // .put("cos", SqlStdOperatorTable.COS)
+ // .put("acos", SqlStdOperatorTable.ACOS)
+ // .put("sin", SqlStdOperatorTable.SIN)
+ // .put("asin", SqlStdOperatorTable.ASIN)
+ // .put("tan", SqlStdOperatorTable.TAN)
+ // .put("atan", SqlStdOperatorTable.ATAN)
+ // .put("atan2", SqlStdOperatorTable.ATAN2)
+ // .put("abs", SqlStdOperatorTable.ABS)
+ // .put("pow", SqlStdOperatorTable.POWER)
+ // .put("div", SqlStdOperatorTable.DIVIDE)
+ // .put("trunc", SqlStdOperatorTable.TRUNCATE)
+
+ // math UDF
+ // .put("is_inf",)
+ // .put("is_nan",)
+ // .put("ieee_divide")
+ // .put("safe_add")
+ // .put("safe_divide")
+ // .put("safe_subtract")
+ // .put("safe_multiply")
+ // .put("safe_negate")
+ // .put("greatest")
+ // .put("least")
+ // .put("log")
+ // .put("cosh")
+ // .put("acosh")
+ // .put("sinh")
+ // .put("asinh")
+ // .put("tanh")
+ // .put("atanh")
+
+ // Analytic functions
+ // .put("dense_rank", SqlStdOperatorTable.DENSE_RANK)
+ // .put("rank", SqlStdOperatorTable.RANK)
+ // .put("row_number", SqlStdOperatorTable.ROW_NUMBER)
+ // .put("percent_rank", SqlStdOperatorTable.PERCENT_RANK)
+ // .put("cume_dist", SqlStdOperatorTable.CUME_DIST)
+ // .put("ntile", SqlStdOperatorTable.NTILE)
+ // .put("lead", SqlStdOperatorTable.LEAD)
+ // .put("lag", SqlStdOperatorTable.LAG)
+ // .put("first_value", SqlStdOperatorTable.FIRST_VALUE)
+ // .put("last_value", SqlStdOperatorTable.LAST_VALUE)
+ // .put("nth_value", SqlStdOperatorTable.NTH_VALUE)
+
+ // .put("percentile_cont", )
+ // .put("percentile_disc",)
+
+ // misc functions
+ // .put("fingerprint")
+ // .put("fingerprint2011")
+
+ // hash functions
+ // .put("md5")
+ // .put("sha1")
+ // .put("sha256")
+ // .put("sha512")
+
+ // date functions
+ // .put("date_add", SqlStdOperatorTable.DATETIME_PLUS)
+ // .put("date_sub", SqlStdOperatorTable.MINUS_DATE)
+ .put("date", SqlOperators.DATE_OP)
+
+ // time functions
+ // .put("time_add", SqlStdOperatorTable.DATETIME_PLUS)
+ // .put("time_sub", SqlStdOperatorTable.MINUS_DATE)
+
+ // timestamp functions
+ .put(
+ "timestamp_add",
+ SqlStdOperatorTable.DATETIME_PLUS) // overridden in ZetaSQLPlannerImpl
+ // .put("timestamp_sub", SqlStdOperatorTable.MINUS_DATE)
+ .put("$extract", SqlStdOperatorTable.EXTRACT)
+ .put("timestamp", SqlOperators.TIMESTAMP_OP)
+
+ // other functions
+ // .put("session_user", SqlStdOperatorTable.SESSION_USER)
+ // .put("bit_cast_to_int32")
+ // .put("bit_cast_to_int64")
+ // .put("bit_cast_to_uint32")
+ // .put("bit_cast_to_uint64")
+ // .put("countif", )
+
+ // case operator
+ .put("$case_no_value", SqlStdOperatorTable.CASE)
+
+ // if operator - IF(cond, pos, neg) can actually be mapped directly to `CASE WHEN cond
+ // THEN pos ELSE neg`
+ .put("if", SqlStdOperatorTable.CASE)
+
+ // $case_no_value specializations
+ // all of these operators can have their operands adjusted to achieve the same thing with
+ // a call to $case_with_value
+ .put("$case_with_value", SqlStdOperatorTable.CASE)
+ .put("coalesce", SqlStdOperatorTable.CASE)
+ .put("ifnull", SqlStdOperatorTable.CASE)
+ .put("nullif", SqlStdOperatorTable.CASE)
+ .build();
+
+ // argument one and two should compose of a interval.
+ public static final ImmutableSet<String> FUNCTION_FAMILY_DATE_ADD =
+ ImmutableSet.of(
+ "date_add",
+ "date_sub",
+ "datetime_add",
+ "datetime_sub",
+ "time_add",
+ "time_sub",
+ "timestamp_add",
+ "timestamp_sub");
+
+ public static final ImmutableMap<String, SqlOperatorRewriter>
+ ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR_REWRITER =
+ ImmutableMap.<String, SqlOperatorRewriter>builder()
+ .put("$case_with_value", new SqlCaseWithValueOperatorRewriter())
+ .put("coalesce", new SqlCoalesceOperatorRewriter())
+ .put("ifnull", new SqlIfNullOperatorRewriter())
+ .put("nullif", new SqlNullIfOperatorRewriter())
+ .put("$extract", new SqlExtractTimestampOperatorRewriter())
+ .build();
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/StringFunctions.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/StringFunctions.java
new file mode 100644
index 0000000..07f8746
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/StringFunctions.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.regex.Pattern;
+import org.apache.calcite.linq4j.function.Strict;
+import org.apache.calcite.runtime.SqlFunctions;
+
+/** StringFunctions. */
+public class StringFunctions {
+ public static final String SUBSTR_PARAMETER_EXCEED_INTEGER =
+ "SUBSTR function only allows: "
+ + Integer.MIN_VALUE
+ + " <= position or length <= "
+ + Integer.MAX_VALUE;
+
+ @Strict
+ public static Boolean startsWith(String str1, String str2) {
+ return str1.startsWith(str2);
+ }
+
+ @Strict
+ public static Boolean endsWith(String str1, String str2) {
+ return str1.endsWith(str2);
+ }
+
+ @Strict
+ public static String concat(String arg) {
+ return arg;
+ }
+
+ @Strict
+ public static String concat(String arg1, String arg2) {
+ return concatIfNotIncludeNull(arg1, arg2);
+ }
+
+ @Strict
+ public static String concat(String arg1, String arg2, String arg3) {
+ return concatIfNotIncludeNull(arg1, arg2, arg3);
+ }
+
+ @Strict
+ public static String concat(String arg1, String arg2, String arg3, String arg4) {
+ return concatIfNotIncludeNull(arg1, arg2, arg3, arg4);
+ }
+
+ @Strict
+ public static String concat(String arg1, String arg2, String arg3, String arg4, String arg5) {
+ return concatIfNotIncludeNull(arg1, arg2, arg3, arg4, arg5);
+ }
+
+ @Strict
+ private static String concatIfNotIncludeNull(String... args) {
+ return String.join("", args);
+ }
+
+ // https://jira.apache.org/jira/browse/CALCITE-2889
+ // public static String concat(String... args) {
+ // StringBuilder stringBuilder = new StringBuilder();
+ // for (String arg : args) {
+ // stringBuilder.append(arg);
+ // }
+ // return stringBuilder.toString();
+ // }
+
+ @Strict
+ public static String replace(String origin, String target, String replacement) {
+ // Java's string.replace behaves differently when target = "". When target = "",
+ // Java's replace function replace every character in origin with replacement,
+ // while origin value should not be changed is expected in SQL.
+ if (target.length() == 0) {
+ return origin;
+ }
+
+ return origin.replace(target, replacement);
+ }
+
+ public static String trim(String str) {
+ return trim(str, " ");
+ }
+
+ @Strict
+ public static String trim(String str, String seek) {
+ return SqlFunctions.trim(true, true, seek, str, false);
+ }
+
+ public static String ltrim(String str) {
+ return ltrim(str, " ");
+ }
+
+ @Strict
+ public static String ltrim(String str, String seek) {
+ return SqlFunctions.trim(true, false, seek, str, false);
+ }
+
+ public static String rtrim(String str) {
+ return rtrim(str, " ");
+ }
+
+ @Strict
+ public static String rtrim(String str, String seek) {
+ return SqlFunctions.trim(false, true, seek, str, false);
+ }
+
+ public static String substr(String str, long from, long len) {
+ if (from > Integer.MAX_VALUE
+ || len > Integer.MAX_VALUE
+ || from < Integer.MIN_VALUE
+ || len < Integer.MIN_VALUE) {
+ throw new RuntimeException(SUBSTR_PARAMETER_EXCEED_INTEGER);
+ }
+ return SqlFunctions.substring(str, (int) from, (int) len);
+ }
+
+ @Strict
+ public static String reverse(String str) {
+ return new StringBuilder(str).reverse().toString();
+ }
+
+ @Strict
+ public static Long charLength(String str) {
+ return (long) str.length();
+ }
+
+ // ZetaSQL's LIKE statement does not support the ESCAPE clause. Instead it
+ // always uses \ as an escape character.
+ @Strict
+ public static Boolean like(String s, String pattern) {
+ String regex = sqlToRegexLike(pattern, '\\');
+ return Pattern.matches(regex, s);
+ }
+
+ private static final String JAVA_REGEX_SPECIALS = "[]()|^-+*?{}$\\.";
+
+ /**
+ * Translates a SQL LIKE pattern to Java regex pattern. Modified from Apache Calcite's
+ * Like.sqlToRegexLike
+ */
+ private static String sqlToRegexLike(String sqlPattern, char escapeChar) {
+ int i;
+ final int len = sqlPattern.length();
+ final StringBuilder javaPattern = new StringBuilder(len + len);
+ for (i = 0; i < len; i++) {
+ char c = sqlPattern.charAt(i);
+ if (c == escapeChar) {
+ if (i == (sqlPattern.length() - 1)) {
+ throw new RuntimeException("LIKE pattern ends with a backslash");
+ }
+ char nextChar = sqlPattern.charAt(++i);
+ if (JAVA_REGEX_SPECIALS.indexOf(nextChar) >= 0) {
+ javaPattern.append('\\');
+ }
+ javaPattern.append(nextChar);
+ } else if (c == '_') {
+ javaPattern.append('.');
+ } else if (c == '%') {
+ javaPattern.append("(?s:.*)");
+ } else {
+ if (JAVA_REGEX_SPECIALS.indexOf(c) >= 0) {
+ javaPattern.append('\\');
+ }
+ javaPattern.append(c);
+ }
+ }
+ return javaPattern.toString();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolution.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolution.java
new file mode 100644
index 0000000..54e3ab1
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolution.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import com.google.zetasql.SimpleTable;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.apache.calcite.plan.Context;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Table;
+
+/** Utility methods to resolve a table, given a top-level Calcite schema and a table path. */
+public class TableResolution {
+
+ /**
+ * Returns Calcite Table by consulting the schema.
+ *
+ * <p>The way the schema is queried is defined by the name resolution strategey implemented by a
+ * TableResolver and stored as a TableResolutionContext in the context.
+ *
+ * <p>If no custom table resolution logic is provided, default one is used, which is: drill down
+ * the getSubschema() path until the second-to-last path element. We expect the path to be a table
+ * path, so the last element should be a valid table id, we don't expect anything else there.
+ *
+ * <p>This resembles a default Calcite planner strategy. One difference is that Calcite doesn't
+ * assume the last element is a table and will continue to call getSubschema(), making it
+ * impossible for a table provider to understand the context.
+ */
+ public static Table resolveCalciteTable(
+ Context context, SchemaPlus schemaPlus, List<String> tablePath) {
+ TableResolutionContext tableResolutionContext = context.unwrap(TableResolutionContext.class);
+ TableResolver tableResolver = getTableResolver(tableResolutionContext, schemaPlus.getName());
+ return tableResolver.resolveCalciteTable(schemaPlus, tablePath);
+ }
+
+ static TableResolver getTableResolver(
+ TableResolutionContext tableResolutionContext, String schemaName) {
+ if (tableResolutionContext == null
+ || !tableResolutionContext.hasCustomResolutionFor(schemaName)) {
+ return TableResolver.DEFAULT_ASSUME_LEAF_IS_TABLE;
+ }
+
+ return tableResolutionContext.getTableResolver(schemaName);
+ }
+
+ /**
+ * Data class to store simple table, its full path (excluding top-level schema), and top-level
+ * schema.
+ */
+ static class SimpleTableWithPath {
+
+ SimpleTable table;
+ List<String> path;
+ String topLevelSchema;
+
+ static SimpleTableWithPath of(String topLevelSchema, List<String> path) {
+ SimpleTableWithPath tableWithPath = new SimpleTableWithPath();
+ tableWithPath.table = new SimpleTable(Iterables.getLast(path));
+ tableWithPath.path = path;
+ tableWithPath.topLevelSchema = topLevelSchema;
+ return tableWithPath;
+ }
+
+ SimpleTable getTable() {
+ return table;
+ }
+
+ List<String> getPath() {
+ return path;
+ }
+
+ String getTopLevelSchema() {
+ return topLevelSchema;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolutionContext.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolutionContext.java
new file mode 100644
index 0000000..e088019
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolutionContext.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.Map;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.calcite.plan.Context;
+import org.codehaus.commons.nullanalysis.Nullable;
+
+/**
+ * Calcite parser context to pass the configuration down to planner and rules so that we can
+ * configure custom name resolution.
+ */
+class TableResolutionContext implements Context {
+
+ /** Table resolvers, associating top-level schema to a custom name resolution logic. */
+ private final Map<String, TableResolver> resolvers;
+
+ /** Assigns a custom table resolver to the given schema. */
+ static TableResolutionContext of(String topLevelSchema, TableResolver resolver) {
+ return new TableResolutionContext(ImmutableMap.of(topLevelSchema, resolver));
+ }
+
+ /**
+ * Uses the resolution logic that joins the table path into a single compound identifier and then
+ * queries the schema once, instead of drilling down into subschemas.
+ */
+ static TableResolutionContext joinCompoundIds(String topLevelSchema) {
+ return of(topLevelSchema, TableResolver.JOIN_INTO_COMPOUND_ID);
+ }
+
+ TableResolutionContext with(String topLevelSchema, TableResolver resolver) {
+ return new TableResolutionContext(
+ ImmutableMap.<String, TableResolver>builder()
+ .putAll(this.resolvers)
+ .put(topLevelSchema, resolver)
+ .build());
+ }
+
+ boolean hasCustomResolutionFor(String schemaName) {
+ return resolvers.containsKey(schemaName);
+ }
+
+ @Nullable
+ TableResolver getTableResolver(String schemaName) {
+ return resolvers.get(schemaName);
+ }
+
+ private TableResolutionContext(Map<String, TableResolver> resolvers) {
+ this.resolvers = resolvers;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <T> T unwrap(Class<T> c) {
+ return c.isAssignableFrom(TableResolutionContext.class) ? (T) this : null;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolver.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolver.java
new file mode 100644
index 0000000..398eaf7
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolver.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.List;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.Table;
+
+/** An interface to implement a custom resolution strategy. */
+interface TableResolver {
+
+ TableResolver DEFAULT_ASSUME_LEAF_IS_TABLE = TableResolverImpl::assumeLeafIsTable;
+ TableResolver JOIN_INTO_COMPOUND_ID = TableResolverImpl::joinIntoCompoundId;
+
+ /**
+ * Returns a resolved table given a table path.
+ *
+ * <p>Returns null if table is not found.
+ */
+ Table resolveCalciteTable(Schema calciteSchema, List<String> tablePath);
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolverImpl.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolverImpl.java
new file mode 100644
index 0000000..fad71dc
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TableResolverImpl.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.apache.calcite.schema.Schema;
+import org.apache.calcite.schema.Table;
+
+/** A couple of implementations of TableResolver. */
+class TableResolverImpl {
+
+ /**
+ * Uses the logic similar to Calcite's EmptyScope.resolve_(...) except assumes the last element in
+ * the table path is a table name (which is assumed by ZetaSQL API getTableNames()).
+ *
+ * <p>This is the default.
+ *
+ * <p>I.e. drills down into schema.getSubschema() until the second last element of the table path,
+ * then calls schema.getTable().
+ */
+ static Table assumeLeafIsTable(Schema schema, List<String> tablePath) {
+ Schema subSchema = schema;
+
+ // subSchema.getSubschema() for all except last
+ for (int i = 0; i < tablePath.size() - 1; i++) {
+ subSchema = subSchema.getSubSchema(tablePath.get(i));
+ }
+
+ // for the final one call getTable()
+ return subSchema.getTable(Iterables.getLast(tablePath));
+ }
+
+ /**
+ * Joins the table name parts into a single ZetaSQL-compatible compound identifier, then calls
+ * schema.getTable().
+ *
+ * <p>This is the input expected, for example, by Data Catalog.
+ *
+ * <p>Escapes slashes, backticks, quotes, for details see {@link
+ * ZetaSqlIdUtils#escapeAndJoin(List)}.
+ */
+ static Table joinIntoCompoundId(Schema schema, List<String> tablePath) {
+ return schema.getTable(ZetaSqlIdUtils.escapeAndJoin(tablePath));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TestInput.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TestInput.java
new file mode 100644
index 0000000..9ad0a65
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TestInput.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.DateType;
+import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils.TimeType;
+import org.apache.beam.sdk.extensions.sql.meta.provider.test.TestBoundedTable;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.Schema.FieldType;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+
+/** TestInput. */
+class TestInput {
+ public static final FieldType DATE = FieldType.logicalType(new DateType());
+ public static final FieldType TIME = FieldType.logicalType(new TimeType());
+
+ public static final TestBoundedTable BASIC_TABLE_ONE =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addInt64Field("Key")
+ .addStringField("Value")
+ .addDateTimeField("ts")
+ .build())
+ .addRows(
+ 14L,
+ "KeyValue234",
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:06"),
+ 15L,
+ "KeyValue235",
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:07"));
+
+ public static final TestBoundedTable BASIC_TABLE_TWO =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addInt64Field("RowKey")
+ .addStringField("Value")
+ .addDateTimeField("ts")
+ .build())
+ .addRows(
+ 15L,
+ "BigTable235",
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:07"),
+ 16L,
+ "BigTable236",
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:08"));
+
+ public static final TestBoundedTable BASIC_TABLE_THREE =
+ TestBoundedTable.of(Schema.builder().addInt64Field("ColId").addStringField("Value").build())
+ .addRows(15L, "Spanner235", 16L, "Spanner236", 17L, "Spanner237");
+
+ public static final TestBoundedTable AGGREGATE_TABLE_ONE =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addInt64Field("Key")
+ .addInt64Field("Key2")
+ .addInt64Field("f_int_1")
+ .addStringField("f_str_1")
+ .addDoubleField("f_double_1")
+ .build())
+ .addRows(1L, 10L, 1L, "1", 1.0)
+ .addRows(1L, 11L, 2L, "2", 2.0)
+ .addRows(2L, 11L, 3L, "3", 3.0)
+ .addRows(2L, 11L, 4L, "4", 4.0)
+ .addRows(2L, 12L, 5L, "5", 5.0)
+ .addRows(3L, 13L, 6L, "6", 6.0)
+ .addRows(3L, 13L, 7L, "7", 7.0);
+
+ public static final TestBoundedTable AGGREGATE_TABLE_TWO =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addInt64Field("Key")
+ .addInt64Field("Key2")
+ .addInt64Field("f_int_1")
+ .addStringField("f_str_1")
+ .build())
+ .addRows(1L, 10L, 1L, "1")
+ .addRows(2L, 11L, 3L, "3")
+ .addRows(2L, 11L, 4L, "4")
+ .addRows(2L, 12L, 5L, "5")
+ .addRows(2L, 13L, 6L, "6")
+ .addRows(3L, 13L, 7L, "7");
+
+ public static final TestBoundedTable TABLE_ALL_TYPES =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addInt64Field("row_id")
+ .addBooleanField("bool_col")
+ .addInt64Field("int64_col")
+ .addDoubleField("double_col")
+ .addStringField("str_col")
+ .addByteArrayField("bytes_col")
+ .build())
+ .addRows(1L, true, -1L, 0.125d, "1", stringToBytes("1"))
+ .addRows(2L, false, -2L, Math.pow(0.1, 324.0), "2", stringToBytes("2"))
+ .addRows(3L, true, -3L, 0.375d, "3", stringToBytes("3"))
+ .addRows(4L, false, -4L, 0.5d, "4", stringToBytes("4"))
+ .addRows(5L, false, -5L, 0.5d, "5", stringToBytes("5"));
+
+ public static final TestBoundedTable TABLE_ALL_TYPES_2 =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addInt64Field("row_id")
+ .addBooleanField("bool_col")
+ .addInt64Field("int64_col")
+ .addDoubleField("double_col")
+ .addStringField("str_col")
+ .addByteArrayField("bytes_col")
+ .build())
+ .addRows(6L, true, -6L, 0.125d, "6", stringToBytes("6"))
+ .addRows(7L, false, -7L, Math.pow(0.1, 324.0), "7", stringToBytes("7"))
+ .addRows(8L, true, -8L, 0.375d, "8", stringToBytes("8"))
+ .addRows(9L, false, -9L, 0.5d, "9", stringToBytes("9"))
+ .addRows(10L, false, -10L, 0.5d, "10", stringToBytes("10"));
+
+ public static final TestBoundedTable TIMESTAMP_TABLE_ONE =
+ TestBoundedTable.of(Schema.builder().addDateTimeField("ts").addInt64Field("value").build())
+ .addRows(
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:06"),
+ 3L,
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:07"),
+ 4L,
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:08"),
+ 6L,
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:09"),
+ 7L);
+
+ public static final TestBoundedTable TIMESTAMP_TABLE_TWO =
+ TestBoundedTable.of(Schema.builder().addDateTimeField("ts").addInt64Field("value").build())
+ .addRows(
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:06"),
+ 3L,
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:07"),
+ 4L,
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:12"),
+ 6L,
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-01 21:26:13"),
+ 7L);
+
+ public static final TestBoundedTable TIME_TABLE =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addNullableField("f_date", DATE)
+ .addNullableField("f_time", TIME)
+ .addNullableField("f_timestamp", FieldType.DATETIME)
+ .addNullableField("f_timestamp_with_time_zone", FieldType.DATETIME)
+ .build())
+ .addRows(
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-07-11 00:00:00"),
+ DateTimeUtils.parseTimestampWithUTCTimeZone("1970-01-01 12:33:59.348"),
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2018-12-20 23:59:59.999"),
+ DateTimeUtils.parseTimestampWithTimeZone("2018-12-10 10:38:59-1000"));
+
+ public static final TestBoundedTable TABLE_ALL_NULL =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addNullableField("primary_key", FieldType.INT64)
+ .addNullableField("bool_val", FieldType.BOOLEAN)
+ .addNullableField("double_val", FieldType.DOUBLE)
+ .addNullableField("int64_val", FieldType.INT64)
+ .addNullableField("str_val", FieldType.STRING)
+ .build())
+ .addRows(1L, null, null, null, null);
+
+ public static final Schema TABLE_WITH_STRUCT_ROW_SCHEMA =
+ Schema.builder().addInt32Field("struct_col_int").addStringField("struct_col_str").build();
+
+ public static final TestBoundedTable TABLE_WITH_STRUCT =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addField("id", FieldType.INT64)
+ .addField("struct_col", FieldType.row(TABLE_WITH_STRUCT_ROW_SCHEMA))
+ .build())
+ .addRows(
+ 1L,
+ Row.withSchema(TABLE_WITH_STRUCT_ROW_SCHEMA).addValues(16, "row_one").build(),
+ 2L,
+ Row.withSchema(TABLE_WITH_STRUCT_ROW_SCHEMA).addValues(17, "row_two").build());
+
+ public static final TestBoundedTable TABLE_WITH_STRUCT_TIMESTAMP_STRING =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addField("struct_col", FieldType.row(TABLE_WITH_STRUCT_ROW_SCHEMA))
+ .build())
+ .addRows(
+ Row.withSchema(TABLE_WITH_STRUCT_ROW_SCHEMA)
+ .addValues(3, "2019-01-15 13:21:03")
+ .build());
+
+ private static final Schema structSchema =
+ Schema.builder().addInt64Field("row_id").addStringField("data").build();
+ private static final Schema structTableSchema =
+ Schema.builder().addRowField("rowCol", structSchema).build();
+
+ public static final TestBoundedTable TABLE_WITH_STRUCT_TWO =
+ TestBoundedTable.of(structTableSchema)
+ .addRows(Row.withSchema(structSchema).addValues(1L, "data1").build())
+ .addRows(Row.withSchema(structSchema).addValues(2L, "data2").build())
+ .addRows(Row.withSchema(structSchema).addValues(3L, "data2").build())
+ .addRows(Row.withSchema(structSchema).addValues(3L, "data3").build());
+
+ public static final TestBoundedTable TABLE_WITH_ARRAY =
+ TestBoundedTable.of(Schema.builder().addArrayField("array_col", FieldType.STRING).build())
+ .addRows(Arrays.asList("1", "2", "3"), ImmutableList.of());
+
+ public static final TestBoundedTable TABLE_WITH_ARRAY_FOR_UNNEST =
+ TestBoundedTable.of(
+ Schema.builder()
+ .addInt64Field("int_col")
+ .addArrayField("int_array_col", FieldType.INT64)
+ .build())
+ .addRows(14L, Arrays.asList(14L, 18L))
+ .addRows(18L, Arrays.asList(22L, 24L));
+
+ public static final TestBoundedTable TABLE_FOR_CASE_WHEN =
+ TestBoundedTable.of(
+ Schema.builder().addInt64Field("f_int").addStringField("f_string").build())
+ .addRows(1L, "20181018");
+
+ public static final TestBoundedTable TABLE_EMPTY =
+ TestBoundedTable.of(Schema.builder().addInt64Field("ColId").addStringField("Value").build());
+
+ private static final Schema TABLE_WTH_MAP_SCHEMA =
+ Schema.builder()
+ .addMapField("map_field", FieldType.STRING, FieldType.STRING)
+ .addRowField("row_field", structSchema)
+ .build();
+ public static final TestBoundedTable TABLE_WITH_MAP =
+ TestBoundedTable.of(TABLE_WTH_MAP_SCHEMA)
+ .addRows(
+ ImmutableMap.of("MAP_KEY_1", "MAP_VALUE_1"),
+ Row.withSchema(structSchema).addValues(1L, "data1").build());
+
+ public static byte[] stringToBytes(String s) {
+ return s.getBytes(StandardCharsets.UTF_8);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TimestampFunctions.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TimestampFunctions.java
new file mode 100644
index 0000000..6e9ef5a
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TimestampFunctions.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import java.util.TimeZone;
+import org.apache.calcite.linq4j.function.Strict;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/** TimestampFunctions. */
+public class TimestampFunctions {
+ public static DateTime timestamp(String timestampStr) {
+ return timestamp(timestampStr, "UTC");
+ }
+
+ @Strict
+ public static DateTime timestamp(String timestampStr, String timezone) {
+ return DateTimeUtils.findDateTimePattern(timestampStr)
+ .withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone(timezone)))
+ .parseDateTime(timestampStr);
+ }
+
+ @Strict
+ public static DateTime timestamp(Integer numOfDays) {
+ return timestamp(numOfDays, "UTC");
+ }
+
+ @Strict
+ public static DateTime timestamp(Integer numOfDays, String timezone) {
+ return new DateTime((long) numOfDays * DateTimeUtils.MILLIS_PER_DAY, DateTimeZone.UTC)
+ .withZoneRetainFields(DateTimeZone.forTimeZone(TimeZone.getTimeZone(timezone)));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TypeUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TypeUtils.java
new file mode 100644
index 0000000..fc5b604
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/TypeUtils.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_BOOL;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_BYTES;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_DATE;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_DOUBLE;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_FLOAT;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_INT32;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_INT64;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_NUMERIC;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_STRING;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_TIME;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_TIMESTAMP;
+import static java.util.stream.Collectors.toList;
+
+import com.google.zetasql.ArrayType;
+import com.google.zetasql.StructType;
+import com.google.zetasql.StructType.StructField;
+import com.google.zetasql.Type;
+import com.google.zetasql.TypeFactory;
+import com.google.zetasql.ZetaSQLType.TypeKind;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+/** Utility to convert types from Calcite Schema types. */
+@Internal
+public class TypeUtils {
+
+ private static final ImmutableMap<SqlTypeName, Type> CALCITE_TO_ZETA_SIMPLE_TYPES =
+ ImmutableMap.<SqlTypeName, Type>builder()
+ .put(SqlTypeName.BIGINT, TypeFactory.createSimpleType(TYPE_INT64))
+ .put(SqlTypeName.INTEGER, TypeFactory.createSimpleType(TYPE_INT32))
+ .put(SqlTypeName.VARCHAR, TypeFactory.createSimpleType(TYPE_STRING))
+ .put(SqlTypeName.BOOLEAN, TypeFactory.createSimpleType(TYPE_BOOL))
+ .put(SqlTypeName.FLOAT, TypeFactory.createSimpleType(TYPE_FLOAT))
+ .put(SqlTypeName.DOUBLE, TypeFactory.createSimpleType(TYPE_DOUBLE))
+ .put(SqlTypeName.VARBINARY, TypeFactory.createSimpleType(TYPE_BYTES))
+ .put(SqlTypeName.TIMESTAMP, TypeFactory.createSimpleType(TYPE_TIMESTAMP))
+ .put(SqlTypeName.DATE, TypeFactory.createSimpleType(TYPE_DATE))
+ .put(SqlTypeName.TIME, TypeFactory.createSimpleType(TYPE_TIME))
+ .build();
+
+ private static final ImmutableMap<TypeKind, Function<RexBuilder, RelDataType>>
+ ZETA_TO_CALCITE_SIMPLE_TYPES =
+ ImmutableMap.<TypeKind, Function<RexBuilder, RelDataType>>builder()
+ .put(TYPE_NUMERIC, relDataTypeFactory(SqlTypeName.DECIMAL))
+ .put(TYPE_INT32, relDataTypeFactory(SqlTypeName.INTEGER))
+ .put(TYPE_INT64, relDataTypeFactory(SqlTypeName.BIGINT))
+ .put(TYPE_FLOAT, relDataTypeFactory(SqlTypeName.FLOAT))
+ .put(TYPE_DOUBLE, relDataTypeFactory(SqlTypeName.DOUBLE))
+ .put(TYPE_STRING, relDataTypeFactory(SqlTypeName.VARCHAR))
+ .put(TYPE_BOOL, relDataTypeFactory(SqlTypeName.BOOLEAN))
+ .put(TYPE_BYTES, relDataTypeFactory(SqlTypeName.VARBINARY))
+ .put(TYPE_DATE, relDataTypeFactory(SqlTypeName.DATE))
+ .put(TYPE_TIME, relDataTypeFactory(SqlTypeName.TIME))
+ // TODO: handle timestamp with time zone.
+ .put(TYPE_TIMESTAMP, relDataTypeFactory(SqlTypeName.TIMESTAMP))
+ .build();
+
+ /** Returns a type matching the corresponding Calcite type. */
+ static Type toZetaType(RelDataType calciteType) {
+
+ if (CALCITE_TO_ZETA_SIMPLE_TYPES.containsKey(calciteType.getSqlTypeName())) {
+ return CALCITE_TO_ZETA_SIMPLE_TYPES.get(calciteType.getSqlTypeName());
+ }
+
+ switch (calciteType.getSqlTypeName()) {
+ case ARRAY:
+ return TypeFactory.createArrayType(toZetaType(calciteType.getComponentType()));
+ case MAP:
+
+ // it is ok to return a simple type for MAP because MAP only exists in pubsub table which
+ // used to save table attribute.
+ // TODO: have a better way to handle MAP given the fact that ZetaSQL has no MAP type.
+ return TypeFactory.createSimpleType(TypeKind.TYPE_STRING);
+ case ROW:
+ List<StructField> structFields =
+ calciteType.getFieldList().stream()
+ .map(f -> new StructField(f.getName(), toZetaType(f.getType())))
+ .collect(toList());
+
+ return TypeFactory.createStructType(structFields);
+ default:
+ throw new RuntimeException("Unsupported RelDataType: " + calciteType);
+ }
+ }
+
+ public static RelDataType toRelDataType(RexBuilder rexBuilder, Type type, boolean isNullable) {
+ if (type.getKind().equals(TypeKind.TYPE_ARRAY)) {
+ return toArrayRelDataType(rexBuilder, type.asArray(), isNullable);
+ } else if (type.getKind().equals(TypeKind.TYPE_STRUCT)) {
+ return toStructRelDataType(rexBuilder, type.asStruct(), isNullable);
+ } else {
+ // TODO: Check type's nullability?
+ return toSimpleRelDataType(type.getKind(), rexBuilder, isNullable);
+ }
+ }
+
+ public static RelDataType toArrayRelDataType(
+ RexBuilder rexBuilder, ArrayType arrayType, boolean isNullable) {
+ // -1 cardinality means unlimited array size.
+ // TODO: is unlimited array size right for general case?
+ // TODO: whether isNullable should be ArrayType's nullablity (not its element type's?)
+ return rexBuilder
+ .getTypeFactory()
+ .createArrayType(toRelDataType(rexBuilder, arrayType.getElementType(), isNullable), -1);
+ }
+
+ private static RelDataType toStructRelDataType(
+ RexBuilder rexBuilder, StructType structType, boolean isNullable) {
+
+ List<StructField> fields = structType.getFieldList();
+ List<String> fieldNames = fields.stream().map(StructField::getName).collect(toList());
+ List<RelDataType> fieldTypes =
+ fields.stream()
+ .map(f -> toRelDataType(rexBuilder, f.getType(), isNullable))
+ .collect(toList());
+
+ return rexBuilder.getTypeFactory().createStructType(fieldTypes, fieldNames);
+ }
+
+ // TODO: convert TIMESTAMP with/without TIMEZONE and DATETIME.
+ public static RelDataType toSimpleRelDataType(TypeKind kind, RexBuilder rexBuilder) {
+ return toSimpleRelDataType(kind, rexBuilder, true);
+ }
+
+ public static RelDataType toSimpleRelDataType(
+ TypeKind kind, RexBuilder rexBuilder, boolean isNullable) {
+ if (!ZETA_TO_CALCITE_SIMPLE_TYPES.containsKey(kind)) {
+ throw new RuntimeException("Unsupported column type: " + kind);
+ }
+
+ RelDataType relDataType = ZETA_TO_CALCITE_SIMPLE_TYPES.get(kind).apply(rexBuilder);
+ return nullable(rexBuilder, relDataType, isNullable);
+ }
+
+ private static RelDataType nullable(RexBuilder r, RelDataType relDataType, boolean isNullable) {
+ return r.getTypeFactory().createTypeWithNullability(relDataType, isNullable);
+ }
+
+ private static Function<RexBuilder, RelDataType> relDataTypeFactory(SqlTypeName typeName) {
+ return (RexBuilder r) -> r.getTypeFactory().createSqlType(typeName);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLCastFunctionImpl.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLCastFunctionImpl.java
new file mode 100644
index 0000000..3b9eec1
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLCastFunctionImpl.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static org.apache.calcite.adapter.enumerable.RexImpTable.createImplementor;
+
+import java.util.List;
+import org.apache.calcite.adapter.enumerable.CallImplementor;
+import org.apache.calcite.adapter.enumerable.NotNullImplementor;
+import org.apache.calcite.adapter.enumerable.NullPolicy;
+import org.apache.calcite.adapter.enumerable.RexImpTable;
+import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
+import org.apache.calcite.linq4j.tree.Expression;
+import org.apache.calcite.linq4j.tree.Expressions;
+import org.apache.calcite.rex.RexCall;
+import org.apache.calcite.schema.Function;
+import org.apache.calcite.schema.FunctionParameter;
+import org.apache.calcite.schema.ImplementableFunction;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
+
+/** ZetaSQLCastFunctionImpl. */
+public class ZetaSQLCastFunctionImpl implements Function, ImplementableFunction {
+ public static final SqlUserDefinedFunction ZETASQL_CAST_OP =
+ new SqlUserDefinedFunction(
+ new SqlIdentifier("CAST", SqlParserPos.ZERO),
+ null,
+ null,
+ null,
+ null,
+ new ZetaSQLCastFunctionImpl());
+
+ @Override
+ public CallImplementor getImplementor() {
+ return createImplementor(new ZetaSQLCastCallNotNullImplementor(), NullPolicy.STRICT, false);
+ }
+
+ @Override
+ public List<FunctionParameter> getParameters() {
+ return null;
+ }
+
+ private static class ZetaSQLCastCallNotNullImplementor implements NotNullImplementor {
+
+ @Override
+ public Expression implement(
+ RexToLixTranslator rexToLixTranslator, RexCall rexCall, List<Expression> list) {
+ if (rexCall.getOperands().size() != 1 || list.size() != 1) {
+ throw new RuntimeException("CAST should have one operand.");
+ }
+ SqlTypeName toType = rexCall.getType().getSqlTypeName();
+ SqlTypeName fromType = rexCall.getOperands().get(0).getType().getSqlTypeName();
+
+ Expression translatedOperand = list.get(0);
+ Expression convertedOperand;
+ // CAST(BYTES AS STRING) - BINARY to VARCHAR in Calcite
+ if (fromType == SqlTypeName.BINARY && toType == SqlTypeName.VARCHAR) {
+ // operand is literal, which is bytes wrapped in ByteString.
+ // this piece of code is same as
+ // BeamCodegenUtils.toStringUTF8(ByeString.getBytes());
+ convertedOperand =
+ Expressions.call(
+ BeamCodegenUtils.class,
+ "toStringUTF8",
+ Expressions.call(translatedOperand, "getBytes"));
+ } else if (fromType == SqlTypeName.VARBINARY && toType == SqlTypeName.VARCHAR) {
+ // translatedOperand is a byte[]
+ // this piece of code is same as
+ // BeamCodegenUtils.toStringUTF8(byte[]);
+ convertedOperand =
+ Expressions.call(BeamCodegenUtils.class, "toStringUTF8", translatedOperand);
+ } else if (fromType == SqlTypeName.BOOLEAN && toType == SqlTypeName.BIGINT) {
+ convertedOperand =
+ Expressions.condition(
+ translatedOperand,
+ Expressions.constant(1L, Long.class),
+ Expressions.constant(0L, Long.class));
+ } else if (fromType == SqlTypeName.BIGINT && toType == SqlTypeName.BOOLEAN) {
+ convertedOperand = Expressions.notEqual(translatedOperand, Expressions.constant(0));
+ } else if (fromType == SqlTypeName.TIMESTAMP && toType == SqlTypeName.VARCHAR) {
+ convertedOperand =
+ Expressions.call(BeamCodegenUtils.class, "toStringTimestamp", translatedOperand);
+ } else {
+ throw new RuntimeException("Unsupported CAST: " + fromType.name() + " to " + toType.name());
+ }
+
+ // If operand is nullable, wrap in a null check
+ if (rexCall.getOperands().get(0).getType().isNullable()) {
+ convertedOperand =
+ Expressions.condition(
+ Expressions.equal(translatedOperand, RexImpTable.NULL_EXPR),
+ RexImpTable.NULL_EXPR,
+ convertedOperand);
+ }
+
+ return convertedOperand;
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLPlannerImpl.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLPlannerImpl.java
new file mode 100644
index 0000000..09b93f3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLPlannerImpl.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import com.google.common.collect.ImmutableList;
+import com.google.zetasql.Value;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedQueryStmt;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedStatement;
+import java.io.Reader;
+import java.util.Map;
+import java.util.logging.Logger;
+import org.apache.beam.sdk.extensions.sql.zetasql.translation.ConversionContext;
+import org.apache.beam.sdk.extensions.sql.zetasql.translation.ExpressionConverter;
+import org.apache.beam.sdk.extensions.sql.zetasql.translation.QueryStatementConverter;
+import org.apache.calcite.adapter.java.JavaTypeFactory;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexExecutor;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.parser.SqlParseException;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.Program;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.ValidationException;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.Util;
+
+/** ZetaSQLPlannerImpl. */
+public class ZetaSQLPlannerImpl implements Planner {
+ private static final Logger logger = Logger.getLogger(ZetaSQLPlannerImpl.class.getName());
+
+ private final SchemaPlus defaultSchemaPlus;
+
+ // variables that are used in Calcite's planner.
+ private final FrameworkConfig config;
+ private RelOptPlanner planner;
+ private JavaTypeFactory typeFactory;
+ private final RexExecutor executor;
+ private RelOptCluster cluster;
+ private final ImmutableList<Program> programs;
+ private ExpressionConverter expressionConverter;
+
+ private static final long ONE_SECOND_IN_MILLIS = 1000L;
+ private static final long ONE_MINUTE_IN_MILLIS = 60L * ONE_SECOND_IN_MILLIS;
+ private static final long ONE_HOUR_IN_MILLIS = 60L * ONE_MINUTE_IN_MILLIS;
+ private static final long ONE_DAY_IN_MILLIS = 24L * ONE_HOUR_IN_MILLIS;
+
+ @SuppressWarnings("unused")
+ private static final long ONE_MONTH_IN_MILLIS = 30L * ONE_DAY_IN_MILLIS;
+
+ @SuppressWarnings("unused")
+ private static final long ONE_YEAR_IN_MILLIS = 365L * ONE_DAY_IN_MILLIS;
+
+ public ZetaSQLPlannerImpl(FrameworkConfig config) {
+ this.config = config;
+ this.executor = config.getExecutor();
+ this.programs = config.getPrograms();
+
+ Frameworks.withPlanner(
+ (cluster, relOptSchema, rootSchema) -> {
+ Util.discard(rootSchema); // use our own defaultSchema
+ typeFactory = (JavaTypeFactory) cluster.getTypeFactory();
+ planner = cluster.getPlanner();
+ planner.setExecutor(executor);
+ return null;
+ },
+ config);
+
+ this.defaultSchemaPlus = config.getDefaultSchema();
+ }
+
+ @Override
+ public SqlNode parse(String s) throws SqlParseException {
+ return null;
+ }
+
+ @Override
+ public SqlNode parse(Reader reader) throws SqlParseException {
+ return null;
+ }
+
+ @Override
+ public SqlNode validate(SqlNode sqlNode) throws ValidationException {
+ return null;
+ }
+
+ @Override
+ public Pair<SqlNode, RelDataType> validateAndGetType(SqlNode sqlNode) throws ValidationException {
+ throw new RuntimeException("validateAndGetType(SqlNode) is not implemented.");
+ }
+
+ @Override
+ public RelRoot rel(SqlNode sqlNode) throws RelConversionException {
+ return null;
+ }
+
+ public RelRoot rel(String sql, Map<String, Value> params) {
+ this.cluster = RelOptCluster.create(planner, new RexBuilder(typeFactory));
+ this.expressionConverter = new ExpressionConverter(cluster, params);
+
+ QueryTrait trait = new QueryTrait();
+
+ ResolvedStatement statement =
+ SqlAnalyzer.withQueryParams(params)
+ .withQueryTrait(trait)
+ .withCalciteContext(config.getContext())
+ .withTopLevelSchema(defaultSchemaPlus)
+ .withTypeFactory((JavaTypeFactory) cluster.getTypeFactory())
+ .analyze(sql);
+
+ if (!(statement instanceof ResolvedQueryStmt)) {
+ throw new UnsupportedOperationException(
+ "Unsupported query statement type: " + sql.getClass().getSimpleName());
+ }
+
+ ConversionContext context = ConversionContext.of(config, expressionConverter, cluster, trait);
+
+ RelNode convertedNode =
+ QueryStatementConverter.convertRootQuery(context, (ResolvedQueryStmt) statement);
+ return RelRoot.of(convertedNode, SqlKind.ALL);
+ }
+
+ @Override
+ public RelNode convert(SqlNode sqlNode) {
+ throw new RuntimeException("convert(SqlNode) is not implemented.");
+ }
+
+ @Override
+ public RelDataTypeFactory getTypeFactory() {
+ throw new RuntimeException("getTypeFactory() is not implemented.");
+ }
+
+ @Override
+ public RelNode transform(int i, RelTraitSet relTraitSet, RelNode relNode)
+ throws RelConversionException {
+ relNode
+ .getCluster()
+ .setMetadataProvider(
+ new CachingRelMetadataProvider(
+ relNode.getCluster().getMetadataProvider(), relNode.getCluster().getPlanner()));
+ Program program = programs.get(i);
+ return program.run(planner, relNode, relTraitSet, ImmutableList.of(), ImmutableList.of());
+ }
+
+ @Override
+ public void reset() {
+ throw new RuntimeException("reset() is not implemented.");
+ }
+
+ @Override
+ public void close() {
+ // no-op
+ }
+
+ @Override
+ public RelTraitSet getEmptyTraitSet() {
+ throw new RuntimeException("getEmptyTraitSet() is not implemented.");
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLQueryPlanner.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLQueryPlanner.java
new file mode 100644
index 0000000..f767d09
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLQueryPlanner.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import com.google.zetasql.Value;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.impl.JdbcConnection;
+import org.apache.beam.sdk.extensions.sql.impl.ParseException;
+import org.apache.beam.sdk.extensions.sql.impl.QueryPlanner;
+import org.apache.beam.sdk.extensions.sql.impl.SqlConversionException;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamLogicalConvention;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.config.CalciteConnectionConfig;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.prepare.CalciteCatalogReader;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParser;
+import org.apache.calcite.sql.parser.SqlParserImplFactory;
+import org.apache.calcite.sql.util.ChainedSqlOperatorTable;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.RelConversionException;
+import org.apache.calcite.tools.RuleSet;
+
+/** ZetaSQLQueryPlanner. */
+public class ZetaSQLQueryPlanner implements QueryPlanner {
+ private final ZetaSQLPlannerImpl plannerImpl;
+
+ public ZetaSQLQueryPlanner(FrameworkConfig config) {
+ plannerImpl = new ZetaSQLPlannerImpl(config);
+ }
+
+ public ZetaSQLQueryPlanner(JdbcConnection jdbcConnection, RuleSet[] ruleSets) {
+ plannerImpl = new ZetaSQLPlannerImpl(defaultConfig(jdbcConnection, ruleSets));
+ }
+
+ @Override
+ public BeamRelNode convertToBeamRel(String sqlStatement)
+ throws ParseException, SqlConversionException {
+ try {
+ return parseQuery(sqlStatement);
+ } catch (RelConversionException e) {
+ throw new SqlConversionException(e.getCause());
+ }
+ }
+
+ @Override
+ public SqlNode parse(String sqlStatement) throws ParseException {
+ return null;
+ }
+
+ public BeamRelNode convertToBeamRel(String sqlStatement, Map<String, Value> queryParams)
+ throws ParseException, SqlConversionException {
+ try {
+ return parseQuery(sqlStatement, queryParams);
+ } catch (RelConversionException e) {
+ throw new SqlConversionException(e.getCause());
+ }
+ }
+
+ public BeamRelNode parseQuery(String sql) throws RelConversionException {
+ return parseQuery(sql, Collections.emptyMap());
+ }
+
+ public BeamRelNode parseQuery(String sql, Map<String, Value> queryParams)
+ throws RelConversionException {
+ RelRoot root = plannerImpl.rel(sql, queryParams);
+ RelTraitSet desiredTraits =
+ root.rel
+ .getTraitSet()
+ .replace(BeamLogicalConvention.INSTANCE)
+ .replace(root.collation)
+ .simplify();
+ BeamRelNode beamRelNode = (BeamRelNode) plannerImpl.transform(0, desiredTraits, root.rel);
+ return beamRelNode;
+ }
+
+ private FrameworkConfig defaultConfig(JdbcConnection connection, RuleSet[] ruleSets) {
+ final CalciteConnectionConfig config = connection.config();
+ final SqlParser.ConfigBuilder parserConfig =
+ SqlParser.configBuilder()
+ .setQuotedCasing(config.quotedCasing())
+ .setUnquotedCasing(config.unquotedCasing())
+ .setQuoting(config.quoting())
+ .setConformance(config.conformance())
+ .setCaseSensitive(config.caseSensitive());
+ final SqlParserImplFactory parserFactory =
+ config.parserFactory(SqlParserImplFactory.class, null);
+ if (parserFactory != null) {
+ parserConfig.setParserFactory(parserFactory);
+ }
+
+ final SchemaPlus schema = connection.getRootSchema();
+ final SchemaPlus defaultSchema = connection.getCurrentSchemaPlus();
+
+ final ImmutableList<RelTraitDef> traitDefs = ImmutableList.of(ConventionTraitDef.INSTANCE);
+
+ final CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ CalciteSchema.from(schema),
+ ImmutableList.of(defaultSchema.getName()),
+ connection.getTypeFactory(),
+ connection.config());
+ final SqlOperatorTable opTab0 =
+ connection.config().fun(SqlOperatorTable.class, SqlStdOperatorTable.instance());
+
+ return Frameworks.newConfigBuilder()
+ .parserConfig(parserConfig.build())
+ .defaultSchema(defaultSchema)
+ .traitDefs(traitDefs)
+ .context(Contexts.of(connection.config()))
+ .ruleSets(ruleSets)
+ .costFactory(null)
+ .typeSystem(connection.getTypeFactory().getTypeSystem())
+ .operatorTable(ChainedSqlOperatorTable.of(opTab0, catalogReader))
+ .build();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlIdUtils.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlIdUtils.java
new file mode 100644
index 0000000..8529241
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlIdUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static java.util.stream.Collectors.joining;
+
+import java.util.List;
+import java.util.regex.Pattern;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+
+/** Utils to work with ZetaSQL-compatible IDs. */
+class ZetaSqlIdUtils {
+
+ /**
+ * Some special characters we explcitily handle.
+ *
+ * <p>Everything else is ignored, e.g. tabs, newlines, etc.
+ */
+ private static final Pattern SPECIAL_CHARS_ESCAPE =
+ Pattern.compile(
+ "(?<SpecialChar>["
+ + "\\\\" // slash
+ + "`" // backtick
+ + "'" // single quote
+ + "\"" // double quote
+ + "?" // question mark
+ + "])");
+
+ private static final ImmutableMap<String, String> WHITESPACES =
+ ImmutableMap.of(
+ "\n", "\\\\n",
+ "\t", "\\\\t",
+ "\r", "\\\\r",
+ "\f", "\\\\f");
+
+ private static final Pattern SIMPLE_ID = Pattern.compile("[A-Za-z_][A-Za-z_0-9]*");
+
+ /**
+ * Joins parts into a single compound ZetaSQL identifier.
+ *
+ * <p>Escapes backticks, slashes, double and single quotes, doesn't handle other special
+ * characters for now.
+ */
+ static String escapeAndJoin(List<String> parts) {
+ String escaped =
+ parts.stream()
+ .map(ZetaSqlIdUtils::escapeSpecialChars)
+ .map(ZetaSqlIdUtils::replaceWhitespaces)
+ .map(ZetaSqlIdUtils::backtickIfNeeded)
+ .collect(joining("."));
+ return escaped;
+ }
+
+ private static String escapeSpecialChars(String str) {
+ return SPECIAL_CHARS_ESCAPE.matcher(str).replaceAll("\\\\${SpecialChar}");
+ }
+
+ private static String replaceWhitespaces(String s) {
+ String result = s;
+ for (String whitespace : WHITESPACES.keySet()) {
+ result = result.replaceAll(whitespace, WHITESPACES.get(whitespace));
+ }
+ return result;
+ }
+
+ private static String backtickIfNeeded(String s) {
+ return SIMPLE_ID.matcher(s).matches() ? s : ("`" + s + "`");
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/package-info.java
new file mode 100644
index 0000000..81438fd
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/package-info.java
@@ -0,0 +1,2 @@
+/** ZetaSQL Dialect package. */
+package org.apache.beam.sdk.extensions.sql.zetasql;
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java
new file mode 100644
index 0000000..9f7b472
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/AggregateScanConverter.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_CAST;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_COLUMN_REF;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_GET_STRUCT_FIELD;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TypeUtils.toSimpleRelDataType;
+
+import com.google.zetasql.FunctionSignature;
+import com.google.zetasql.ZetaSQLType.TypeKind;
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedAggregateFunctionCall;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedAggregateScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedComputedColumn;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedExpr;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.extensions.sql.zetasql.SqlStdOperatorMappingTable;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.ImmutableBitSet;
+
+/** Converts aggregate calls. */
+class AggregateScanConverter extends RelConverter<ResolvedAggregateScan> {
+ private static final String AVG_ILLEGAL_LONG_INPUT_TYPE =
+ "AVG(LONG) is not supported. You might want to use AVG(CAST(expression AS DOUBLE).";
+
+ AggregateScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedAggregateScan zetaNode) {
+ return Collections.singletonList(zetaNode.getInputScan());
+ }
+
+ @Override
+ public RelNode convert(ResolvedAggregateScan zetaNode, List<RelNode> inputs) {
+ RelNode input = convertAggregateScanInputScanToLogicalProject(zetaNode, inputs.get(0));
+
+ // Calcite LogicalAggregate's GroupSet is indexes of group fields starting from 0.
+ int groupFieldsListSize = zetaNode.getGroupByList().size();
+ ImmutableBitSet groupSet;
+ if (groupFieldsListSize != 0) {
+ groupSet =
+ ImmutableBitSet.of(
+ IntStream.rangeClosed(0, groupFieldsListSize - 1)
+ .boxed()
+ .collect(Collectors.toList()));
+ } else {
+ groupSet = ImmutableBitSet.of();
+ }
+
+ // TODO: add support for indicator
+
+ List<AggregateCall> aggregateCalls;
+ if (zetaNode.getAggregateList().isEmpty()) {
+ aggregateCalls = ImmutableList.of();
+ } else {
+ aggregateCalls = new ArrayList<>();
+ // For aggregate calls, their input ref follow after GROUP BY input ref.
+ int columnRefoff = groupFieldsListSize;
+ for (ResolvedComputedColumn computedColumn : zetaNode.getAggregateList()) {
+ aggregateCalls.add(convertAggCall(computedColumn, columnRefoff));
+ columnRefoff++;
+ }
+ }
+
+ LogicalAggregate logicalAggregate =
+ new LogicalAggregate(
+ getCluster(),
+ input.getTraitSet(),
+ input,
+ false,
+ groupSet,
+ ImmutableList.of(groupSet),
+ aggregateCalls);
+
+ return logicalAggregate;
+ }
+
+ private RelNode convertAggregateScanInputScanToLogicalProject(
+ ResolvedAggregateScan node, RelNode input) {
+ // AggregateScan's input is the source of data (e.g. TableScan), which is different from the
+ // design of CalciteSQL, in which the LogicalAggregate's input is a LogicalProject, whose input
+ // is a LogicalTableScan. When AggregateScan's input is WithRefScan, the WithRefScan is
+ // ebullient to a LogicalTableScan. So it's still required to build another LogicalProject as
+ // the input of LogicalAggregate.
+ List<RexNode> projects = new ArrayList<>();
+ List<String> fieldNames = new ArrayList<>();
+
+ // LogicalProject has a list of expr, which including UDF in GROUP BY clause for
+ // LogicalAggregate.
+ for (ResolvedComputedColumn computedColumn : node.getGroupByList()) {
+ projects.add(
+ getExpressionConverter()
+ .convertRexNodeFromResolvedExpr(
+ computedColumn.getExpr(),
+ node.getInputScan().getColumnList(),
+ input.getRowType().getFieldList()));
+ fieldNames.add(getTrait().resolveAlias(computedColumn.getColumn()));
+ }
+
+ // LogicalProject should also include columns used by aggregate functions. These columns should
+ // follow after GROUP BY columns.
+ // TODO: remove duplicate columns in projects.
+ for (ResolvedComputedColumn resolvedComputedColumn : node.getAggregateList()) {
+ // Should create Calcite's RexInputRef from ResolvedColumn from ResolvedComputedColumn.
+ // TODO: handle aggregate function with more than one argument and handle OVER
+ // TODO: is there is general way for column reference tracking and deduplication for
+ // aggregation?
+ ResolvedAggregateFunctionCall aggregateFunctionCall =
+ ((ResolvedAggregateFunctionCall) resolvedComputedColumn.getExpr());
+ if (aggregateFunctionCall.getArgumentList() != null
+ && aggregateFunctionCall.getArgumentList().size() == 1) {
+ ResolvedExpr resolvedExpr = aggregateFunctionCall.getArgumentList().get(0);
+
+ // TODO: assume aggregate function's input is either a ColumnRef or a cast(ColumnRef).
+ // TODO: user might use multiple CAST so we need to handle this rare case.
+ projects.add(
+ getExpressionConverter()
+ .convertRexNodeFromResolvedExpr(
+ resolvedExpr,
+ node.getInputScan().getColumnList(),
+ input.getRowType().getFieldList()));
+ fieldNames.add(getTrait().resolveAlias(resolvedComputedColumn.getColumn()));
+ } else if (aggregateFunctionCall.getArgumentList() != null
+ && aggregateFunctionCall.getArgumentList().size() > 1) {
+ throw new RuntimeException(
+ aggregateFunctionCall.getFunction().getName() + " has more than one argument.");
+ }
+ }
+
+ return LogicalProject.create(input, projects, fieldNames);
+ }
+
+ private AggregateCall convertAggCall(ResolvedComputedColumn computedColumn, int columnRefOff) {
+ ResolvedAggregateFunctionCall aggregateFunctionCall =
+ (ResolvedAggregateFunctionCall) computedColumn.getExpr();
+
+ // Reject AVG(INT64)
+ if (aggregateFunctionCall.getFunction().getName().equals("avg")) {
+ FunctionSignature signature = aggregateFunctionCall.getSignature();
+ if (signature
+ .getFunctionArgumentList()
+ .get(0)
+ .getType()
+ .getKind()
+ .equals(TypeKind.TYPE_INT64)) {
+ throw new RuntimeException(AVG_ILLEGAL_LONG_INPUT_TYPE);
+ }
+ }
+
+ // Reject aggregation DISTINCT
+ if (aggregateFunctionCall.getDistinct()) {
+ throw new RuntimeException(
+ "Does not support "
+ + aggregateFunctionCall.getFunction().getSqlName()
+ + " DISTINCT. 'SELECT DISTINCT' syntax could be used to deduplicate before"
+ + " aggregation.");
+ }
+
+ SqlAggFunction sqlAggFunction =
+ (SqlAggFunction)
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get(
+ aggregateFunctionCall.getFunction().getName());
+ if (sqlAggFunction == null) {
+ throw new RuntimeException(
+ "Does not support ZetaSQL aggregate function: "
+ + aggregateFunctionCall.getFunction().getName());
+ }
+
+ List<Integer> argList = new ArrayList<>();
+ for (ResolvedExpr expr :
+ ((ResolvedAggregateFunctionCall) computedColumn.getExpr()).getArgumentList()) {
+ // Throw an error if aggregate function's input isn't either a ColumnRef or a cast(ColumnRef).
+ // TODO: is there a general way to handle aggregation calls conversion?
+ if (expr.nodeKind() == RESOLVED_CAST
+ || expr.nodeKind() == RESOLVED_COLUMN_REF
+ || expr.nodeKind() == RESOLVED_GET_STRUCT_FIELD) {
+ argList.add(columnRefOff);
+ } else {
+ throw new RuntimeException(
+ "Aggregate function only accepts Column Reference or CAST(Column Reference) as its"
+ + " input.");
+ }
+ }
+
+ // TODO: there should be a general way to decide if a return type of a aggcall is nullable.
+ RelDataType returnType;
+ if (sqlAggFunction.equals(SqlStdOperatorTable.ANY_VALUE)) {
+ returnType =
+ toSimpleRelDataType(
+ computedColumn.getColumn().getType().getKind(), getCluster().getRexBuilder(), true);
+ } else {
+ returnType =
+ toSimpleRelDataType(
+ computedColumn.getColumn().getType().getKind(), getCluster().getRexBuilder(), false);
+ }
+
+ String aggName = getTrait().resolveAlias(computedColumn.getColumn());
+ return AggregateCall.create(sqlAggFunction, false, false, argList, -1, returnType, aggName);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToJoinConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToJoinConverter.java
new file mode 100644
index 0000000..66e6bc3
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToJoinConverter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedArrayScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedColumnRef;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.CorrelationId;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Uncollect;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalValues;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts array scan that represents join of an uncollect(array_field) to uncollect. */
+class ArrayScanToJoinConverter extends RelConverter<ResolvedArrayScan> {
+
+ ArrayScanToJoinConverter(ConversionContext context) {
+ super(context);
+ }
+
+ /** This is the case of {@code table [LEFT|INNER] JOIN UNNEST(table.array_field) on join_expr}. */
+ @Override
+ public boolean canConvert(ResolvedArrayScan zetaNode) {
+ return zetaNode.getInputScan() != null && zetaNode.getJoinExpr() != null;
+ }
+
+ /** Left input is converted from input scan. */
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedArrayScan zetaNode) {
+ return Collections.singletonList(zetaNode.getInputScan());
+ }
+
+ /** Returns a LogicJoin. */
+ @Override
+ public RelNode convert(ResolvedArrayScan zetaNode, List<RelNode> inputs) {
+ List<RexNode> projects = new ArrayList<>();
+
+ RelNode leftInput = inputs.get(0);
+
+ ResolvedColumnRef columnRef = (ResolvedColumnRef) zetaNode.getArrayExpr();
+ CorrelationId correlationId = getCluster().createCorrel();
+ getCluster().getQuery().mapCorrel(correlationId.getName(), leftInput);
+ String columnName =
+ String.format(
+ "%s%s",
+ zetaNode.getElementColumn().getTableName(), zetaNode.getElementColumn().getName());
+
+ projects.add(
+ getCluster()
+ .getRexBuilder()
+ .makeFieldAccess(
+ getCluster().getRexBuilder().makeCorrel(leftInput.getRowType(), correlationId),
+ getExpressionConverter()
+ .indexOfProjectionColumnRef(
+ columnRef.getColumn().getId(), zetaNode.getInputScan().getColumnList())));
+
+ RelNode projectNode =
+ LogicalProject.create(
+ LogicalValues.createOneRow(getCluster()), projects, ImmutableList.of(columnName));
+
+ // Create an UnCollect
+ // TODO: how to handle ordinality.
+ Uncollect uncollectNode = Uncollect.create(projectNode.getTraitSet(), projectNode, false);
+ // The InputRef should only be 0 because Uncollect has only one field.
+ RelNode rightInput =
+ LogicalProject.create(
+ uncollectNode,
+ ImmutableList.of(getCluster().getRexBuilder().makeInputRef(uncollectNode, 0)),
+ ImmutableList.of(columnName));
+
+ // Join condition should be a RexNode converted from join_expr.
+ RexNode condition =
+ getExpressionConverter().convertRexNodeFromResolvedExpr(zetaNode.getJoinExpr());
+ JoinRelType joinRelType = zetaNode.getIsOuter() ? JoinRelType.LEFT : JoinRelType.INNER;
+
+ return LogicalJoin.create(leftInput, rightInput, condition, ImmutableSet.of(), joinRelType);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToUncollectConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToUncollectConverter.java
new file mode 100644
index 0000000..55a42ce
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ArrayScanToUncollectConverter.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedArrayScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedLiteral;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Uncollect;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalValues;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts array scan that represents an array literal to uncollect. */
+class ArrayScanToUncollectConverter extends RelConverter<ResolvedArrayScan> {
+
+ ArrayScanToUncollectConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean canConvert(ResolvedArrayScan zetaNode) {
+ return zetaNode.getInputScan() == null;
+ }
+
+ @Override
+ public RelNode convert(ResolvedArrayScan zetaNode, List<RelNode> inputs) {
+ RexNode arrayLiteralExpression =
+ getExpressionConverter().convertResolvedLiteral((ResolvedLiteral) zetaNode.getArrayExpr());
+
+ String fieldName =
+ String.format(
+ "%s%s",
+ zetaNode.getElementColumn().getTableName(), zetaNode.getElementColumn().getName());
+
+ RelNode projectNode =
+ LogicalProject.create(
+ LogicalValues.createOneRow(getCluster()),
+ Collections.singletonList(arrayLiteralExpression),
+ ImmutableList.of(fieldName));
+
+ // TODO: how to handle ordinarily.
+ return Uncollect.create(projectNode.getTraitSet(), projectNode, false);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ConversionContext.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ConversionContext.java
new file mode 100644
index 0000000..be367d0
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ConversionContext.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import org.apache.beam.sdk.extensions.sql.zetasql.QueryTrait;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.tools.FrameworkConfig;
+
+/** Conversion context, some rules need this data to convert the nodes. */
+public class ConversionContext {
+ private final FrameworkConfig config;
+ private final ExpressionConverter expressionConverter;
+ private final RelOptCluster cluster;
+ private final QueryTrait trait;
+
+ public static ConversionContext of(
+ FrameworkConfig config,
+ ExpressionConverter expressionConverter,
+ RelOptCluster cluster,
+ QueryTrait trait) {
+ return new ConversionContext(config, expressionConverter, cluster, trait);
+ }
+
+ private ConversionContext(
+ FrameworkConfig config,
+ ExpressionConverter expressionConverter,
+ RelOptCluster cluster,
+ QueryTrait trait) {
+ this.config = config;
+ this.expressionConverter = expressionConverter;
+ this.cluster = cluster;
+ this.trait = trait;
+ }
+
+ FrameworkConfig getConfig() {
+ return config;
+ }
+
+ ExpressionConverter getExpressionConverter() {
+ return expressionConverter;
+ }
+
+ RelOptCluster cluster() {
+ return cluster;
+ }
+
+ QueryTrait getTrait() {
+ return trait;
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java
new file mode 100644
index 0000000..8a75a35
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java
@@ -0,0 +1,1019 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_FUNCTION_CALL;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_BOOL;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_BYTES;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_DOUBLE;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_INT64;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_STRING;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_TIMESTAMP;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.convertDateValueToDateString;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.convertTimeValueToTimeString;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.safeMicrosToMillis;
+import static org.apache.beam.sdk.extensions.sql.zetasql.SqlStdOperatorMappingTable.FUNCTION_FAMILY_DATE_ADD;
+import static org.apache.beam.sdk.extensions.sql.zetasql.ZetaSQLCastFunctionImpl.ZETASQL_CAST_OP;
+
+import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.zetasql.ArrayType;
+import com.google.zetasql.Type;
+import com.google.zetasql.Value;
+import com.google.zetasql.ZetaSQLType.TypeKind;
+import com.google.zetasql.functions.ZetaSQLDateTime.DateTimestampPart;
+import com.google.zetasql.resolvedast.ResolvedColumn;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedAggregateScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedCast;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedColumnRef;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedComputedColumn;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedExpr;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedFunctionCall;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedGetStructField;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedLiteral;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedOrderByScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedParameter;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedProjectScan;
+import io.grpc.Status;
+import java.math.BigDecimal;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.extensions.sql.zetasql.SqlOperatorRewriter;
+import org.apache.beam.sdk.extensions.sql.zetasql.SqlOperators;
+import org.apache.beam.sdk.extensions.sql.zetasql.SqlStdOperatorMappingTable;
+import org.apache.beam.sdk.extensions.sql.zetasql.TypeUtils;
+import org.apache.calcite.avatica.util.ByteString;
+import org.apache.calcite.avatica.util.TimeUnit;
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.util.TimestampString;
+
+/**
+ * Extracts expressions (function calls, field accesses) from the resolve query nodes, converts them
+ * to RexNodes.
+ */
+@Internal
+public class ExpressionConverter {
+
+ private static final String PRE_DEFINED_WINDOW_FUNCTIONS = "pre_defined_window_functions";
+
+ // Constants of pre-defined functions.
+ private static final String WINDOW_START = "_START";
+ private static final String WINDOW_END = "_END";
+ private static final String FIXED_WINDOW = "TUMBLE";
+ private static final String FIXED_WINDOW_START = FIXED_WINDOW + WINDOW_START;
+ private static final String FIXED_WINDOW_END = FIXED_WINDOW + WINDOW_END;
+ private static final String SLIDING_WINDOW = "HOP";
+ private static final String SLIDING_WINDOW_START = SLIDING_WINDOW + WINDOW_START;
+ private static final String SLIDING_WINDOW_END = SLIDING_WINDOW + WINDOW_END;
+ private static final String SESSION_WINDOW = "SESSION";
+ private static final String SESSION_WINDOW_START = SESSION_WINDOW + WINDOW_START;
+ private static final String SESSION_WINDOW_END = SESSION_WINDOW + WINDOW_END;
+
+ private static final ImmutableMap<String, String> WINDOW_START_END_TO_WINDOW_MAP =
+ ImmutableMap.<String, String>builder()
+ .put(FIXED_WINDOW_START, FIXED_WINDOW)
+ .put(FIXED_WINDOW_END, FIXED_WINDOW)
+ .put(SLIDING_WINDOW_START, SLIDING_WINDOW)
+ .put(SLIDING_WINDOW_END, SLIDING_WINDOW)
+ .put(SESSION_WINDOW_START, SESSION_WINDOW)
+ .put(SESSION_WINDOW_END, SESSION_WINDOW)
+ .build();
+
+ private static final ImmutableSet<String> WINDOW_START_END_FUNCTION_SET =
+ ImmutableSet.of(
+ FIXED_WINDOW_START,
+ FIXED_WINDOW_END,
+ SLIDING_WINDOW_START,
+ SLIDING_WINDOW_END,
+ SESSION_WINDOW_START,
+ SESSION_WINDOW_END);
+
+ private static final ImmutableMap<TypeKind, ImmutableSet<TypeKind>> UNSUPPORTED_CASTING =
+ ImmutableMap.<TypeKind, ImmutableSet<TypeKind>>builder()
+ .put(TYPE_INT64, ImmutableSet.of(TYPE_DOUBLE))
+ .put(TYPE_BOOL, ImmutableSet.of(TYPE_STRING))
+ .put(TYPE_STRING, ImmutableSet.of(TYPE_BOOL, TYPE_DOUBLE))
+ .build();
+
+ private static final ImmutableMap<Integer, TimeUnit> TIME_UNIT_CASTING_MAP =
+ ImmutableMap.<Integer, TimeUnit>builder()
+ .put(DateTimestampPart.YEAR.getNumber(), TimeUnit.YEAR)
+ .put(DateTimestampPart.MONTH.getNumber(), TimeUnit.MONTH)
+ .put(DateTimestampPart.DAY.getNumber(), TimeUnit.DAY)
+ .put(DateTimestampPart.DAYOFWEEK.getNumber(), TimeUnit.DOW)
+ .put(DateTimestampPart.DAYOFYEAR.getNumber(), TimeUnit.DOY)
+ .put(DateTimestampPart.QUARTER.getNumber(), TimeUnit.QUARTER)
+ .put(DateTimestampPart.HOUR.getNumber(), TimeUnit.HOUR)
+ .put(DateTimestampPart.MINUTE.getNumber(), TimeUnit.MINUTE)
+ .put(DateTimestampPart.SECOND.getNumber(), TimeUnit.SECOND)
+ .put(DateTimestampPart.MILLISECOND.getNumber(), TimeUnit.MILLISECOND)
+ .put(DateTimestampPart.MICROSECOND.getNumber(), TimeUnit.MICROSECOND)
+ .put(DateTimestampPart.NANOSECOND.getNumber(), TimeUnit.NANOSECOND)
+ .put(DateTimestampPart.ISOYEAR.getNumber(), TimeUnit.ISOYEAR)
+ .put(DateTimestampPart.ISOWEEK.getNumber(), TimeUnit.WEEK)
+ .build();
+
+ private static final ImmutableSet<String> DATE_PART_UNITS_TO_MILLIS =
+ ImmutableSet.of("DAY", "HOUR", "MINUTE", "SECOND");
+ private static final ImmutableSet<String> DATE_PART_UNITS_TO_MONTHS = ImmutableSet.of("YEAR");
+
+ private static final long ONE_SECOND_IN_MILLIS = 1000L;
+ private static final long ONE_MINUTE_IN_MILLIS = 60L * ONE_SECOND_IN_MILLIS;
+ private static final long ONE_HOUR_IN_MILLIS = 60L * ONE_MINUTE_IN_MILLIS;
+ private static final long ONE_DAY_IN_MILLIS = 24L * ONE_HOUR_IN_MILLIS;
+
+ @SuppressWarnings("unused")
+ private static final long ONE_MONTH_IN_MILLIS = 30L * ONE_DAY_IN_MILLIS;
+
+ @SuppressWarnings("unused")
+ private static final long ONE_YEAR_IN_MILLIS = 365L * ONE_DAY_IN_MILLIS;
+
+ // Constants of error messages.
+ private static final String INTERVAL_DATE_PART_MSG =
+ "YEAR, QUARTER, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND, MILLISECOND";
+ private static final String INTERVAL_FORMAT_MSG =
+ "INTERVAL should be set as a STRING in the specific format: \"INTERVAL int64 date_part\"."
+ + " The date_part includes: "
+ + INTERVAL_DATE_PART_MSG;
+
+ private final RelOptCluster cluster;
+ private final Map<String, Value> queryParams;
+
+ public ExpressionConverter(RelOptCluster cluster, Map<String, Value> params) {
+ this.cluster = cluster;
+ this.queryParams = params;
+ }
+
+ /** Extract expressions from a project scan node. */
+ public List<RexNode> retrieveRexNode(ResolvedProjectScan node, List<RelDataTypeField> fieldList) {
+ List<RexNode> ret = new ArrayList<>();
+
+ for (ResolvedColumn column : node.getColumnList()) {
+ int index = -1;
+ if ((index = indexOfResolvedColumnInExprList(node.getExprList(), column)) != -1) {
+ ResolvedComputedColumn computedColumn = node.getExprList().get(index);
+ int windowFieldIndex = -1;
+ if (computedColumn.getExpr().nodeKind() == RESOLVED_FUNCTION_CALL) {
+ String functionName =
+ ((ResolvedFunctionCall) computedColumn.getExpr()).getFunction().getName();
+ if (WINDOW_START_END_FUNCTION_SET.contains(functionName)) {
+ ResolvedAggregateScan resolvedAggregateScan =
+ (ResolvedAggregateScan) node.getInputScan();
+ windowFieldIndex =
+ indexOfWindowField(
+ resolvedAggregateScan.getGroupByList(),
+ resolvedAggregateScan.getColumnList(),
+ WINDOW_START_END_TO_WINDOW_MAP.get(functionName));
+ }
+ }
+ ret.add(
+ convertRexNodeFromComputedColumnWithFieldList(
+ computedColumn, node.getInputScan().getColumnList(), fieldList, windowFieldIndex));
+ } else {
+ // ResolvedColumn is not a expression, which means it has to be an input column reference.
+ index = indexOfProjectionColumnRef(column.getId(), node.getInputScan().getColumnList());
+ if (index < 0 || index >= node.getInputScan().getColumnList().size()) {
+ throw new RuntimeException(
+ String.format("Cannot find %s in fieldList %s", column, fieldList));
+ }
+
+ ret.add(rexBuilder().makeInputRef(fieldList.get(index).getType(), index));
+ }
+ }
+ return ret;
+ }
+
+ /** Extract expressions from order by scan node. */
+ public List<RexNode> retrieveRexNodeFromOrderByScan(
+ RelOptCluster cluster, ResolvedOrderByScan node, List<RelDataTypeField> fieldList) {
+ final RexBuilder rexBuilder = cluster.getRexBuilder();
+ List<RexNode> ret = new ArrayList<>();
+
+ for (ResolvedColumn column : node.getColumnList()) {
+ int index = indexOfProjectionColumnRef(column.getId(), node.getInputScan().getColumnList());
+ ret.add(rexBuilder.makeInputRef(fieldList.get(index).getType(), index));
+ }
+
+ return ret;
+ }
+
+ private static int indexOfResolvedColumnInExprList(
+ ImmutableList<ResolvedComputedColumn> exprList, ResolvedColumn column) {
+ if (exprList == null || exprList.isEmpty()) {
+ return -1;
+ }
+
+ for (int i = 0; i < exprList.size(); i++) {
+ ResolvedComputedColumn computedColumn = exprList.get(i);
+ if (computedColumn.getColumn().equals(column)) {
+ return i;
+ }
+ }
+
+ return -1;
+ }
+
+ private static int indexOfWindowField(
+ List<ResolvedComputedColumn> groupByList, List<ResolvedColumn> columnList, String windowFn) {
+ for (ResolvedComputedColumn groupByComputedColumn : groupByList) {
+ if (groupByComputedColumn.getExpr().nodeKind() == RESOLVED_FUNCTION_CALL) {
+ ResolvedFunctionCall functionCall = (ResolvedFunctionCall) groupByComputedColumn.getExpr();
+ if (functionCall.getFunction().getName().equals(windowFn)) {
+ int ret =
+ indexOfResolvedColumnInColumnList(columnList, groupByComputedColumn.getColumn());
+ if (ret == -1) {
+ throw new RuntimeException("Cannot find " + windowFn + " in " + groupByList);
+ } else {
+ return ret;
+ }
+ }
+ }
+ }
+
+ throw new RuntimeException("Cannot find " + windowFn + " in " + groupByList);
+ }
+
+ private static int indexOfResolvedColumnInColumnList(
+ List<ResolvedColumn> columnList, ResolvedColumn column) {
+ if (columnList == null || columnList.isEmpty()) {
+ return -1;
+ }
+
+ for (int i = 0; i < columnList.size(); i++) {
+ if (columnList.get(i).equals(column)) {
+ return i;
+ }
+ }
+
+ return -1;
+ }
+
+ /** Create a RexNode for a corresponding resolved expression node. */
+ public RexNode convertRexNodeFromResolvedExpr(
+ ResolvedExpr expr, List<ResolvedColumn> columnList, List<RelDataTypeField> fieldList) {
+ if (columnList == null || fieldList == null) {
+ return convertRexNodeFromResolvedExpr(expr);
+ }
+
+ RexNode ret;
+
+ switch (expr.nodeKind()) {
+ case RESOLVED_LITERAL:
+ ret = convertResolvedLiteral((ResolvedLiteral) expr);
+ break;
+ case RESOLVED_COLUMN_REF:
+ ret = convertResolvedColumnRef((ResolvedColumnRef) expr, columnList, fieldList);
+ break;
+ case RESOLVED_FUNCTION_CALL:
+ ret = convertResolvedFunctionCall((ResolvedFunctionCall) expr, columnList, fieldList);
+ break;
+ case RESOLVED_CAST:
+ ret = convertResolvedCast((ResolvedCast) expr, columnList, fieldList);
+ break;
+ case RESOLVED_PARAMETER:
+ ret = convertResolvedParameter((ResolvedParameter) expr);
+ break;
+ case RESOLVED_GET_STRUCT_FIELD:
+ ret =
+ convertResolvedStructFieldAccess((ResolvedGetStructField) expr, columnList, fieldList);
+ break;
+ default:
+ ret = convertRexNodeFromResolvedExpr(expr);
+ }
+
+ return ret;
+ }
+
+ /** Create a RexNode for a corresponding resolved expression. */
+ public RexNode convertRexNodeFromResolvedExpr(ResolvedExpr expr) {
+ RexNode ret;
+
+ switch (expr.nodeKind()) {
+ case RESOLVED_LITERAL:
+ ret = convertResolvedLiteral((ResolvedLiteral) expr);
+ break;
+ case RESOLVED_COLUMN_REF:
+ ret = convertResolvedColumnRef((ResolvedColumnRef) expr);
+ break;
+ case RESOLVED_FUNCTION_CALL:
+ // TODO: is there a better way to shared code for different cases of
+ // convertResolvedFunctionCall than passing into two nulls?
+ ret = convertResolvedFunctionCall((ResolvedFunctionCall) expr, null, null);
+ break;
+ case RESOLVED_CAST:
+ ret = convertResolvedCast((ResolvedCast) expr, null, null);
+ break;
+ case RESOLVED_PARAMETER:
+ ret = convertResolvedParameter((ResolvedParameter) expr);
+ break;
+ case RESOLVED_GET_STRUCT_FIELD:
+ ret = convertResolvedStructFieldAccess((ResolvedGetStructField) expr);
+ break;
+ case RESOLVED_SUBQUERY_EXPR:
+ throw new IllegalArgumentException("Does not support sub-queries");
+ default:
+ throw new RuntimeException("Does not support expr node kind " + expr.nodeKind());
+ }
+
+ return ret;
+ }
+
+ /** Extract the RexNode from expression with ref scan. */
+ public RexNode convertRexNodeFromResolvedExprWithRefScan(
+ ResolvedExpr expr,
+ List<ResolvedColumn> refScanLeftColumnList,
+ List<RelDataTypeField> leftFieldList,
+ List<ResolvedColumn> originalLeftColumnList,
+ List<ResolvedColumn> refScanRightColumnList,
+ List<RelDataTypeField> rightFieldList,
+ List<ResolvedColumn> originalRightColumnList) {
+ RexNode ret;
+
+ switch (expr.nodeKind()) {
+ case RESOLVED_LITERAL:
+ ret = convertResolvedLiteral((ResolvedLiteral) expr);
+ break;
+ case RESOLVED_COLUMN_REF:
+ ResolvedColumnRef columnRef = (ResolvedColumnRef) expr;
+ // first look for column ref on the left side
+ ret =
+ convertRexNodeFromResolvedColumnRefWithRefScan(
+ columnRef, refScanLeftColumnList, originalLeftColumnList, leftFieldList);
+
+ // if not found there look on the right
+ if (ret == null) {
+ ret =
+ convertRexNodeFromResolvedColumnRefWithRefScan(
+ columnRef, refScanRightColumnList, originalRightColumnList, rightFieldList);
+ }
+
+ break;
+ case RESOLVED_FUNCTION_CALL:
+ // JOIN only support equal join.
+ ResolvedFunctionCall resolvedFunctionCall = (ResolvedFunctionCall) expr;
+ List<RexNode> operands = new ArrayList<>();
+
+ for (ResolvedExpr resolvedExpr : resolvedFunctionCall.getArgumentList()) {
+ operands.add(
+ convertRexNodeFromResolvedExprWithRefScan(
+ resolvedExpr,
+ refScanLeftColumnList,
+ leftFieldList,
+ originalLeftColumnList,
+ refScanRightColumnList,
+ rightFieldList,
+ originalRightColumnList));
+ }
+
+ SqlOperator op =
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get(
+ resolvedFunctionCall.getFunction().getName());
+ ret = rexBuilder().makeCall(op, operands);
+ break;
+ case RESOLVED_CAST:
+ ResolvedCast resolvedCast = (ResolvedCast) expr;
+ RexNode operand =
+ convertRexNodeFromResolvedExprWithRefScan(
+ resolvedCast.getExpr(),
+ refScanLeftColumnList,
+ leftFieldList,
+ originalLeftColumnList,
+ refScanRightColumnList,
+ rightFieldList,
+ originalRightColumnList);
+
+ TypeKind fromType = resolvedCast.getExpr().getType().getKind();
+ TypeKind toType = resolvedCast.getType().getKind();
+ isCastingSupported(fromType, toType);
+
+ RelDataType outputType =
+ TypeUtils.toSimpleRelDataType(toType, rexBuilder(), operand.getType().isNullable());
+
+ if (isZetaSQLCast(fromType, toType)) {
+ ret = rexBuilder().makeCall(outputType, ZETASQL_CAST_OP, ImmutableList.of(operand));
+ } else {
+ ret = rexBuilder().makeCast(outputType, operand);
+ }
+ break;
+ default:
+ throw new RuntimeException("Does not support expr node kind " + expr.nodeKind());
+ }
+
+ return ret;
+ }
+
+ private RexNode convertRexNodeFromComputedColumnWithFieldList(
+ ResolvedComputedColumn column,
+ List<ResolvedColumn> columnList,
+ List<RelDataTypeField> fieldList,
+ int windowFieldIndex) {
+ if (column.getExpr().nodeKind() != RESOLVED_FUNCTION_CALL) {
+ return convertRexNodeFromResolvedExpr(column.getExpr(), columnList, fieldList);
+ }
+
+ ResolvedFunctionCall functionCall = (ResolvedFunctionCall) column.getExpr();
+
+ // TODO: is there any other illegal case?
+ if (functionCall.getFunction().getName().equals(FIXED_WINDOW)
+ || functionCall.getFunction().getName().equals(SLIDING_WINDOW)
+ || functionCall.getFunction().getName().equals(SESSION_WINDOW)) {
+ throw new RuntimeException(
+ functionCall.getFunction().getName() + " shouldn't appear in SELECT exprlist.");
+ }
+
+ if (!functionCall.getFunction().getGroup().equals(PRE_DEFINED_WINDOW_FUNCTIONS)) {
+ // non-window function should still go through normal FunctionCall conversion process.
+ return convertRexNodeFromResolvedExpr(column.getExpr(), columnList, fieldList);
+ }
+
+ // ONLY window_start and window_end should arrive here.
+ // TODO: Have extra verification here to make sure window start/end functions have the same
+ // parameter with window function.
+ List<RexNode> operands = new ArrayList<>();
+ switch (functionCall.getFunction().getName()) {
+ case FIXED_WINDOW_START:
+ case SLIDING_WINDOW_START:
+ case SESSION_WINDOW_START:
+ // TODO: in Calcite implementation, session window's start is equal to end. Need to fix it
+ // in Calcite.
+ case SESSION_WINDOW_END:
+ return rexBuilder()
+ .makeInputRef(fieldList.get(windowFieldIndex).getType(), windowFieldIndex);
+ case FIXED_WINDOW_END:
+ // WINDOW END is a function call
+ operands.add(
+ rexBuilder().makeInputRef(fieldList.get(windowFieldIndex).getType(), windowFieldIndex));
+ // TODO: check window_end 's duration is the same as it's aggregate window.
+ operands.add(
+ convertIntervalToRexIntervalLiteral(
+ (ResolvedLiteral) functionCall.getArgumentList().get(0)));
+ return rexBuilder().makeCall(SqlStdOperatorTable.PLUS, operands);
+ case SLIDING_WINDOW_END:
+ operands.add(
+ rexBuilder().makeInputRef(fieldList.get(windowFieldIndex).getType(), windowFieldIndex));
+ operands.add(
+ convertIntervalToRexIntervalLiteral(
+ (ResolvedLiteral) functionCall.getArgumentList().get(1)));
+ return rexBuilder().makeCall(SqlStdOperatorTable.PLUS, operands);
+ default:
+ throw new RuntimeException(
+ "Does not support window start/end: " + functionCall.getFunction().getName());
+ }
+ }
+
+ /** Convert a resolved literal to a RexNode. */
+ public RexNode convertResolvedLiteral(ResolvedLiteral resolvedLiteral) {
+ TypeKind kind = resolvedLiteral.getType().getKind();
+ RexNode ret;
+ switch (kind) {
+ case TYPE_BOOL:
+ case TYPE_INT32:
+ case TYPE_INT64:
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ case TYPE_STRING:
+ case TYPE_TIMESTAMP:
+ case TYPE_DATE:
+ case TYPE_TIME:
+ // case TYPE_DATETIME:
+ case TYPE_BYTES:
+ case TYPE_ARRAY:
+ case TYPE_STRUCT:
+ case TYPE_ENUM:
+ ret = convertValueToRexNode(resolvedLiteral.getType(), resolvedLiteral.getValue());
+ break;
+ default:
+ throw new RuntimeException(
+ MessageFormat.format(
+ "Unsupported ResolvedLiteral type: {0}, kind: {1}, value: {2}, class: {3}",
+ resolvedLiteral.getType().typeName(),
+ kind,
+ resolvedLiteral.getValue(),
+ resolvedLiteral.getClass()));
+ }
+
+ return ret;
+ }
+
+ private RexNode convertValueToRexNode(Type type, Value value) {
+ RexNode ret;
+ switch (type.getKind()) {
+ case TYPE_BOOL:
+ case TYPE_INT32:
+ case TYPE_INT64:
+ case TYPE_FLOAT:
+ case TYPE_DOUBLE:
+ case TYPE_STRING:
+ case TYPE_TIMESTAMP:
+ case TYPE_DATE:
+ case TYPE_TIME:
+ // case TYPE_DATETIME:
+ case TYPE_BYTES:
+ ret = convertSimpleValueToRexNode(type.getKind(), value);
+ break;
+ case TYPE_ARRAY:
+ ret = convertArrayValueToRexNode(type.asArray(), value);
+ break;
+ case TYPE_ENUM:
+ ret = convertEnumToRexNode(type, value);
+ break;
+ default:
+ // TODO: convert struct literal.
+ throw new RuntimeException(
+ "Unsupported ResolvedLiteral kind: " + type.getKind() + " type: " + type.typeName());
+ }
+
+ return ret;
+ }
+
+ private RexNode convertSimpleValueToRexNode(TypeKind kind, Value value) {
+ if (value.isNull()) {
+ return rexBuilder().makeNullLiteral(TypeUtils.toSimpleRelDataType(kind, rexBuilder()));
+ }
+
+ RexNode ret;
+ switch (kind) {
+ case TYPE_BOOL:
+ ret = rexBuilder().makeLiteral(value.getBoolValue());
+ break;
+ case TYPE_INT32:
+ ret =
+ rexBuilder()
+ .makeExactLiteral(
+ new BigDecimal(value.getInt32Value()),
+ TypeUtils.toSimpleRelDataType(kind, rexBuilder()));
+ break;
+ case TYPE_INT64:
+ ret =
+ rexBuilder()
+ .makeExactLiteral(
+ new BigDecimal(value.getInt64Value()),
+ TypeUtils.toSimpleRelDataType(kind, rexBuilder()));
+ break;
+ case TYPE_FLOAT:
+ ret =
+ rexBuilder()
+ .makeApproxLiteral(
+ new BigDecimal(value.getFloatValue()),
+ TypeUtils.toSimpleRelDataType(kind, rexBuilder()));
+ break;
+ case TYPE_DOUBLE:
+ ret =
+ rexBuilder()
+ .makeApproxLiteral(
+ new BigDecimal(value.getDoubleValue()),
+ TypeUtils.toSimpleRelDataType(kind, rexBuilder()));
+ break;
+ case TYPE_STRING:
+ // has to allow CAST because Calcite create CHAR type first and does a CAST to VARCHAR.
+ // If not allow cast, rexBuilder() will only build a literal with CHAR type.
+ ret =
+ rexBuilder()
+ .makeLiteral(
+ value.getStringValue(), typeFactory().createSqlType(SqlTypeName.VARCHAR), true);
+ break;
+ case TYPE_TIMESTAMP:
+ ret =
+ rexBuilder()
+ .makeTimestampLiteral(
+ TimestampString.fromMillisSinceEpoch(
+ safeMicrosToMillis(value.getTimestampUnixMicros())),
+ typeFactory().getTypeSystem().getMaxPrecision(SqlTypeName.TIMESTAMP));
+ break;
+ case TYPE_DATE:
+ ret = rexBuilder().makeDateLiteral(convertDateValueToDateString(value));
+ break;
+ case TYPE_TIME:
+ RelDataType timeType =
+ typeFactory()
+ .createSqlType(
+ SqlTypeName.TIME,
+ typeFactory().getTypeSystem().getMaxPrecision(SqlTypeName.TIME));
+ // TODO: Doing micro to mills truncation, need to throw exception.
+ ret = rexBuilder().makeLiteral(convertTimeValueToTimeString(value), timeType, false);
+ break;
+ case TYPE_BYTES:
+ ret = rexBuilder().makeBinaryLiteral(new ByteString(value.getBytesValue().toByteArray()));
+ break;
+ default:
+ throw new RuntimeException("Unsupported column type: " + kind);
+ }
+
+ return ret;
+ }
+
+ private RexNode convertArrayValueToRexNode(ArrayType arrayType, Value value) {
+ if (value.isNull()) {
+ // TODO: should the nullable be false for a array?
+ return rexBuilder()
+ .makeNullLiteral(TypeUtils.toArrayRelDataType(rexBuilder(), arrayType, false));
+ }
+
+ List<RexNode> operands = new ArrayList<>();
+ for (Value v : value.getElementList()) {
+ operands.add(convertValueToRexNode(arrayType.getElementType(), v));
+ }
+ return rexBuilder().makeCall(SqlStdOperatorTable.ARRAY_VALUE_CONSTRUCTOR, operands);
+ }
+
+ private RexNode convertEnumToRexNode(Type type, Value value) {
+ if (type.typeName().equals("`zetasql.functions.DateTimestampPart`")) {
+ return convertTimeUnitRangeEnumToRexNode(type, value);
+ } else {
+ throw new RuntimeException(
+ MessageFormat.format(
+ "Unsupported enum. Kind: {0} Type: {1}", type.getKind(), type.typeName()));
+ }
+ }
+
+ private RexNode convertTimeUnitRangeEnumToRexNode(Type type, Value value) {
+ TimeUnit mappedUnit = TIME_UNIT_CASTING_MAP.get(value.getEnumValue());
+ if (mappedUnit == null) {
+ throw new RuntimeException(
+ MessageFormat.format(
+ "Unsupported enum value. Kind: {0} Type: {1} Value: {2} EnumName: {3}",
+ type.getKind(), type.typeName(), value.getEnumName(), value.getEnumValue()));
+ }
+
+ TimeUnitRange mappedRange = TimeUnitRange.of(mappedUnit, null);
+ return rexBuilder().makeFlag(mappedRange);
+ }
+
+ private RexNode convertResolvedColumnRef(
+ ResolvedColumnRef columnRef,
+ List<ResolvedColumn> columnList,
+ List<RelDataTypeField> fieldList) {
+ int index = indexOfProjectionColumnRef(columnRef.getColumn().getId(), columnList);
+ if (index < 0 || index >= columnList.size()) {
+ throw new RuntimeException(
+ String.format("Cannot find %s in fieldList %s", columnRef.getColumn(), fieldList));
+ }
+ return rexBuilder().makeInputRef(fieldList.get(index).getType(), index);
+ }
+
+ private RexNode convertResolvedColumnRef(ResolvedColumnRef columnRef) {
+ // TODO: id - 1 might be only correct if the columns read from TableScan.
+ // What if the columns come from other scans (which means their id are not indexed from 0),
+ // and what if there are some mis-order?
+ // TODO: can join key be NULL?
+ return rexBuilder()
+ .makeInputRef(
+ TypeUtils.toRelDataType(rexBuilder(), columnRef.getType(), false),
+ (int) columnRef.getColumn().getId() - 1);
+ }
+
+ /** Return an index of the projection column reference. */
+ public int indexOfProjectionColumnRef(long colId, List<ResolvedColumn> columnList) {
+ int ret = -1;
+ for (int i = 0; i < columnList.size(); i++) {
+ if (columnList.get(i).getId() == colId) {
+ ret = i;
+ break;
+ }
+ }
+
+ return ret;
+ }
+
+ private RexNode convertResolvedFunctionCall(
+ ResolvedFunctionCall functionCall,
+ List<ResolvedColumn> columnList,
+ List<RelDataTypeField> fieldList) {
+ RexNode ret;
+ SqlOperator op;
+ List<RexNode> operands = new ArrayList<>();
+
+ if (functionCall.getFunction().getGroup().equals(PRE_DEFINED_WINDOW_FUNCTIONS)) {
+ switch (functionCall.getFunction().getName()) {
+ case FIXED_WINDOW:
+ case SESSION_WINDOW:
+ op =
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get(
+ functionCall.getFunction().getName());
+ // TODO: check size and type of window function argument list.
+ // Add ts column reference to operands.
+ operands.add(
+ convertRexNodeFromResolvedExpr(
+ functionCall.getArgumentList().get(0), columnList, fieldList));
+ // Add fixed window size or session window gap to operands.
+ operands.add(
+ convertIntervalToRexIntervalLiteral(
+ (ResolvedLiteral) functionCall.getArgumentList().get(1)));
+ break;
+ case SLIDING_WINDOW:
+ op =
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get(
+ SLIDING_WINDOW);
+ // Add ts column reference to operands.
+ operands.add(
+ convertRexNodeFromResolvedExpr(
+ functionCall.getArgumentList().get(0), columnList, fieldList));
+ // add sliding window emit frequency to operands.
+ operands.add(
+ convertIntervalToRexIntervalLiteral(
+ (ResolvedLiteral) functionCall.getArgumentList().get(1)));
+ // add sliding window size to operands.
+ operands.add(
+ convertIntervalToRexIntervalLiteral(
+ (ResolvedLiteral) functionCall.getArgumentList().get(2)));
+ break;
+ default:
+ throw new RuntimeException("Only support TUMBLE, HOP AND SESSION functions right now.");
+ }
+ } else if (functionCall.getFunction().getGroup().equals("ZetaSQL")) {
+ op =
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get(
+ functionCall.getFunction().getName());
+
+ if (op == null) {
+ throw new RuntimeException(
+ "Does not support ZetaSQL function: " + functionCall.getFunction().getName());
+ }
+
+ // There are different processes to handle argument conversion because INTERVAL is not a
+ // type in ZetaSQL.
+ if (FUNCTION_FAMILY_DATE_ADD.contains(functionCall.getFunction().getName())) {
+ return convertTimestampAddFunction(functionCall, columnList, fieldList);
+ } else {
+ for (ResolvedExpr expr : functionCall.getArgumentList()) {
+ operands.add(convertRexNodeFromResolvedExpr(expr, columnList, fieldList));
+ }
+ }
+ } else {
+ throw new RuntimeException(
+ "Does not support function group: " + functionCall.getFunction().getGroup());
+ }
+
+ SqlOperatorRewriter rewriter =
+ SqlStdOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR_REWRITER.get(
+ functionCall.getFunction().getName());
+
+ if (rewriter != null) {
+ ret = rewriter.apply(rexBuilder(), operands);
+ } else {
+ ret = rexBuilder().makeCall(op, operands);
+ }
+ return ret;
+ }
+
+ private RexNode convertTimestampAddFunction(
+ ResolvedFunctionCall functionCall,
+ List<ResolvedColumn> columnList,
+ List<RelDataTypeField> fieldList) {
+
+ TimeUnit unit =
+ TIME_UNIT_CASTING_MAP.get(
+ ((ResolvedLiteral) functionCall.getArgumentList().get(2)).getValue().getEnumValue());
+
+ if ((unit == TimeUnit.MICROSECOND) || (unit == TimeUnit.NANOSECOND)) {
+ throw Status.UNIMPLEMENTED
+ .withDescription("Micro and Nanoseconds are not supported by Beam ZetaSQL")
+ .asRuntimeException();
+ }
+
+ SqlIntervalQualifier qualifier = new SqlIntervalQualifier(unit, null, SqlParserPos.ZERO);
+
+ RexNode intervalArgumentNode =
+ convertRexNodeFromResolvedExpr(
+ functionCall.getArgumentList().get(1), columnList, fieldList);
+
+ RexNode validatedIntervalArgument =
+ rexBuilder()
+ .makeCall(
+ SqlOperators.VALIDATE_TIME_INTERVAL,
+ intervalArgumentNode,
+ rexBuilder().makeFlag(unit));
+
+ RexNode intervalNode =
+ rexBuilder()
+ .makeCall(
+ SqlStdOperatorTable.MULTIPLY,
+ rexBuilder().makeIntervalLiteral(unit.multiplier, qualifier),
+ validatedIntervalArgument);
+
+ RexNode timestampNode =
+ convertRexNodeFromResolvedExpr(
+ functionCall.getArgumentList().get(0), columnList, fieldList);
+
+ RexNode dateTimePlusResult =
+ rexBuilder().makeCall(SqlStdOperatorTable.DATETIME_PLUS, timestampNode, intervalNode);
+
+ RexNode validatedTimestampResult =
+ rexBuilder().makeCall(SqlOperators.VALIDATE_TIMESTAMP, dateTimePlusResult);
+
+ return validatedTimestampResult;
+ }
+
+ private RexNode convertIntervalToRexIntervalLiteral(ResolvedLiteral resolvedLiteral) {
+ if (resolvedLiteral.getType().getKind() != TYPE_STRING) {
+ throw new IllegalArgumentException(INTERVAL_FORMAT_MSG);
+ }
+
+ String valStr = resolvedLiteral.getValue().getStringValue();
+ List<String> stringList =
+ Arrays.stream(valStr.split(" ")).filter(s -> !s.isEmpty()).collect(Collectors.toList());
+
+ if (stringList.size() != 3) {
+ throw new IllegalArgumentException(INTERVAL_FORMAT_MSG);
+ }
+
+ if (!Ascii.toUpperCase(stringList.get(0)).equals("INTERVAL")) {
+ throw new IllegalArgumentException(INTERVAL_FORMAT_MSG);
+ }
+
+ long intervalValue;
+ try {
+ intervalValue = Long.parseLong(stringList.get(1));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(INTERVAL_FORMAT_MSG, e);
+ }
+
+ String intervalDatepart = Ascii.toUpperCase(stringList.get(2));
+ return createCalciteIntervalRexLiteral(intervalValue, intervalDatepart);
+ }
+
+ private RexLiteral createCalciteIntervalRexLiteral(long intervalValue, String intervalTimeUnit) {
+ SqlIntervalQualifier sqlIntervalQualifier =
+ convertIntervalDatepartToSqlIntervalQualifier(intervalTimeUnit);
+ BigDecimal decimalValue;
+ if (DATE_PART_UNITS_TO_MILLIS.contains(intervalTimeUnit)) {
+ decimalValue = convertIntervalValueToMillis(sqlIntervalQualifier, intervalValue);
+ } else if (DATE_PART_UNITS_TO_MONTHS.contains(intervalTimeUnit)) {
+ decimalValue = new BigDecimal(intervalValue * 12);
+ } else {
+ decimalValue = new BigDecimal(intervalValue);
+ }
+ return rexBuilder().makeIntervalLiteral(decimalValue, sqlIntervalQualifier);
+ }
+
+ private static BigDecimal convertIntervalValueToMillis(
+ SqlIntervalQualifier qualifier, long value) {
+ switch (qualifier.typeName()) {
+ case INTERVAL_DAY:
+ return new BigDecimal(value * ONE_DAY_IN_MILLIS);
+ case INTERVAL_HOUR:
+ return new BigDecimal(value * ONE_HOUR_IN_MILLIS);
+ case INTERVAL_MINUTE:
+ return new BigDecimal(value * ONE_MINUTE_IN_MILLIS);
+ case INTERVAL_SECOND:
+ return new BigDecimal(value * ONE_SECOND_IN_MILLIS);
+ default:
+ throw new IllegalArgumentException(qualifier.typeName().toString());
+ }
+ }
+
+ private static SqlIntervalQualifier convertIntervalDatepartToSqlIntervalQualifier(
+ String datePart) {
+ switch (datePart) {
+ case "YEAR":
+ return new SqlIntervalQualifier(TimeUnit.YEAR, null, SqlParserPos.ZERO);
+ case "MONTH":
+ return new SqlIntervalQualifier(TimeUnit.MONTH, null, SqlParserPos.ZERO);
+ case "DAY":
+ return new SqlIntervalQualifier(TimeUnit.DAY, null, SqlParserPos.ZERO);
+ case "HOUR":
+ return new SqlIntervalQualifier(TimeUnit.HOUR, null, SqlParserPos.ZERO);
+ case "MINUTE":
+ return new SqlIntervalQualifier(TimeUnit.MINUTE, null, SqlParserPos.ZERO);
+ case "SECOND":
+ return new SqlIntervalQualifier(TimeUnit.SECOND, null, SqlParserPos.ZERO);
+ case "WEEK":
+ return new SqlIntervalQualifier(TimeUnit.WEEK, null, SqlParserPos.ZERO);
+ case "QUARTER":
+ return new SqlIntervalQualifier(TimeUnit.QUARTER, null, SqlParserPos.ZERO);
+ case "MILLISECOND":
+ return new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, SqlParserPos.ZERO);
+ default:
+ throw new RuntimeException(
+ String.format(
+ "Received an undefined INTERVAL unit: %s. Please specify unit from the following"
+ + " list: %s.",
+ datePart, INTERVAL_DATE_PART_MSG));
+ }
+ }
+
+ private RexNode convertResolvedCast(
+ ResolvedCast resolvedCast,
+ List<ResolvedColumn> columnList,
+ List<RelDataTypeField> fieldList) {
+ TypeKind fromType = resolvedCast.getExpr().getType().getKind();
+ TypeKind toType = resolvedCast.getType().getKind();
+ isCastingSupported(fromType, toType);
+
+ RexNode inputNode =
+ convertRexNodeFromResolvedExpr(resolvedCast.getExpr(), columnList, fieldList);
+ // nullability of the output type should match that of the input node's type
+ RelDataType outputType =
+ TypeUtils.toSimpleRelDataType(
+ resolvedCast.getType().getKind(), rexBuilder(), inputNode.getType().isNullable());
+
+ if (isZetaSQLCast(fromType, toType)) {
+ return rexBuilder().makeCall(outputType, ZETASQL_CAST_OP, ImmutableList.of(inputNode));
+ } else {
+ return rexBuilder().makeCast(outputType, inputNode);
+ }
+ }
+
+ private static void isCastingSupported(TypeKind fromType, TypeKind toType) {
+ if (UNSUPPORTED_CASTING.containsKey(toType)
+ && UNSUPPORTED_CASTING.get(toType).contains(fromType)) {
+ throw new IllegalArgumentException(
+ "Does not support CAST(" + fromType + " AS " + toType + ")");
+ }
+ }
+
+ private static boolean isZetaSQLCast(TypeKind fromType, TypeKind toType) {
+ // TODO: Structure ZETASQL_CAST_OP so that we don't have to repeat the supported types
+ // here
+ return (fromType.equals(TYPE_BYTES) && toType.equals(TYPE_STRING))
+ || (fromType.equals(TYPE_INT64) && toType.equals(TYPE_BOOL))
+ || (fromType.equals(TYPE_BOOL) && toType.equals(TYPE_INT64))
+ || (fromType.equals(TYPE_TIMESTAMP) && toType.equals(TYPE_STRING));
+ }
+
+ private RexNode convertRexNodeFromResolvedColumnRefWithRefScan(
+ ResolvedColumnRef columnRef,
+ List<ResolvedColumn> refScanColumnList,
+ List<ResolvedColumn> originalColumnList,
+ List<RelDataTypeField> fieldList) {
+
+ for (int i = 0; i < refScanColumnList.size(); i++) {
+ if (refScanColumnList.get(i).getId() == columnRef.getColumn().getId()) {
+ boolean nullable = fieldList.get(i).getType().isNullable();
+ int off = (int) originalColumnList.get(i).getId() - 1;
+ return rexBuilder()
+ .makeInputRef(
+ TypeUtils.toSimpleRelDataType(
+ columnRef.getType().getKind(), rexBuilder(), nullable),
+ off);
+ }
+ }
+
+ return null;
+ }
+
+ private RexNode convertResolvedParameter(ResolvedParameter parameter) {
+ assert parameter.getType().equals(queryParams.get(parameter.getName()).getType());
+ return convertValueToRexNode(
+ queryParams.get(parameter.getName()).getType(), queryParams.get(parameter.getName()));
+ }
+
+ private RexNode convertResolvedStructFieldAccess(ResolvedGetStructField resolvedGetStructField) {
+ return rexBuilder()
+ .makeFieldAccess(
+ convertRexNodeFromResolvedExpr(resolvedGetStructField.getExpr()),
+ (int) resolvedGetStructField.getFieldIdx());
+ }
+
+ private RexNode convertResolvedStructFieldAccess(
+ ResolvedGetStructField resolvedGetStructField,
+ List<ResolvedColumn> columnList,
+ List<RelDataTypeField> fieldList) {
+ return rexBuilder()
+ .makeFieldAccess(
+ convertRexNodeFromResolvedExpr(resolvedGetStructField.getExpr(), columnList, fieldList),
+ (int) resolvedGetStructField.getFieldIdx());
+ }
+
+ private RexBuilder rexBuilder() {
+ return cluster.getRexBuilder();
+ }
+
+ private RelDataTypeFactory typeFactory() {
+ return cluster.getTypeFactory();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/FilterScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/FilterScanConverter.java
new file mode 100644
index 0000000..bcddd9c
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/FilterScanConverter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedFilterScan;
+import java.util.Collections;
+import java.util.List;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalFilter;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts filter. */
+class FilterScanConverter extends RelConverter<ResolvedFilterScan> {
+
+ FilterScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedFilterScan zetaNode) {
+ return Collections.singletonList(zetaNode.getInputScan());
+ }
+
+ @Override
+ public RelNode convert(ResolvedFilterScan zetaNode, List<RelNode> inputs) {
+ RelNode input = inputs.get(0);
+ RexNode condition =
+ getExpressionConverter()
+ .convertRexNodeFromResolvedExpr(
+ zetaNode.getFilterExpr(),
+ zetaNode.getInputScan().getColumnList(),
+ input.getRowType().getFieldList());
+
+ return LogicalFilter.create(input, condition);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanConverter.java
new file mode 100644
index 0000000..e5e8cef
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanConverter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedColumn;
+import com.google.zetasql.resolvedast.ResolvedJoinScanEnums.JoinType;
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedJoinScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedWithRefScan;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts joins if neither side of the join is a WithRefScan. */
+class JoinScanConverter extends RelConverter<ResolvedJoinScan> {
+
+ private static final ImmutableMap<JoinType, JoinRelType> JOIN_TYPES =
+ ImmutableMap.of(
+ JoinType.INNER,
+ JoinRelType.INNER,
+ JoinType.FULL,
+ JoinRelType.FULL,
+ JoinType.LEFT,
+ JoinRelType.LEFT,
+ JoinType.RIGHT,
+ JoinRelType.RIGHT);
+
+ JoinScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean canConvert(ResolvedJoinScan zetaNode) {
+ return !(zetaNode.getLeftScan() instanceof ResolvedWithRefScan)
+ && !(zetaNode.getRightScan() instanceof ResolvedWithRefScan);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedJoinScan zetaNode) {
+ return ImmutableList.of(zetaNode.getLeftScan(), zetaNode.getRightScan());
+ }
+
+ @Override
+ public RelNode convert(ResolvedJoinScan zetaNode, List<RelNode> inputs) {
+ RelNode convertedLeftInput = inputs.get(0);
+ RelNode convertedRightInput = inputs.get(1);
+
+ List<ResolvedColumn> combinedZetaFieldsList =
+ ImmutableList.<ResolvedColumn>builder()
+ .addAll(zetaNode.getLeftScan().getColumnList())
+ .addAll(zetaNode.getRightScan().getColumnList())
+ .build();
+
+ List<RelDataTypeField> combinedCalciteFieldsList =
+ ImmutableList.<RelDataTypeField>builder()
+ .addAll(convertedLeftInput.getRowType().getFieldList())
+ .addAll(convertedRightInput.getRowType().getFieldList())
+ .build();
+
+ RexNode condition =
+ getExpressionConverter()
+ .convertRexNodeFromResolvedExpr(
+ zetaNode.getJoinExpr(), combinedZetaFieldsList, combinedCalciteFieldsList);
+
+ return LogicalJoin.create(
+ convertedLeftInput,
+ convertedRightInput,
+ condition,
+ ImmutableSet.of(),
+ convertResolvedJoinType(zetaNode.getJoinType()));
+ }
+
+ static JoinRelType convertResolvedJoinType(JoinType joinType) {
+ if (!JOIN_TYPES.containsKey(joinType)) {
+ throw new UnsupportedOperationException("JOIN type: " + joinType + " is unsupported.");
+ }
+
+ return JOIN_TYPES.get(joinType);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanWithRefConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanWithRefConverter.java
new file mode 100644
index 0000000..55594f9
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/JoinScanWithRefConverter.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import static org.apache.beam.sdk.extensions.sql.zetasql.translation.JoinScanConverter.convertResolvedJoinType;
+
+import com.google.zetasql.resolvedast.ResolvedColumn;
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedJoinScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedWithRefScan;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalJoin;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts joins where at least one of the inputs is a WITH subquery. */
+class JoinScanWithRefConverter extends RelConverter<ResolvedJoinScan> {
+
+ JoinScanWithRefConverter(ConversionContext context) {
+ super(context);
+ }
+
+ /** This is a special logic due to re-indexed column reference in WithScan. */
+ @Override
+ public boolean canConvert(ResolvedJoinScan zetaNode) {
+ return zetaNode.getLeftScan() instanceof ResolvedWithRefScan
+ || zetaNode.getRightScan() instanceof ResolvedWithRefScan;
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedJoinScan zetaNode) {
+ return ImmutableList.of(zetaNode.getLeftScan(), zetaNode.getRightScan());
+ }
+
+ @Override
+ public RelNode convert(ResolvedJoinScan zetaNode, List<RelNode> inputs) {
+ RelNode calciteLeftInput = inputs.get(0);
+ RelNode calciteRightInput = inputs.get(1);
+
+ List<ResolvedColumn> zetaLeftColumnList = getColumnsForScan(zetaNode.getLeftScan());
+ List<ResolvedColumn> zetaRightColumnList = getColumnsForScan(zetaNode.getRightScan());
+
+ RexNode condition =
+ getExpressionConverter()
+ .convertRexNodeFromResolvedExprWithRefScan(
+ zetaNode.getJoinExpr(),
+ zetaNode.getLeftScan().getColumnList(),
+ calciteLeftInput.getRowType().getFieldList(),
+ zetaLeftColumnList,
+ zetaNode.getRightScan().getColumnList(),
+ calciteRightInput.getRowType().getFieldList(),
+ zetaRightColumnList);
+
+ return LogicalJoin.create(
+ calciteLeftInput,
+ calciteRightInput,
+ condition,
+ ImmutableSet.of(),
+ convertResolvedJoinType(zetaNode.getJoinType()));
+ }
+
+ /**
+ * WithRefScan doesn't have columns in it, it only references a WITH query by name, we have to
+ * look up the actual query node in the context by that name.
+ *
+ * <p>The context has a map of WITH queries populated when the inputs to this JOIN are parsed.
+ */
+ private List<ResolvedColumn> getColumnsForScan(ResolvedScan resolvedScan) {
+ return resolvedScan instanceof ResolvedWithRefScan
+ ? getTrait()
+ .withEntries
+ .get(((ResolvedWithRefScan) resolvedScan).getWithQueryName())
+ .getWithSubquery()
+ .getColumnList()
+ : resolvedScan.getColumnList();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToLimitConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToLimitConverter.java
new file mode 100644
index 0000000..bbe930b
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToLimitConverter.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedLimitOffsetScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedOrderByScan;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollations;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts LIMIT without ORDER BY. */
+class LimitOffsetScanToLimitConverter extends RelConverter<ResolvedLimitOffsetScan> {
+
+ LimitOffsetScanToLimitConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean canConvert(ResolvedLimitOffsetScan zetaNode) {
+ return !(zetaNode.getInputScan() instanceof ResolvedOrderByScan);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedLimitOffsetScan zetaNode) {
+ return Collections.singletonList(zetaNode.getInputScan());
+ }
+
+ @Override
+ public RelNode convert(ResolvedLimitOffsetScan zetaNode, List<RelNode> inputs) {
+ RelNode input = inputs.get(0);
+ RelCollation relCollation = RelCollations.of(ImmutableList.of());
+ RexNode offset =
+ zetaNode.getOffset() == null
+ ? null
+ : getExpressionConverter().convertRexNodeFromResolvedExpr(zetaNode.getOffset());
+ RexNode fetch =
+ getExpressionConverter()
+ .convertRexNodeFromResolvedExpr(
+ zetaNode.getLimit(), zetaNode.getColumnList(), input.getRowType().getFieldList());
+ return LogicalSort.create(input, relCollation, offset, fetch);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToOrderByLimitConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToOrderByLimitConverter.java
new file mode 100644
index 0000000..d38abc7
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/LimitOffsetScanToOrderByLimitConverter.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.calcite.rel.RelFieldCollation.Direction.ASCENDING;
+import static org.apache.calcite.rel.RelFieldCollation.Direction.DESCENDING;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedLimitOffsetScan;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedOrderByItem;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedOrderByScan;
+import java.util.Collections;
+import java.util.List;
+import org.apache.calcite.rel.RelCollation;
+import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.calcite.rel.RelFieldCollation.Direction;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.logical.LogicalSort;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts ORDER BY LIMIT. */
+class LimitOffsetScanToOrderByLimitConverter extends RelConverter<ResolvedLimitOffsetScan> {
+
+ LimitOffsetScanToOrderByLimitConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean canConvert(ResolvedLimitOffsetScan zetaNode) {
+ return zetaNode.getInputScan() instanceof ResolvedOrderByScan;
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedLimitOffsetScan zetaNode) {
+ // The immediate input is the ORDER BY scan which we don't support,
+ // but we can handle the ORDER BY LIMIT if we know the underlying projection, for example.
+ return Collections.singletonList(
+ ((ResolvedOrderByScan) zetaNode.getInputScan()).getInputScan());
+ }
+
+ @Override
+ public RelNode convert(ResolvedLimitOffsetScan zetaNode, List<RelNode> inputs) {
+ ResolvedOrderByScan inputOrderByScan = (ResolvedOrderByScan) zetaNode.getInputScan();
+ RelNode input = convertOrderByScanToLogicalScan(inputOrderByScan, inputs.get(0));
+ RelCollation relCollation = getRelCollation(inputOrderByScan);
+
+ RexNode offset =
+ zetaNode.getOffset() == null
+ ? null
+ : getExpressionConverter().convertRexNodeFromResolvedExpr(zetaNode.getOffset());
+ RexNode fetch =
+ getExpressionConverter()
+ .convertRexNodeFromResolvedExpr(
+ zetaNode.getLimit(), zetaNode.getColumnList(), input.getRowType().getFieldList());
+
+ return LogicalSort.create(input, relCollation, offset, fetch);
+ }
+
+ /** Collation is a sort order, as in ORDER BY DESCENDING/ASCENDING. */
+ private static RelCollation getRelCollation(ResolvedOrderByScan node) {
+ List<RelFieldCollation> fieldCollations =
+ node.getOrderByItemList().stream()
+ .map(LimitOffsetScanToOrderByLimitConverter::orderByItemToFieldCollation)
+ .collect(toList());
+ return RelCollationImpl.of(fieldCollations);
+ }
+
+ private static RelFieldCollation orderByItemToFieldCollation(ResolvedOrderByItem item) {
+ // TODO: might need a column ref mapping here.
+ Direction sortDirection = item.getIsDescending() ? DESCENDING : ASCENDING;
+ int fieldIndex = (int) item.getColumnRef().getColumn().getId();
+ return new RelFieldCollation(fieldIndex, sortDirection);
+ }
+
+ private RelNode convertOrderByScanToLogicalScan(ResolvedOrderByScan node, RelNode input) {
+ List<RexNode> projects =
+ getExpressionConverter()
+ .retrieveRexNodeFromOrderByScan(getCluster(), node, input.getRowType().getFieldList());
+ List<String> fieldNames = getTrait().retrieveFieldNames(node.getColumnList());
+
+ return LogicalProject.create(input, projects, fieldNames);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/OrderByScanUnsupportedConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/OrderByScanUnsupportedConverter.java
new file mode 100644
index 0000000..a496862
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/OrderByScanUnsupportedConverter.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedOrderByScan;
+import java.util.List;
+import org.apache.calcite.rel.RelNode;
+
+/**
+ * Always throws exception, represents the case when order by is used without limit.
+ *
+ * <p>Order by limit is a special case that is handled in {@link LimitOffsetScanToLimitConverter}.
+ */
+class OrderByScanUnsupportedConverter extends RelConverter<ResolvedOrderByScan> {
+
+ OrderByScanUnsupportedConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public RelNode convert(ResolvedOrderByScan zetaNode, List<RelNode> inputs) {
+ throw new UnsupportedOperationException("ORDER BY without a LIMIT is not supported.");
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ProjectScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ProjectScanConverter.java
new file mode 100644
index 0000000..323277f
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ProjectScanConverter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedProjectScan;
+import java.util.Collections;
+import java.util.List;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rex.RexNode;
+
+/** Converts projection. */
+class ProjectScanConverter extends RelConverter<ResolvedProjectScan> {
+
+ ProjectScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedProjectScan zetaNode) {
+ return Collections.singletonList(zetaNode.getInputScan());
+ }
+
+ @Override
+ public RelNode convert(ResolvedProjectScan zetaNode, List<RelNode> inputs) {
+ RelNode input = inputs.get(0);
+
+ List<RexNode> projects =
+ getExpressionConverter().retrieveRexNode(zetaNode, input.getRowType().getFieldList());
+ List<String> fieldNames = getTrait().retrieveFieldNames(zetaNode.getColumnList());
+ return LogicalProject.create(input, projects, fieldNames);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/QueryStatementConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/QueryStatementConverter.java
new file mode 100644
index 0000000..29c5cd4
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/QueryStatementConverter.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_AGGREGATE_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_ARRAY_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_FILTER_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_JOIN_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_LIMIT_OFFSET_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_ORDER_BY_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_PROJECT_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_SET_OPERATION_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_SINGLE_ROW_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_TABLE_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_WITH_REF_SCAN;
+import static com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind.RESOLVED_WITH_SCAN;
+import static java.util.stream.Collectors.toList;
+
+import com.google.zetasql.ZetaSQLResolvedNodeKind.ResolvedNodeKind;
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedQueryStmt;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMultimap;
+import org.apache.calcite.rel.RelNode;
+
+/**
+ * Converts a resolved Zeta SQL query represented by a tree to corresponding Calcite representation.
+ */
+public class QueryStatementConverter extends RelConverter<ResolvedQueryStmt> {
+
+ /** Conversion rules, multimap from node kind to conversion rule. */
+ private final ImmutableMultimap<ResolvedNodeKind, RelConverter> rules;
+
+ public static RelNode convertRootQuery(ConversionContext context, ResolvedQueryStmt query) {
+ return new QueryStatementConverter(context).convert(query, Collections.emptyList());
+ }
+
+ private QueryStatementConverter(ConversionContext context) {
+ super(context);
+ this.rules =
+ ImmutableMultimap.<ResolvedNodeKind, RelConverter>builder()
+ .put(RESOLVED_AGGREGATE_SCAN, new AggregateScanConverter(context))
+ .put(RESOLVED_ARRAY_SCAN, new ArrayScanToJoinConverter(context))
+ .put(RESOLVED_ARRAY_SCAN, new ArrayScanToUncollectConverter(context))
+ .put(RESOLVED_FILTER_SCAN, new FilterScanConverter(context))
+ .put(RESOLVED_JOIN_SCAN, new JoinScanConverter(context))
+ .put(RESOLVED_JOIN_SCAN, new JoinScanWithRefConverter(context))
+ .put(RESOLVED_LIMIT_OFFSET_SCAN, new LimitOffsetScanToLimitConverter(context))
+ .put(RESOLVED_LIMIT_OFFSET_SCAN, new LimitOffsetScanToOrderByLimitConverter(context))
+ .put(RESOLVED_ORDER_BY_SCAN, new OrderByScanUnsupportedConverter(context))
+ .put(RESOLVED_PROJECT_SCAN, new ProjectScanConverter(context))
+ .put(RESOLVED_SET_OPERATION_SCAN, new SetOperationScanConverter(context))
+ .put(RESOLVED_SINGLE_ROW_SCAN, new SingleRowScanConverter(context))
+ .put(RESOLVED_TABLE_SCAN, new TableScanConverter(context))
+ .put(RESOLVED_WITH_REF_SCAN, new WithRefScanConverter(context))
+ .put(RESOLVED_WITH_SCAN, new WithScanConverter(context))
+ .build();
+ }
+
+ @Override
+ public RelNode convert(ResolvedQueryStmt zetaNode, List<RelNode> inputs) {
+ if (zetaNode.getIsValueTable()) {
+ throw new UnsupportedOperationException("Value Tables are not supported");
+ }
+
+ getTrait().addOutputColumnList(zetaNode.getOutputColumnList());
+
+ return convertNode(zetaNode.getQuery());
+ }
+
+ /**
+ * Convert node.
+ *
+ * <p>Finds a matching rule, uses the rule to extract inputs from the node, then converts the
+ * inputs (recursively), then converts the node using the converted inputs.
+ */
+ private RelNode convertNode(ResolvedNode zetaNode) {
+ RelConverter nodeConverter = getConverterRule(zetaNode);
+ List<ResolvedNode> inputs = nodeConverter.getInputs(zetaNode);
+ List<RelNode> convertedInputs = inputs.stream().map(this::convertNode).collect(toList());
+ return nodeConverter.convert(zetaNode, convertedInputs);
+ }
+
+ private RelConverter getConverterRule(ResolvedNode zetaNode) {
+ if (!rules.containsKey(zetaNode.nodeKind())) {
+ throw new UnsupportedOperationException(
+ String.format("Conversion of %s is not supported", zetaNode.nodeKind()));
+ }
+
+ return rules.get(zetaNode.nodeKind()).stream()
+ .filter(relConverter -> relConverter.canConvert(zetaNode))
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new UnsupportedOperationException(
+ String.format("Cannot find a conversion rule for: %s", zetaNode)));
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/RelConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/RelConverter.java
new file mode 100644
index 0000000..69f01e4
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/RelConverter.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import java.util.List;
+import org.apache.beam.sdk.extensions.sql.zetasql.QueryTrait;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.tools.FrameworkConfig;
+
+/** A rule that converts Zeta SQL resolved relational node to corresponding Calcite rel node. */
+abstract class RelConverter<T extends ResolvedNode> {
+
+ /**
+ * Conversion context, contains things like FrameworkConfig, QueryTrait and other state used
+ * during conversion.
+ */
+ protected ConversionContext context;
+
+ RelConverter(ConversionContext context) {
+ this.context = context;
+ }
+
+ /** Whether this rule can handle the conversion of the specific node. */
+ public boolean canConvert(T zetaNode) {
+ return true;
+ }
+
+ /** Extract Zeta SQL resolved nodes that correspond to the inputs of the current node. */
+ public List<ResolvedNode> getInputs(T zetaNode) {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Converts given Zeta SQL node to corresponding Calcite node.
+ *
+ * <p>{@code inputs} are node inputs that have already been converter to Calcite versions. They
+ * correspond to the nodes in {@link #getInputs(ResolvedNode)}.
+ */
+ public abstract RelNode convert(T zetaNode, List<RelNode> inputs);
+
+ protected RelOptCluster getCluster() {
+ return context.cluster();
+ }
+
+ protected FrameworkConfig getConfig() {
+ return context.getConfig();
+ }
+
+ protected ExpressionConverter getExpressionConverter() {
+ return context.getExpressionConverter();
+ }
+
+ protected QueryTrait getTrait() {
+ return context.getTrait();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SetOperationScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SetOperationScanConverter.java
new file mode 100644
index 0000000..7c52d6e
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SetOperationScanConverter.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import static com.google.zetasql.resolvedast.ResolvedSetOperationScanEnums.SetOperationType.EXCEPT_ALL;
+import static com.google.zetasql.resolvedast.ResolvedSetOperationScanEnums.SetOperationType.EXCEPT_DISTINCT;
+import static com.google.zetasql.resolvedast.ResolvedSetOperationScanEnums.SetOperationType.INTERSECT_ALL;
+import static com.google.zetasql.resolvedast.ResolvedSetOperationScanEnums.SetOperationType.INTERSECT_DISTINCT;
+import static com.google.zetasql.resolvedast.ResolvedSetOperationScanEnums.SetOperationType.UNION_ALL;
+import static com.google.zetasql.resolvedast.ResolvedSetOperationScanEnums.SetOperationType.UNION_DISTINCT;
+import static java.util.stream.Collectors.toList;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedSetOperationItem;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedSetOperationScan;
+import com.google.zetasql.resolvedast.ResolvedSetOperationScanEnums.SetOperationType;
+import java.util.List;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalIntersect;
+import org.apache.calcite.rel.logical.LogicalMinus;
+import org.apache.calcite.rel.logical.LogicalUnion;
+
+/** Converts set operations. */
+class SetOperationScanConverter extends RelConverter<ResolvedSetOperationScan> {
+ private enum Type {
+ DISTINCT,
+ ALL
+ }
+
+ private static final ImmutableMap<SetOperationType, Function<List<RelNode>, RelNode>>
+ SET_OPERATION_FACTORIES =
+ ImmutableMap.<SetOperationType, Function<List<RelNode>, RelNode>>builder()
+ .put(UNION_ALL, createFactoryFor(LogicalUnion::create, Type.ALL))
+ .put(UNION_DISTINCT, createFactoryFor(LogicalUnion::create, Type.DISTINCT))
+ .put(INTERSECT_ALL, createFactoryFor(LogicalIntersect::create, Type.ALL))
+ .put(INTERSECT_DISTINCT, createFactoryFor(LogicalIntersect::create, Type.DISTINCT))
+ .put(EXCEPT_ALL, createFactoryFor(LogicalMinus::create, Type.ALL))
+ .put(EXCEPT_DISTINCT, createFactoryFor(LogicalMinus::create, Type.DISTINCT))
+ .build();
+
+ /**
+ * A little closure to wrap the invocation of the factory method (e.g. LogicalUnion::create) for
+ * the set operation node.
+ */
+ private static Function<List<RelNode>, RelNode> createFactoryFor(
+ BiFunction<List<RelNode>, Boolean, RelNode> setOperationFactory, Type type) {
+ return (List<RelNode> inputs) -> createRel(setOperationFactory, type == Type.ALL, inputs);
+ }
+
+ SetOperationScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedSetOperationScan zetaNode) {
+ return zetaNode.getInputItemList().stream()
+ .map(ResolvedSetOperationItem::getScan)
+ .collect(toList());
+ }
+
+ @Override
+ public RelNode convert(ResolvedSetOperationScan zetaNode, List<RelNode> inputs) {
+ if (!SET_OPERATION_FACTORIES.containsKey(zetaNode.getOpType())) {
+ throw new UnsupportedOperationException(
+ "Operation " + zetaNode.getOpType() + " is unsupported");
+ }
+
+ return SET_OPERATION_FACTORIES.get(zetaNode.getOpType()).apply(inputs);
+ }
+
+ /** Beam set operations rel expects two inputs, so we are constructing a binary tree here. */
+ private static RelNode createRel(
+ BiFunction<List<RelNode>, Boolean, RelNode> factory, boolean all, List<RelNode> inputs) {
+ return inputs.stream()
+ .skip(2)
+ .reduce(
+ // start with creating a set node for two first inputs
+ invokeFactory(factory, inputs.get(0), inputs.get(1), all),
+ // create another operation node with previous op node and the next input
+ (setOpNode, nextInput) -> invokeFactory(factory, setOpNode, nextInput, all));
+ }
+
+ /**
+ * Creates a set operation rel with two inputs.
+ *
+ * <p>Factory is, for example, LogicalUnion::create.
+ */
+ private static RelNode invokeFactory(
+ BiFunction<List<RelNode>, Boolean, RelNode> factory,
+ RelNode input1,
+ RelNode input2,
+ boolean all) {
+ return factory.apply(ImmutableList.of(input1, input2), all);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SingleRowScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SingleRowScanConverter.java
new file mode 100644
index 0000000..f4553e5
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SingleRowScanConverter.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedSingleRowScan;
+import java.util.List;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.logical.LogicalValues;
+
+/** Converts a single row value. */
+class SingleRowScanConverter extends RelConverter<ResolvedSingleRowScan> {
+
+ SingleRowScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public boolean canConvert(ResolvedSingleRowScan zetaNode) {
+ return zetaNode.getColumnList() == null || zetaNode.getColumnList().isEmpty();
+ }
+
+ @Override
+ public RelNode convert(ResolvedSingleRowScan zetaNode, List<RelNode> inputs) {
+ return LogicalValues.createOneRow(getCluster());
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/TableScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/TableScanConverter.java
new file mode 100644
index 0000000..2ad54db
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/TableScanConverter.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_DATETIME;
+import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_NUMERIC;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.zetasql.ZetaSQLType.TypeKind;
+import com.google.zetasql.resolvedast.ResolvedColumn;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedTableScan;
+import java.util.List;
+import java.util.Properties;
+import org.apache.beam.sdk.extensions.sql.zetasql.TableResolution;
+import org.apache.calcite.config.CalciteConnectionConfigImpl;
+import org.apache.calcite.jdbc.CalciteSchema;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelOptTable;
+import org.apache.calcite.prepare.CalciteCatalogReader;
+import org.apache.calcite.prepare.RelOptTableImpl;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.RelRoot;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.schema.TranslatableTable;
+
+/** Converts table scan. */
+class TableScanConverter extends RelConverter<ResolvedTableScan> {
+
+ private static final ImmutableSet<TypeKind> UNSUPPORTED_DATA_TYPES =
+ ImmutableSet.of(TYPE_DATETIME, TYPE_NUMERIC);
+
+ TableScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public RelNode convert(ResolvedTableScan zetaNode, List<RelNode> inputs) {
+ checkTableScanSchema(zetaNode.getColumnList());
+
+ List<String> tablePath = getTablePath(zetaNode.getTable());
+
+ SchemaPlus defaultSchemaPlus = getConfig().getDefaultSchema();
+ // TODO: reject incorrect top-level schema
+
+ Table calciteTable =
+ TableResolution.resolveCalciteTable(getConfig().getContext(), defaultSchemaPlus, tablePath);
+
+ // we already resolved the table before passing the query to Analyzer, so it should be there
+ checkNotNull(
+ calciteTable,
+ "Unable to resolve the table path %s in schema %s",
+ tablePath,
+ defaultSchemaPlus.getName());
+
+ String defaultSchemaName = defaultSchemaPlus.getName();
+
+ final CalciteCatalogReader catalogReader =
+ new CalciteCatalogReader(
+ CalciteSchema.from(defaultSchemaPlus),
+ ImmutableList.of(defaultSchemaName),
+ getCluster().getTypeFactory(),
+ new CalciteConnectionConfigImpl(new Properties()));
+
+ RelOptTableImpl relOptTable =
+ RelOptTableImpl.create(
+ catalogReader,
+ calciteTable.getRowType(getCluster().getTypeFactory()),
+ calciteTable,
+ ImmutableList.<String>builder().add(defaultSchemaName).addAll(tablePath).build());
+
+ if (calciteTable instanceof TranslatableTable) {
+ return ((TranslatableTable) calciteTable).toRel(createToRelContext(), relOptTable);
+ } else {
+ throw new RuntimeException("Does not support non TranslatableTable type table!");
+ }
+ }
+
+ private List<String> getTablePath(com.google.zetasql.Table table) {
+ if (!getTrait().isTableResolved(table)) {
+ throw new RuntimeException(
+ "Unexpected table found when converting to Calcite rel node: " + table);
+ }
+
+ return getTrait().getTablePath(table);
+ }
+
+ private RelOptTable.ToRelContext createToRelContext() {
+ return new RelOptTable.ToRelContext() {
+ @Override
+ public RelRoot expandView(
+ RelDataType relDataType, String s, List<String> list, List<String> list1) {
+ throw new UnsupportedOperationException("This RelContext does not support expandView");
+ }
+
+ @Override
+ public RelOptCluster getCluster() {
+ return TableScanConverter.this.getCluster();
+ }
+ };
+ }
+
+ private void checkTableScanSchema(List<ResolvedColumn> columnList) {
+ if (columnList != null) {
+ for (ResolvedColumn resolvedColumn : columnList) {
+ if (UNSUPPORTED_DATA_TYPES.contains(resolvedColumn.getType().getKind())) {
+ throw new IllegalArgumentException(
+ "Does not support " + UNSUPPORTED_DATA_TYPES + " types in source tables");
+ }
+ }
+ }
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/WithRefScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/WithRefScanConverter.java
new file mode 100644
index 0000000..11180c6
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/WithRefScanConverter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedWithRefScan;
+import java.util.Collections;
+import java.util.List;
+import org.apache.calcite.rel.RelNode;
+
+/** Converts a call-site reference to a named WITH subquery. */
+class WithRefScanConverter extends RelConverter<ResolvedWithRefScan> {
+
+ WithRefScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedWithRefScan zetaNode) {
+ // WithRefScan contains only a name of a WITH query,
+ // but to actually convert it to the node we need to get the resolved node representation
+ // of the query. Here we take it from the trait, where it was persisted previously
+ // in WithScanConverter that actually parses the WITH query part.
+ //
+ // This query node returned from here will be converted by some other converter,
+ // (e.g. if the WITH query root is a projection it will go through ProjectScanConverter)
+ // and will reach the convert() method below as an already converted rel node.
+ return Collections.singletonList(
+ getTrait().withEntries.get(zetaNode.getWithQueryName()).getWithSubquery());
+ }
+
+ @Override
+ public RelNode convert(ResolvedWithRefScan zetaNode, List<RelNode> inputs) {
+ // Here the actual WITH query body has already been converted by, e.g. a ProjectScnaConverter,
+ // so to resolve the reference we just return that converter rel node.
+ return inputs.get(0);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/WithScanConverter.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/WithScanConverter.java
new file mode 100644
index 0000000..208c396
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/WithScanConverter.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
+
+import com.google.zetasql.resolvedast.ResolvedNode;
+import com.google.zetasql.resolvedast.ResolvedNodes.ResolvedWithScan;
+import java.util.Collections;
+import java.util.List;
+import org.apache.calcite.rel.RelNode;
+
+/** Converts a named WITH. */
+class WithScanConverter extends RelConverter<ResolvedWithScan> {
+
+ WithScanConverter(ConversionContext context) {
+ super(context);
+ }
+
+ @Override
+ public List<ResolvedNode> getInputs(ResolvedWithScan zetaNode) {
+ // We must persist the named WITH queries nodes,
+ // so that when they are referenced by name (e.g. in FROM/JOIN), we can
+ // resolve them. We need this because the nodes that represent the references (WithRefScan)
+ // only contain the names of the queries, so we need to keep this map for resolution of the
+ // names.
+ zetaNode
+ .getWithEntryList()
+ .forEach(withEntry -> getTrait().withEntries.put(withEntry.getWithQueryName(), withEntry));
+
+ // Returning the body of the query, it is something like ProjectScan that will be converted
+ // by ProjectScanConverter before it reaches the convert() method below.
+ return Collections.singletonList(zetaNode.getQuery());
+ }
+
+ @Override
+ public RelNode convert(ResolvedWithScan zetaNode, List<RelNode> inputs) {
+ // The body of the WITH query is already converted at this point so we just
+ // return it, nothing else is needed.
+ return inputs.get(0);
+ }
+}
diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/package-info.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/package-info.java
new file mode 100644
index 0000000..aa596ee
--- /dev/null
+++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/package-info.java
@@ -0,0 +1,2 @@
+/** Conversion logic between ZetaSQL resolved query nodes and Calcite rel nodes. */
+package org.apache.beam.sdk.extensions.sql.zetasql.translation;
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/JoinCompoundIdentifiersTestZetaSQL.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/JoinCompoundIdentifiersTestZetaSQL.java
new file mode 100644
index 0000000..3195a60
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/JoinCompoundIdentifiersTestZetaSQL.java
@@ -0,0 +1,342 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.BASIC_TABLE_ONE;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.BASIC_TABLE_TWO;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_WITH_STRUCT;
+
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.impl.JdbcConnection;
+import org.apache.beam.sdk.extensions.sql.impl.JdbcDriver;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRuleSets;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamSqlRelUtils;
+import org.apache.beam.sdk.extensions.sql.meta.provider.ReadOnlyTableProvider;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.joda.time.Duration;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for identifiers. */
+@RunWith(JUnit4.class)
+public class JoinCompoundIdentifiersTestZetaSQL {
+
+ private static final Long TWO_MINUTES = 2L;
+ private static final String DEFAULT_SCHEMA = "beam";
+ private static final String FULL_ON_ID =
+ "a.`b-\\`c`.d.`httz://d.e-f.g:233333/blah\\?yes=1&true=false`";
+ private static final String TABLE_WITH_STRUCTS_ID = "a.`table:com`.`..::with-struct::..`";
+
+ private static final TableProvider TEST_TABLES =
+ new ReadOnlyTableProvider(
+ "test_table_provider",
+ ImmutableMap.<String, BeamSqlTable>builder()
+ .put("KeyValue", BASIC_TABLE_ONE)
+ .put("a.b", BASIC_TABLE_ONE)
+ .put("c.d.e", BASIC_TABLE_ONE)
+ .put("c.d.f", BASIC_TABLE_TWO)
+ .put("c.g.e", BASIC_TABLE_TWO)
+ .put("weird.`\\n\\t\\r\\f`", BASIC_TABLE_ONE)
+ .put("a.`b-\\`c`.d", BASIC_TABLE_TWO)
+ .put(FULL_ON_ID, BASIC_TABLE_TWO)
+ .put(TABLE_WITH_STRUCTS_ID, TABLE_WITH_STRUCT)
+ .build());
+
+ @Rule public transient TestPipeline pipeline = TestPipeline.create();
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testComplexTableName() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT Key FROM a.b");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testComplexTableName3Levels() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT Key FROM c.d.e");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testOnePartWithBackticks() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT RowKey FROM a.`b-\\`c`.d");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(16L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testNewLinesAndOtherWhitespace() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(pipeline, cfg, "SELECT Key FROM weird.`\\n\\t\\r\\f`");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testFullOnWithBackticks() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT RowKey FROM " + FULL_ON_ID);
+
+ PAssert.that(result).containsInAnyOrder(singleValue(16L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testJoinWithFullOnWithBackticks() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(
+ pipeline,
+ cfg,
+ "SELECT t1.RowKey FROM "
+ + FULL_ON_ID
+ + " AS t1 \n"
+ + " INNER JOIN a.`b-\\`c`.d t2 on t1.RowKey = t2.RowKey");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(16L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedFieldAccessWithAliasedComplexTableName() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT t.Key FROM a.b AS t");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedFieldAccessWithAliasedComplexTableName3Levels() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT t.Key FROM c.d.e AS t");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedFieldAccessWithUnaliasedComplexTableName() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT b.Key FROM a.b");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedFieldAccessWithUnaliasedComplexTableName3Levels() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT e.Key FROM c.d.e");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedFieldAccessWithUnaliasedComplexTableName3Levels2() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result = applySqlTransform(pipeline, cfg, "SELECT e.Key FROM c.d.e");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedFieldAccessWithJoinOfAliasedComplexTableNames() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(
+ pipeline,
+ cfg,
+ "SELECT t1.Key FROM a.b AS t1 INNER JOIN c.d.e AS t2 ON t1.Key = t2.Key");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testJoinTwoTablesWithLastPartIdDifferent() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(
+ pipeline,
+ cfg,
+ "SELECT t1.Key FROM c.d.e AS t1 INNER JOIN c.d.f AS t2 ON t1.Key = t2.RowKey");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testJoinTwoTablesWithMiddlePartIdDifferent() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(
+ pipeline,
+ cfg,
+ "SELECT t1.Key FROM c.d.e AS t1 INNER JOIN c.g.e AS t2 ON t1.Key = t2.RowKey");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedFieldAccessWithJoinOfUnaliasedComplexTableNames() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(pipeline, cfg, "SELECT b.Key FROM a.b INNER JOIN c.d.e ON b.Key = e.Key");
+
+ PAssert.that(result).containsInAnyOrder(singleValue(14L), singleValue(15L));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testStructFieldAccess() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(
+ pipeline,
+ cfg,
+ "SELECT struct_col.struct_col_str FROM a.`table:com`.`..::with-struct::..`");
+
+ PAssert.that(result).containsInAnyOrder(singleValue("row_one"), singleValue("row_two"));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testQualifiedStructFieldAccess() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(
+ pipeline,
+ cfg,
+ "SELECT `..::with-struct::..`.struct_col.struct_col_str \n"
+ + " FROM a.`table:com`.`..::with-struct::..`");
+
+ PAssert.that(result).containsInAnyOrder(singleValue("row_one"), singleValue("row_two"));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @Test
+ public void testAliasedStructFieldAccess() throws Exception {
+ FrameworkConfig cfg = initializeCalcite();
+
+ PCollection<Row> result =
+ applySqlTransform(
+ pipeline,
+ cfg,
+ "SELECT t.struct_col.struct_col_str FROM " + TABLE_WITH_STRUCTS_ID + " t");
+
+ PAssert.that(result).containsInAnyOrder(singleValue("row_one"), singleValue("row_two"));
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(TWO_MINUTES));
+ }
+
+ @SuppressWarnings("unxchecked")
+ private static FrameworkConfig initializeCalcite() {
+ JdbcConnection jdbcConnection =
+ JdbcDriver.connect(TEST_TABLES, PipelineOptionsFactory.create());
+ SchemaPlus defaultSchemaPlus = jdbcConnection.getCurrentSchemaPlus();
+ List<RelTraitDef> traitDefs = ImmutableList.of(ConventionTraitDef.INSTANCE);
+
+ Object[] contexts =
+ ImmutableList.of(
+ Contexts.of(jdbcConnection.config()),
+ TableResolutionContext.joinCompoundIds(DEFAULT_SCHEMA))
+ .toArray();
+
+ return Frameworks.newConfigBuilder()
+ .defaultSchema(defaultSchemaPlus)
+ .traitDefs(traitDefs)
+ .context(Contexts.of(contexts))
+ .ruleSets(BeamRuleSets.getRuleSets())
+ .costFactory(null)
+ .typeSystem(jdbcConnection.getTypeFactory().getTypeSystem())
+ .build();
+ }
+
+ private PCollection<Row> applySqlTransform(
+ Pipeline pipeline, FrameworkConfig config, String query) throws Exception {
+
+ BeamRelNode beamRelNode = new ZetaSQLQueryPlanner(config).parseQuery(query);
+ return BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ }
+
+ private Row singleValue(long value) {
+ return Row.withSchema(singleLongField()).addValue(value).build();
+ }
+
+ private Row singleValue(String value) {
+ return Row.withSchema(singleStringField()).addValue(value).build();
+ }
+
+ private Schema singleLongField() {
+ return Schema.builder().addInt64Field("field1").build();
+ }
+
+ private Schema singleStringField() {
+ return Schema.builder().addStringField("field1").build();
+ }
+}
diff --git a/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLDialectSpecTestZetaSQL.java b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLDialectSpecTestZetaSQL.java
new file mode 100644
index 0000000..fb3f811
--- /dev/null
+++ b/sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSQLDialectSpecTestZetaSQL.java
@@ -0,0 +1,3800 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.extensions.sql.zetasql;
+
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseDate;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseDateToValue;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTime;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimeToValue;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimestampWithTZToValue;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimestampWithTimeZone;
+import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimestampWithUTCTimeZone;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.AGGREGATE_TABLE_ONE;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.AGGREGATE_TABLE_TWO;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.BASIC_TABLE_ONE;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.BASIC_TABLE_THREE;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.BASIC_TABLE_TWO;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_ALL_NULL;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_ALL_TYPES;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_ALL_TYPES_2;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_EMPTY;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_FOR_CASE_WHEN;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_WITH_ARRAY;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_WITH_ARRAY_FOR_UNNEST;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_WITH_MAP;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_WITH_STRUCT;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_WITH_STRUCT_TIMESTAMP_STRING;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TABLE_WITH_STRUCT_TWO;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TIMESTAMP_TABLE_ONE;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TIMESTAMP_TABLE_TWO;
+import static org.apache.beam.sdk.extensions.sql.zetasql.TestInput.TIME_TABLE;
+import static org.apache.beam.sdk.schemas.Schema.FieldType.DATETIME;
+
+import com.google.protobuf.ByteString;
+import com.google.zetasql.SqlException;
+import com.google.zetasql.StructType.StructField;
+import com.google.zetasql.TypeFactory;
+import com.google.zetasql.Value;
+import com.google.zetasql.ZetaSQLType.TypeKind;
+import com.google.zetasql.ZetaSQLValue.ValueProto;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.beam.sdk.extensions.sql.BeamSqlTable;
+import org.apache.beam.sdk.extensions.sql.impl.JdbcConnection;
+import org.apache.beam.sdk.extensions.sql.impl.JdbcDriver;
+import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRuleSets;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode;
+import org.apache.beam.sdk.extensions.sql.impl.rel.BeamSqlRelUtils;
+import org.apache.beam.sdk.extensions.sql.meta.provider.ReadOnlyTableProvider;
+import org.apache.beam.sdk.extensions.sql.meta.provider.TableProvider;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.Schema.Field;
+import org.apache.beam.sdk.schemas.Schema.FieldType;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap;
+import org.apache.calcite.plan.Context;
+import org.apache.calcite.plan.Contexts;
+import org.apache.calcite.plan.ConventionTraitDef;
+import org.apache.calcite.plan.RelTraitDef;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.chrono.ISOChronology;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** ZetaSQLDialectSpecTest. */
+@RunWith(JUnit4.class)
+public class ZetaSQLDialectSpecTestZetaSQL {
+ private static final Long PIPELINE_EXECUTION_WAITTIME_MINUTES = 2L;
+
+ private FrameworkConfig config;
+
+ private TableProvider tableProvider;
+
+ @Rule public transient TestPipeline pipeline = TestPipeline.create();
+ @Rule public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() {
+ initializeBeamTableProvider();
+ initializeCalciteEnvironment();
+ }
+
+ @Test
+ public void testSimpleSelect() {
+ String sql =
+ "SELECT CAST (1243 as INT64), "
+ + "CAST ('2018-09-15 12:59:59.000000+00' as TIMESTAMP), "
+ + "CAST ('string' as STRING);";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addDateTimeField("field2")
+ .addStringField("field3")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 1243L,
+ new DateTime(2018, 9, 15, 12, 59, 59, ISOChronology.getInstanceUTC()),
+ "string")
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testEQ1() {
+ String sql = "SELECT @p0 = @p1 AS ColA";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createSimpleNullValue(TypeKind.TYPE_BOOL))
+ .put("p1", Value.createBoolValue(true))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Boolean) null).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore(
+ "Does not support inf/-inf/nan in double/float literals because double/float literals are"
+ + " converted to BigDecimal in Calcite codegen.")
+ public void testEQ2() {
+ String sql = "SELECT @p0 = @p1 AS ColA";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createDoubleValue(0))
+ .put("p1", Value.createDoubleValue(Double.POSITIVE_INFINITY))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addBooleanField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(false).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testEQ3() {
+ String sql = "SELECT @p0 = @p1 AS ColA";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createSimpleNullValue(TypeKind.TYPE_DOUBLE))
+ .put("p1", Value.createDoubleValue(3.14))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Boolean) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testEQ4() {
+ String sql = "SELECT @p0 = @p1 AS ColA";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createBytesValue(ByteString.copyFromUtf8("hello")))
+ .put("p1", Value.createBytesValue(ByteString.copyFromUtf8("hello")))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testEQ5() {
+ String sql = "SELECT b'hello' = b'hello' AS ColA";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIsNotNull1() {
+ String sql = "SELECT @p0 IS NOT NULL AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(false).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIsNotNull2() {
+ String sql = "SELECT @p0 IS NOT NULL AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createNullValue(
+ TypeFactory.createArrayType(TypeFactory.createSimpleType(TypeKind.TYPE_INT64))));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(false).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Does not support struct literal.")
+ public void testIsNotNull3() {
+ String sql = "SELECT @p0 IS NOT NULL AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createNullValue(
+ TypeFactory.createStructType(
+ Arrays.asList(
+ new StructField(
+ "a", TypeFactory.createSimpleType(TypeKind.TYPE_STRING))))));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(false).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIfBasic() {
+ String sql = "SELECT IF(@p0, @p1, @p2) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createBoolValue(true),
+ "p1",
+ Value.createInt64Value(1),
+ "p2",
+ Value.createInt64Value(2));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.INT64).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(1L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCoalesceBasic() {
+ String sql = "SELECT COALESCE(@p0, @p1, @p2) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p1",
+ Value.createStringValue("yay"),
+ "p2",
+ Value.createStringValue("nay"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("yay").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCoalesceSingleArgument() {
+ String sql = "SELECT COALESCE(@p0) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createSimpleNullValue(TypeKind.TYPE_INT64));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder().addNullableField("field1", FieldType.array(FieldType.INT64)).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValue(null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCoalesceNullArray() {
+ String sql = "SELECT COALESCE(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createNullValue(
+ TypeFactory.createArrayType(TypeFactory.createSimpleType(TypeKind.TYPE_INT64))),
+ "p1",
+ Value.createNullValue(
+ TypeFactory.createArrayType(TypeFactory.createSimpleType(TypeKind.TYPE_INT64))));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder().addNullableField("field1", FieldType.array(FieldType.INT64)).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValue(null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testNullIfCoercion() {
+ String sql = "SELECT NULLIF(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createInt64Value(3L),
+ "p1",
+ Value.createSimpleNullValue(TypeKind.TYPE_DOUBLE));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.DOUBLE).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValue(3.0).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Struct literals are not currently supported")
+ public void testCoalesceNullStruct() {
+ String sql = "SELECT COALESCE(NULL, STRUCT(\"a\" AS s, -33 AS i))";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema innerSchema =
+ Schema.of(Field.of("s", FieldType.STRING), Field.of("i", FieldType.INT64));
+ final Schema schema =
+ Schema.builder().addNullableField("field1", FieldType.row(innerSchema)).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValue(Row.withSchema(innerSchema).addValues("a", -33).build())
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIfTimestamp() {
+ String sql = "SELECT IF(@p0, @p1, @p2) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createBoolValue(false),
+ "p1",
+ Value.createTimestampValueFromUnixMicros(0),
+ "p2",
+ Value.createTimestampValueFromUnixMicros(
+ DateTime.parse("2019-01-01T00:00:00Z").getMillis() * 1000));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", DATETIME).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(DateTime.parse("2019-01-01T00:00:00Z")).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("$make_array is not implemented")
+ public void testMakeArray() {
+ String sql = "SELECT [s3, s1, s2] FROM (SELECT \"foo\" AS s1, \"bar\" AS s2, \"baz\" AS s3);";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder().addNullableField("field1", FieldType.array(FieldType.STRING)).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValue(ImmutableList.of("baz", "foo", "bar")).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testNullIfPositive() {
+ String sql = "SELECT NULLIF(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("null"), "p1", Value.createStringValue("null"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValue(null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testNullIfNegative() {
+ String sql = "SELECT NULLIF(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("foo"), "p1", Value.createStringValue("null"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("foo").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIfNullPositive() {
+ String sql = "SELECT IFNULL(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("foo"), "p1", Value.createStringValue("default"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("foo").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIfNullNegative() {
+ String sql = "SELECT IFNULL(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p1",
+ Value.createStringValue("yay"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("yay").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Throws IndexOutOfBoundsException")
+ public void testConstructEmptyArrayLiteral() {
+ String sql = "SELECT @p0 AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createArrayValue(
+ TypeFactory.createArrayType(TypeFactory.createSimpleType(TypeKind.TYPE_INT64)),
+ ImmutableList.of()));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addArrayField("field1", FieldType.INT64).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValue(ImmutableList.of()).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("")
+ public void testLike1() {
+ String sql = "SELECT @p0 LIKE @p1 AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("ab%"), "p1", Value.createStringValue("ab\\%"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testLikeNullPattern() {
+ String sql = "SELECT @p0 LIKE @p1 AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createStringValue("ab%"),
+ "p1",
+ Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Object) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("")
+ public void testLikeAllowsEscapingNonSpecialCharacter() {
+ String sql = "SELECT @p0 LIKE @p1 AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createStringValue("ab"), "p1", Value.createStringValue("\\ab"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("")
+ public void testLikeAllowsEscapingBackslash() {
+ String sql = "SELECT @p0 LIKE @p1 AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("a\\c"), "p1", Value.createStringValue("a\\\\c"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Currently non UTF-8 values are coerced to UTF-8")
+ public void testThrowsErrorForNonUTF8() {
+ String sql = "SELECT @p0 LIKE @p1 AS ColA";
+ byte[] bytes = {(byte) 0xe8, (byte) 0xb0};
+ Value bad =
+ Value.deserialize(
+ TypeFactory.createSimpleType(TypeKind.TYPE_STRING),
+ ValueProto.newBuilder().setStringValueBytes(ByteString.copyFrom(bytes)).build());
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createStringValue("abc"), "p1", bad);
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ thrown.expect(RuntimeException.class);
+ // TODO: message should be a constant on ZetaSQLPlannerImpl
+ thrown.expectMessage("invalid UTF-8");
+ zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ }
+
+ @Test
+ @Ignore("Does not support BYTES for like")
+ public void testLikeBytes() {
+ String sql = "SELECT @p0 LIKE @p1 AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createBytesValue(ByteString.copyFromUtf8("abcd")),
+ "p1",
+ Value.createBytesValue(ByteString.copyFromUtf8("__%")));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testMod() {
+ String sql = "SELECT MOD(4, 2)";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(0L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSimpleUnionAll() {
+ String sql =
+ "SELECT CAST (1243 as INT64), "
+ + "CAST ('2018-09-15 12:59:59.000000+00' as TIMESTAMP), "
+ + "CAST ('string' as STRING) "
+ + " UNION ALL "
+ + " SELECT CAST (1243 as INT64), "
+ + "CAST ('2018-09-15 12:59:59.000000+00' as TIMESTAMP), "
+ + "CAST ('string' as STRING);";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addDateTimeField("field2")
+ .addStringField("field3")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 1243L,
+ new DateTime(2018, 9, 15, 12, 59, 59, ISOChronology.getInstanceUTC()),
+ "string")
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 1243L,
+ new DateTime(2018, 9, 15, 12, 59, 59, ISOChronology.getInstanceUTC()),
+ "string")
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testThreeWayUnionAll() {
+ String sql = "SELECT a FROM (SELECT 1 a UNION ALL SELECT 2 UNION ALL SELECT 3)";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L).build(),
+ Row.withSchema(schema).addValues(2L).build(),
+ Row.withSchema(schema).addValues(3L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSimpleUnionDISTINCT() {
+ String sql =
+ "SELECT CAST (1243 as INT64), "
+ + "CAST ('2018-09-15 12:59:59.000000+00' as TIMESTAMP), "
+ + "CAST ('string' as STRING) "
+ + " UNION DISTINCT "
+ + " SELECT CAST (1243 as INT64), "
+ + "CAST ('2018-09-15 12:59:59.000000+00' as TIMESTAMP), "
+ + "CAST ('string' as STRING);";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addDateTimeField("field2")
+ .addStringField("field3")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 1243L,
+ new DateTime(2018, 9, 15, 12, 59, 59, ISOChronology.getInstanceUTC()),
+ "string")
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLInnerJoin() {
+ String sql =
+ "SELECT t1.Key "
+ + "FROM KeyValue AS t1"
+ + " INNER JOIN BigTable AS t2"
+ + " on "
+ + " t1.Key = t2.RowKey AND t1.ts = t2.ts";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addInt64Field("field1").build())
+ .addValues(15L)
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ // JOIN USING(col) is equivalent to JOIN on left.col = right.col.
+ public void testZetaSQLInnerJoinWithUsing() {
+ String sql = "SELECT t1.Key " + "FROM KeyValue AS t1" + " INNER JOIN BigTable AS t2 USING(ts)";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addInt64Field("field1").build())
+ .addValues(15L)
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ // testing ordering of the JOIN conditions.
+ public void testZetaSQLInnerJoinTwo() {
+ String sql =
+ "SELECT t2.RowKey "
+ + "FROM KeyValue AS t1"
+ + " INNER JOIN BigTable AS t2"
+ + " on "
+ + " t2.RowKey = t1.Key AND t2.ts = t1.ts";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addInt64Field("field1").build())
+ .addValues(15L)
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLLeftOuterJoin() {
+ String sql =
+ "SELECT * "
+ + "FROM KeyValue AS t1"
+ + " LEFT JOIN BigTable AS t2"
+ + " on "
+ + " t1.Key = t2.RowKey";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schemaOne =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addStringField("field2")
+ .addDateTimeField("field3")
+ .addNullableField("field4", FieldType.INT64)
+ .addNullableField("field5", FieldType.STRING)
+ .addNullableField("field6", DATETIME)
+ .build();
+
+ final Schema schemaTwo =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addStringField("field2")
+ .addDateTimeField("field3")
+ .addInt64Field("field4")
+ .addStringField("field5")
+ .addDateTimeField("field6")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schemaOne)
+ .addValues(
+ 14L,
+ "KeyValue234",
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()),
+ null,
+ null,
+ null)
+ .build(),
+ Row.withSchema(schemaTwo)
+ .addValues(
+ 15L,
+ "KeyValue235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ 15L,
+ "BigTable235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLRightOuterJoin() {
+ String sql =
+ "SELECT * "
+ + "FROM KeyValue AS t1"
+ + " RIGHT JOIN BigTable AS t2"
+ + " on "
+ + " t1.Key = t2.RowKey";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schemaOne =
+ Schema.builder()
+ .addNullableField("field1", FieldType.INT64)
+ .addNullableField("field2", FieldType.STRING)
+ .addNullableField("field3", DATETIME)
+ .addInt64Field("field4")
+ .addStringField("field5")
+ .addDateTimeField("field6")
+ .build();
+
+ final Schema schemaTwo =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addStringField("field2")
+ .addDateTimeField("field3")
+ .addInt64Field("field4")
+ .addStringField("field5")
+ .addDateTimeField("field6")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schemaOne)
+ .addValues(
+ null,
+ null,
+ null,
+ 16L,
+ "BigTable236",
+ new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schemaTwo)
+ .addValues(
+ 15L,
+ "KeyValue235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ 15L,
+ "BigTable235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLFullOuterJoin() {
+ String sql =
+ "SELECT * "
+ + "FROM KeyValue AS t1"
+ + " FULL JOIN BigTable AS t2"
+ + " on "
+ + " t1.Key = t2.RowKey";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schemaOne =
+ Schema.builder()
+ .addNullableField("field1", FieldType.INT64)
+ .addNullableField("field2", FieldType.STRING)
+ .addNullableField("field3", DATETIME)
+ .addInt64Field("field4")
+ .addStringField("field5")
+ .addDateTimeField("field6")
+ .build();
+
+ final Schema schemaTwo =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addStringField("field2")
+ .addDateTimeField("field3")
+ .addInt64Field("field4")
+ .addStringField("field5")
+ .addDateTimeField("field6")
+ .build();
+
+ final Schema schemaThree =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addStringField("field2")
+ .addDateTimeField("field3")
+ .addNullableField("field4", FieldType.INT64)
+ .addNullableField("field5", FieldType.STRING)
+ .addNullableField("field6", DATETIME)
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schemaOne)
+ .addValues(
+ null,
+ null,
+ null,
+ 16L,
+ "BigTable236",
+ new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schemaTwo)
+ .addValues(
+ 15L,
+ "KeyValue235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ 15L,
+ "BigTable235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schemaThree)
+ .addValues(
+ 14L,
+ "KeyValue234",
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()),
+ null,
+ null,
+ null)
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("BeamSQL only supports equal join")
+ public void testZetaSQLFullOuterJoinTwo() {
+ String sql =
+ "SELECT * "
+ + "FROM KeyValue AS t1"
+ + " FULL JOIN BigTable AS t2"
+ + " on "
+ + " t1.Key + t2.RowKey = 30";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLThreeWayInnerJoin() {
+ String sql =
+ "SELECT t3.Value, t2.Value, t1.Value, t1.Key, t3.ColId FROM KeyValue as t1 "
+ + "JOIN BigTable as t2 "
+ + "ON (t1.Key = t2.RowKey) "
+ + "JOIN Spanner as t3 "
+ + "ON (t3.ColId = t1.Key)";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(
+ Schema.builder()
+ .addStringField("t3.Value")
+ .addStringField("t2.Value")
+ .addStringField("t1.Value")
+ .addInt64Field("t1.Key")
+ .addInt64Field("t3.ColId")
+ .build())
+ .addValues("Spanner235", "BigTable235", "KeyValue235", 15L, 15L)
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLTableJoinOnItselfWithFiltering() {
+ String sql =
+ "SELECT * FROM Spanner as t1 "
+ + "JOIN Spanner as t2 "
+ + "ON (t1.ColId = t2.ColId) WHERE t1.ColId = 17";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(
+ Schema.builder()
+ .addInt64Field("field1")
+ .addStringField("field2")
+ .addInt64Field("field3")
+ .addStringField("field4")
+ .build())
+ .addValues(17L, "Spanner237", 17L, "Spanner237")
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLSelectFromSelect() {
+ String sql = "SELECT * FROM (SELECT \"apple\" AS fruit, \"carrot\" AS vegetable);";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder().addStringField("field1").addStringField("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues("apple", "carrot").build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+
+ Schema outputSchema = stream.getSchema();
+ Assert.assertEquals(2, outputSchema.getFieldCount());
+ Assert.assertEquals("fruit", outputSchema.getField(0).getName());
+ Assert.assertEquals("vegetable", outputSchema.getField(1).getName());
+ }
+
+ @Test
+ public void testZetaSQLSelectFromTable() {
+ String sql = "SELECT Key, Value FROM KeyValue;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addStringField("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, "KeyValue234").build(),
+ Row.withSchema(schema).addValues(15L, "KeyValue235").build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLSelectFromTableLimit() {
+ String sql = "SELECT Key, Value FROM KeyValue LIMIT 2;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addStringField("field2").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, "KeyValue234").build(),
+ Row.withSchema(schema).addValues(15L, "KeyValue235").build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLSelectFromTableLimit0() {
+ String sql = "SELECT Key, Value FROM KeyValue LIMIT 0;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ PAssert.that(stream).containsInAnyOrder();
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLSelectFromTableLimitOffset() {
+ String sql =
+ "SELECT COUNT(a) FROM (\n"
+ + "SELECT a FROM (SELECT 1 a UNION ALL SELECT 2 UNION ALL SELECT 3) LIMIT 3 OFFSET 1);";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addInt64Field("field1").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(2L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ // There is really no order for a PCollection, so this query does not test
+ // ORDER BY but just a test to see if ORDER BY LIMIT can work.
+ @Test
+ public void testZetaSQLSelectFromTableOrderByLimit() {
+ String sql = "SELECT Key, Value FROM KeyValue ORDER BY Key DESC LIMIT 2;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addStringField("field2").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, "KeyValue234").build(),
+ Row.withSchema(schema).addValues(15L, "KeyValue235").build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLSelectFromTableOrderBy() {
+ String sql = "SELECT Key, Value FROM KeyValue ORDER BY Key DESC;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("ORDER BY without a LIMIT is not supported.");
+ zetaSQLQueryPlanner.convertToBeamRel(sql);
+ }
+
+ @Test
+ public void testZetaSQLSelectFromTableWithStructType2() {
+ String sql =
+ "SELECT table_with_struct.struct_col.struct_col_str FROM table_with_struct WHERE id = 1;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValue("row_one").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLStructFieldAccessInFilter() {
+ String sql =
+ "SELECT table_with_struct.id FROM table_with_struct WHERE"
+ + " table_with_struct.struct_col.struct_col_str = 'row_one';";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addInt64Field("field").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValue(1L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLStructFieldAccessInCast() {
+ String sql =
+ "SELECT CAST(table_with_struct.id AS STRING) FROM table_with_struct WHERE"
+ + " table_with_struct.struct_col.struct_col_str = 'row_one';";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValue("1").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLStructFieldAccessInCast2() {
+ String sql =
+ "SELECT CAST(A.struct_col.struct_col_str AS TIMESTAMP) FROM table_with_struct_ts_string AS"
+ + " A";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addDateTimeField("field").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValue(parseTimestampWithUTCTimeZone("2019-01-15 13:21:03"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLStructFieldAccessInTumble() {
+ String sql =
+ "SELECT TUMBLE_START('INTERVAL 1 MINUTE') FROM table_with_struct_ts_string AS A GROUP BY "
+ + "TUMBLE(CAST(A.struct_col.struct_col_str AS TIMESTAMP), 'INTERVAL 1 MINUTE')";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addDateTimeField("field").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValue(parseTimestampWithUTCTimeZone("2019-01-15 13:21:00"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLStructFieldAccessInGroupBy() {
+ String sql = "SELECT rowCol.row_id, COUNT(*) FROM table_with_struct_two GROUP BY rowCol.row_id";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 1L).build(),
+ Row.withSchema(schema).addValues(2L, 1L).build(),
+ Row.withSchema(schema).addValues(3L, 2L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLStructFieldAccessInGroupBy2() {
+ String sql =
+ "SELECT rowCol.data, MAX(rowCol.row_id), MIN(rowCol.row_id) FROM table_with_struct_two"
+ + " GROUP BY rowCol.data";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema =
+ Schema.builder()
+ .addStringField("field1")
+ .addInt64Field("field2")
+ .addInt64Field("field3")
+ .build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("data1", 1L, 1L).build(),
+ Row.withSchema(schema).addValues("data2", 3L, 2L).build(),
+ Row.withSchema(schema).addValues("data3", 3L, 3L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLSelectFromTableWithArrayType() {
+ String sql = "SELECT array_col FROM table_with_array;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addArrayField("field", FieldType.STRING).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValue(Arrays.asList("1", "2", "3")).build(),
+ Row.withSchema(schema).addValue(ImmutableList.of()).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLSelectStarFromTable() {
+ String sql = "SELECT * FROM BigTable;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addStringField("field2")
+ .addDateTimeField("field3")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 15L,
+ "BigTable235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 16L,
+ "BigTable236",
+ new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()))
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicFiltering() {
+ String sql = "SELECT Key, Value FROM KeyValue WHERE Key = 14;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(
+ Schema.builder().addInt64Field("field1").addStringField("field2").build())
+ .addValues(14L, "KeyValue234")
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicFilteringTwo() {
+ String sql = "SELECT Key, Value FROM KeyValue WHERE Key = 14 AND Value = 'non-existing';";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ PAssert.that(stream).containsInAnyOrder();
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicFilteringThree() {
+ String sql = "SELECT Key, Value FROM KeyValue WHERE Key = 14 OR Key = 15;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addStringField("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, "KeyValue234").build(),
+ Row.withSchema(schema).addValues(15L, "KeyValue235").build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLCountOnAColumn() {
+ String sql = "SELECT COUNT(Key) FROM KeyValue";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(2L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLAggDistinct() {
+ String sql = "SELECT Key, COUNT(DISTINCT Value) FROM KeyValue GROUP BY Key";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("Does not support COUNT DISTINCT");
+ zetaSQLQueryPlanner.convertToBeamRel(sql);
+ }
+
+ @Test
+ public void testZetaSQLBasicAgg() {
+ String sql = "SELECT Key, COUNT(*) FROM KeyValue GROUP BY Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, 1L).build(),
+ Row.withSchema(schema).addValues(15L, 1L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLColumnAlias1() {
+ String sql = "SELECT Key, COUNT(*) AS count_col FROM KeyValue GROUP BY Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+
+ Schema outputSchema = stream.getSchema();
+ Assert.assertEquals(2, outputSchema.getFieldCount());
+ Assert.assertEquals("Key", outputSchema.getField(0).getName());
+ Assert.assertEquals("count_col", outputSchema.getField(1).getName());
+ }
+
+ @Test
+ public void testZetaSQLColumnAlias2() {
+ String sql =
+ "SELECT Key AS k1, (count_col + 1) AS k2 FROM (SELECT Key, COUNT(*) AS count_col FROM"
+ + " KeyValue GROUP BY Key)";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+
+ Schema outputSchema = stream.getSchema();
+ Assert.assertEquals(2, outputSchema.getFieldCount());
+ Assert.assertEquals("k1", outputSchema.getField(0).getName());
+ Assert.assertEquals("k2", outputSchema.getField(1).getName());
+ }
+
+ @Test
+ public void testZetaSQLColumnAlias3() {
+ String sql = "SELECT Key AS v1, Value AS v2, ts AS v3 FROM KeyValue";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+
+ Schema outputSchema = stream.getSchema();
+ Assert.assertEquals(3, outputSchema.getFieldCount());
+ Assert.assertEquals("v1", outputSchema.getField(0).getName());
+ Assert.assertEquals("v2", outputSchema.getField(1).getName());
+ Assert.assertEquals("v3", outputSchema.getField(2).getName());
+ }
+
+ @Test
+ public void testZetaSQLColumnAlias4() {
+ String sql = "SELECT CAST(123 AS INT64) AS cast_col";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+
+ Schema outputSchema = stream.getSchema();
+ Assert.assertEquals(1, outputSchema.getFieldCount());
+ Assert.assertEquals("cast_col", outputSchema.getField(0).getName());
+ }
+
+ @Test
+ public void testZetaSQLAmbiguousAlias() {
+ String sql = "SELECT row_id as ID, int64_col as ID FROM table_all_types GROUP BY ID;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+
+ thrown.expectMessage(
+ "Name ID in GROUP BY clause is ambiguous; it may refer to multiple columns in the"
+ + " SELECT-list [at 1:68]");
+ zetaSQLQueryPlanner.convertToBeamRel(sql);
+ }
+
+ @Test
+ public void testZetaSQLAggWithOrdinalReference() {
+ String sql = "SELECT Key, COUNT(*) FROM aggregate_test_table GROUP BY 1";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 2L).build(),
+ Row.withSchema(schema).addValues(2L, 3L).build(),
+ Row.withSchema(schema).addValues(3L, 2L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLAggWithAliasReference() {
+ String sql = "SELECT Key AS K, COUNT(*) FROM aggregate_test_table GROUP BY K";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 2L).build(),
+ Row.withSchema(schema).addValues(2L, 3L).build(),
+ Row.withSchema(schema).addValues(3L, 2L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicAgg2() {
+ String sql = "SELECT Key, COUNT(*) FROM aggregate_test_table GROUP BY Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 2L).build(),
+ Row.withSchema(schema).addValues(2L, 3L).build(),
+ Row.withSchema(schema).addValues(3L, 2L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Need further investigation on root cause. Seems a bug of direct runner.")
+ public void testZetaSQLBasicAgg3() {
+ String sql = "SELECT Key, Key2, COUNT(*) FROM aggregate_test_table GROUP BY Key2, Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addInt64Field("field3")
+ .addInt64Field("field2")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 10L, 1L).build(),
+ Row.withSchema(schema).addValues(1L, 11L, 1L).build(),
+ Row.withSchema(schema).addValues(2L, 11L, 2L).build(),
+ Row.withSchema(schema).addValues(2L, 12L, 1L).build(),
+ Row.withSchema(schema).addValues(3L, 13L, 2L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Need further investigation on root cause. Seems a bug of direct runner.")
+ public void testZetaSQLBasicAgg4() {
+ String sql =
+ "SELECT Key, Key2, MAX(f_int_1), MIN(f_int_1), SUM(f_int_1), SUM(f_double_1) "
+ + "FROM aggregate_test_table GROUP BY Key2, Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addInt64Field("field3")
+ .addInt64Field("field2")
+ .addInt64Field("field4")
+ .addInt64Field("field5")
+ .addDoubleField("field6")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 10L, 1L, 1L, 1L, 1.0).build(),
+ Row.withSchema(schema).addValues(1L, 11L, 2L, 2L, 2L, 2.0).build(),
+ Row.withSchema(schema).addValues(2L, 11L, 4L, 3L, 7L, 7.0).build(),
+ Row.withSchema(schema).addValues(2L, 12L, 5L, 5L, 5L, 5.0).build(),
+ Row.withSchema(schema).addValues(3L, 13L, 7L, 6L, 13L, 13.0).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Need further investigation on root cause. Seems a bug of direct runner.")
+ public void testZetaSQLBasicAgg5() {
+ String sql =
+ "SELECT Key, Key2, AVG(CAST(f_int_1 AS FLOAT64)), AVG(f_double_1) "
+ + "FROM aggregate_test_table GROUP BY Key2, Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addInt64Field("field2")
+ .addDoubleField("field3")
+ .addDoubleField("field4")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 10L, 1.0, 1.0).build(),
+ Row.withSchema(schema).addValues(1L, 11L, 2.0, 2.0).build(),
+ Row.withSchema(schema).addValues(2L, 11L, 3.5, 3.5).build(),
+ Row.withSchema(schema).addValues(2L, 12L, 5.0, 5.0).build(),
+ Row.withSchema(schema).addValues(3L, 13L, 6.5, 6.5).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore(
+ "Calcite infers return type of AVG(int64) as BIGINT while ZetaSQL requires it as either"
+ + " NUMERIC or DOUBLE/FLOAT64")
+ public void testZetaSQLTestAVG() {
+ String sql = "SELECT Key, AVG(f_int_1)" + "FROM aggregate_test_table GROUP BY Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addInt64Field("field2")
+ .addInt64Field("field3")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 10L, 1L).build(),
+ Row.withSchema(schema).addValues(1L, 11L, 6L).build(),
+ Row.withSchema(schema).addValues(2L, 11L, 6L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLGroupByExprInSelect() {
+ String sql = "SELECT int64_col + 1 FROM table_all_types GROUP BY int64_col + 1;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValue(0L).build(),
+ Row.withSchema(schema).addValue(-1L).build(),
+ Row.withSchema(schema).addValue(-2L).build(),
+ Row.withSchema(schema).addValue(-3L).build(),
+ Row.withSchema(schema).addValue(-4L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLGroupByAndFiltering() {
+ String sql = "SELECT int64_col FROM table_all_types WHERE int64_col = 1 GROUP BY int64_col;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ PAssert.that(stream).containsInAnyOrder();
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLGroupByAndFilteringOnNonGroupByColumn() {
+ String sql = "SELECT int64_col FROM table_all_types WHERE double_col = 0.5 GROUP BY int64_col;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addInt64Field("field").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValue(-5L).build(),
+ Row.withSchema(schema).addValue(-4L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicHaving() {
+ String sql = "SELECT Key, COUNT(*) FROM aggregate_test_table GROUP BY Key HAVING COUNT(*) > 2";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(2L, 3L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicFixedWindowing() {
+ String sql =
+ "SELECT "
+ + "COUNT(*) as field_count, "
+ + "TUMBLE_START(\"INTERVAL 1 SECOND\") as window_start, "
+ + "TUMBLE_END(\"INTERVAL 1 SECOND\") as window_end "
+ + "FROM KeyValue "
+ + "GROUP BY TUMBLE(ts, \"INTERVAL 1 SECOND\");";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("count_start")
+ .addDateTimeField("field1")
+ .addDateTimeField("field2")
+ .build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 1L,
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 1L,
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicSlidingWindowing() {
+ String sql =
+ "SELECT "
+ + "COUNT(*) as field_count, "
+ + "HOP_START(\"INTERVAL 1 SECOND\", \"INTERVAL 2 SECOND\") as window_start, "
+ + "HOP_END(\"INTERVAL 1 SECOND\", \"INTERVAL 2 SECOND\") as window_end "
+ + "FROM window_test_table "
+ + "GROUP BY HOP(ts, \"INTERVAL 1 SECOND\", \"INTERVAL 2 SECOND\");";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("count_star")
+ .addDateTimeField("field1")
+ .addDateTimeField("field2")
+ .build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 2L,
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 9, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 1L,
+ new DateTime(2018, 7, 1, 21, 26, 5, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 2L,
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 2L,
+ new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 10, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 1L,
+ new DateTime(2018, 7, 1, 21, 26, 9, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 11, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLBasicSessionWindowing() {
+ String sql =
+ "SELECT "
+ + "COUNT(*) as field_count, "
+ + "SESSION_START(\"INTERVAL 3 SECOND\") as window_start, "
+ + "SESSION_END(\"INTERVAL 3 SECOND\") as window_end "
+ + "FROM window_test_table_two "
+ + "GROUP BY SESSION(ts, \"INTERVAL 3 SECOND\");";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("count_star")
+ .addDateTimeField("field1")
+ .addDateTimeField("field2")
+ .build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 2L,
+ new DateTime(2018, 7, 1, 21, 26, 12, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 12, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 2L,
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ // Test nested selection
+ @Test
+ public void testZetaSQLNestedQueryOne() {
+ String sql =
+ "SELECT a.Value, a.Key FROM (SELECT Key, Value FROM KeyValue WHERE Key = 14 OR Key = 15)"
+ + " as a;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field2").addInt64Field("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("KeyValue234", 14L).build(),
+ Row.withSchema(schema).addValues("KeyValue235", 15L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ // Test selection, filtering and aggregation combined query.
+ @Test
+ @Ignore("Need further investigation on root cause. Seems a bug of direct runner.")
+ public void testZetaSQLNestedQueryTwo() {
+ String sql =
+ "SELECT a.Key, a.Key2, COUNT(*) FROM "
+ + " (SELECT * FROM aggregate_test_table WHERE Key != 10) as a "
+ + " GROUP BY a.Key2, a.Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("field1")
+ .addInt64Field("field3")
+ .addInt64Field("field2")
+ .build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L, 10L, 1L).build(),
+ Row.withSchema(schema).addValues(1L, 11L, 1L).build(),
+ Row.withSchema(schema).addValues(2L, 11L, 2L).build(),
+ Row.withSchema(schema).addValues(2L, 12L, 1L).build(),
+ Row.withSchema(schema).addValues(3L, 13L, 2L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ // test selection and join combined query
+ @Test
+ public void testZetaSQLNestedQueryThree() {
+ String sql =
+ "SELECT * FROM (SELECT * FROM KeyValue) AS t1 INNER JOIN (SELECT * FROM BigTable) AS t2 on"
+ + " t1.Key = t2.RowKey";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(
+ Schema.builder()
+ .addInt64Field("Key")
+ .addStringField("Value")
+ .addDateTimeField("ts")
+ .addInt64Field("RowKey")
+ .addStringField("Value2")
+ .addDateTimeField("ts2")
+ .build())
+ .addValues(
+ 15L,
+ "KeyValue235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ 15L,
+ "BigTable235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testZetaSQLNestedQueryFour() {
+ String sql =
+ "SELECT t1.Value, TUMBLE_START('INTERVAL 1 SECOND') AS period_start, MIN(t2.Value) as"
+ + " min_v FROM KeyValue AS t1 INNER JOIN BigTable AS t2 on t1.Key = t2.RowKey GROUP BY"
+ + " t1.Value, TUMBLE(t2.ts, 'INTERVAL 1 SECOND')";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(
+ Schema.builder()
+ .addStringField("value")
+ .addDateTimeField("min_v")
+ .addStringField("period_start")
+ .build())
+ .addValues(
+ "KeyValue235",
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ "BigTable235")
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ // Test nested select with out of order columns.
+ @Test
+ public void testZetaSQLNestedQueryFive() {
+ String sql =
+ "SELECT a.Value, a.Key FROM (SELECT Value, Key FROM KeyValue WHERE Key = 14 OR Key = 15)"
+ + " as a;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field2").addInt64Field("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("KeyValue234", 14L).build(),
+ Row.withSchema(schema).addValues("KeyValue235", 15L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Does not support DATE_ADD and TIME_ADD.")
+ public void testDateAndTimeAddSub() {
+ String sql =
+ "SELECT "
+ + "DATE_ADD(DATE '2008-12-25', INTERVAL 5 DAY), "
+ + "TIME_ADD(TIME '13:24:30', INTERVAL 3 HOUR)";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(
+ Schema.builder()
+ .addDateTimeField("f_date_plus")
+ .addDateTimeField("f_time_plus")
+ .build())
+ .addValues(parseDate("2008-12-30"), parseTime("16:24:30"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testTimestampAddSub() {
+ String sql =
+ "SELECT "
+ + "TIMESTAMP_ADD(TIMESTAMP '2008-12-25 15:30:00 UTC', INTERVAL 10 MINUTE), "
+ + "TIMESTAMP_ADD(TIMESTAMP '2008-12-25 15:30:00+07:30', INTERVAL 10 MINUTE)";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(
+ Schema.builder()
+ .addDateTimeField("f_timestamp_plus")
+ .addDateTimeField("f_timestamp_with_time_zone_plus")
+ .build())
+ .addValues(
+ DateTimeUtils.parseTimestampWithUTCTimeZone("2008-12-25 15:40:00"),
+ parseTimestampWithTimeZone("2008-12-25 15:40:00+0730"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testTimeZone() {
+ String sql = "SELECT TIMESTAMP '2018-12-10 10:38:59-10:00'";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addDateTimeField("f_timestamp_with_time_zone").build())
+ .addValues(parseTimestampWithTimeZone("2018-12-10 10:38:59-1000"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testDistinct() {
+ String sql = "SELECT DISTINCT Key2 FROM aggregate_test_table";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema schema = Schema.builder().addInt64Field("Key2").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(10L).build(),
+ Row.withSchema(schema).addValues(11L).build(),
+ Row.withSchema(schema).addValues(12L).build(),
+ Row.withSchema(schema).addValues(13L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testDistinctOnNull() {
+ String sql = "SELECT DISTINCT str_val FROM all_null_table";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema schema = Schema.builder().addNullableField("str_val", FieldType.DOUBLE).build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Object) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("BeamSQL does not support ANY_VALUE")
+ public void testAnyValue() {
+ String sql = "SELECT ANY_VALUE(double_val) FROM all_null_table";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema schema = Schema.builder().addNullableField("double_val", FieldType.DOUBLE).build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Object) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectNULL() {
+ String sql = "SELECT NULL";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema schema = Schema.builder().addNullableField("long_val", FieldType.INT64).build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Object) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testWithQueryOne() {
+ String sql =
+ "With T1 AS (SELECT * FROM KeyValue), T2 AS (SELECT * FROM BigTable) SELECT T2.RowKey FROM"
+ + " T1 INNER JOIN T2 on T1.Key = T2.RowKey;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addInt64Field("field1").build())
+ .addValues(15L)
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testWithQueryTwo() {
+ String sql =
+ "WITH T1 AS (SELECT Key, COUNT(*) as value FROM KeyValue GROUP BY Key) SELECT T1.Key,"
+ + " T1.value FROM T1";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, 1L).build(),
+ Row.withSchema(schema).addValues(15L, 1L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testWithQueryThree() {
+ String sql =
+ "WITH T1 as (SELECT Value, Key FROM KeyValue WHERE Key = 14 OR Key = 15) SELECT T1.Value,"
+ + " T1.Key FROM T1;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("KeyValue234", 14L).build(),
+ Row.withSchema(schema).addValues("KeyValue235", 15L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testWithQueryFour() {
+ String sql =
+ "WITH T1 as (SELECT Value, Key FROM KeyValue) SELECT T1.Value, T1.Key FROM T1 WHERE T1.Key"
+ + " = 14 OR T1.Key = 15;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field2").addInt64Field("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("KeyValue234", 14L).build(),
+ Row.withSchema(schema).addValues("KeyValue235", 15L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testWithQueryFive() {
+ String sql =
+ "WITH T1 AS (SELECT * FROM KeyValue) SELECT T1.Key, COUNT(*) FROM T1 GROUP BY T1.Key";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addInt64Field("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, 1L).build(),
+ Row.withSchema(schema).addValues(15L, 1L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testWithQuerySix() {
+ String sql =
+ "WITH T1 AS (SELECT * FROM window_test_table_two) SELECT "
+ + "COUNT(*) as field_count, "
+ + "SESSION_START(\"INTERVAL 3 SECOND\") as window_start, "
+ + "SESSION_END(\"INTERVAL 3 SECOND\") as window_end "
+ + "FROM T1 "
+ + "GROUP BY SESSION(ts, \"INTERVAL 3 SECOND\");";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("count_star")
+ .addDateTimeField("field1")
+ .addDateTimeField("field2")
+ .build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 2L,
+ new DateTime(2018, 7, 1, 21, 26, 12, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 12, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 2L,
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testWithQuerySeven() {
+ String sql =
+ "WITH T1 AS (SELECT * FROM KeyValue) SELECT "
+ + "COUNT(*) as field_count, "
+ + "TUMBLE_START(\"INTERVAL 1 SECOND\") as window_start, "
+ + "TUMBLE_END(\"INTERVAL 1 SECOND\") as window_end "
+ + "FROM T1 "
+ + "GROUP BY TUMBLE(ts, \"INTERVAL 1 SECOND\");";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addInt64Field("count_start")
+ .addDateTimeField("field1")
+ .addDateTimeField("field2")
+ .build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ 1L,
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 8, ISOChronology.getInstanceUTC()))
+ .build(),
+ Row.withSchema(schema)
+ .addValues(
+ 1L,
+ new DateTime(2018, 7, 1, 21, 26, 6, ISOChronology.getInstanceUTC()),
+ new DateTime(2018, 7, 1, 21, 26, 7, ISOChronology.getInstanceUTC()))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testUNNESTLiteral() {
+ String sql = "SELECT * FROM UNNEST(ARRAY<STRING>['foo', 'bar']);";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ Schema schema = Schema.builder().addStringField("str_field").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("foo").build(),
+ Row.withSchema(schema).addValues("bar").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testNamedUNNESTLiteral() {
+ String sql = "SELECT *, T1 FROM UNNEST(ARRAY<STRING>['foo', 'bar']) AS T1";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ Schema schema =
+ Schema.builder().addStringField("str_field").addStringField("str2_field").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("foo", "foo").build(),
+ Row.withSchema(schema).addValues("bar", "bar").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Seeing exception in Beam, need further investigation on the cause of this failed query.")
+ public void testNamedUNNESTJoin() {
+ String sql =
+ "SELECT * "
+ + "FROM table_with_array_for_unnest AS t1"
+ + " LEFT JOIN UNNEST(t1.int_array_col) AS t2"
+ + " on "
+ + " t1.int_col = t2";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream).containsInAnyOrder();
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCaseNoValue() {
+ String sql = "SELECT CASE WHEN 1 > 2 THEN 'not possible' ELSE 'seems right' END";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addStringField("str_field").build())
+ .addValue("seems right")
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCaseWithValue() {
+ String sql = "SELECT CASE 1 WHEN 2 THEN 'not possible' ELSE 'seems right' END";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addStringField("str_field").build())
+ .addValue("seems right")
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCaseWithValueMultipleCases() {
+ String sql =
+ "SELECT CASE 2 WHEN 1 THEN 'not possible' WHEN 2 THEN 'seems right' ELSE 'also not"
+ + " possible' END";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addStringField("str_field").build())
+ .addValue("seems right")
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCaseWithValueNoElse() {
+ String sql = "SELECT CASE 2 WHEN 1 THEN 'not possible' WHEN 2 THEN 'seems right' END";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addStringField("str_field").build())
+ .addValue("seems right")
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCaseNoValueNoElseNoMatch() {
+ String sql = "SELECT CASE WHEN 'abc' = '123' THEN 'not possible' END";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addNullableField("str_field", FieldType.STRING).build())
+ .addValue(null)
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCaseWithValueNoElseNoMatch() {
+ String sql = "SELECT CASE 2 WHEN 1 THEN 'not possible' END";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addNullableField("str_field", FieldType.STRING).build())
+ .addValue(null)
+ .build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore(
+ "Codegen generates code that Janino cannot compile, need further investigation on root"
+ + " cause.")
+ public void testCastToDateWithCase() {
+ String sql =
+ "SELECT f_int, \n"
+ + "CASE WHEN CHAR_LENGTH(TRIM(f_string)) = 8 \n"
+ + " THEN CAST (CONCAT(\n"
+ + " SUBSTR(TRIM(f_string), 0, 4) \n"
+ + " , '-' \n"
+ + " , SUBSTR(TRIM(f_string), 4, 2) \n"
+ + " , '-' \n"
+ + " , SUBSTR(TRIM(f_string), 6, 2)) AS DATE)\n"
+ + " ELSE NULL\n"
+ + "END \n"
+ + "FROM table_for_case_when";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema resultType =
+ Schema.builder().addInt32Field("f_int").addNullableField("f_date", DATETIME).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(resultType).addValues(1, parseDate("2018-10-18")).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIntersectAll() {
+ String sql =
+ "SELECT Key FROM aggregate_test_table "
+ + "INTERSECT ALL "
+ + "SELECT Key FROM aggregate_test_table_two";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema resultType = Schema.builder().addInt64Field("field").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(resultType).addValues(1L).build(),
+ Row.withSchema(resultType).addValues(2L).build(),
+ Row.withSchema(resultType).addValues(2L).build(),
+ Row.withSchema(resultType).addValues(2L).build(),
+ Row.withSchema(resultType).addValues(3L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testIntersectDistinct() {
+ String sql =
+ "SELECT Key FROM aggregate_test_table "
+ + "INTERSECT DISTINCT "
+ + "SELECT Key FROM aggregate_test_table_two";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema resultType = Schema.builder().addInt64Field("field").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(resultType).addValues(1L).build(),
+ Row.withSchema(resultType).addValues(2L).build(),
+ Row.withSchema(resultType).addValues(3L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testExceptAll() {
+ String sql =
+ "SELECT Key FROM aggregate_test_table "
+ + "EXCEPT ALL "
+ + "SELECT Key FROM aggregate_test_table_two";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ Schema resultType = Schema.builder().addInt64Field("field").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(resultType).addValues(1L).build(),
+ Row.withSchema(resultType).addValues(3L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectFromEmptyTable() {
+ String sql = "SELECT * FROM table_empty;";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ PAssert.that(stream).containsInAnyOrder();
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testStartsWithString() {
+ String sql = "SELECT STARTS_WITH('string1', 'stri')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(true).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testStartsWithString2() {
+ String sql = "SELECT STARTS_WITH(@p0, @p1)";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING))
+ .put("p1", Value.createStringValue(""))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Boolean) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testStartsWithString3() {
+ String sql = "SELECT STARTS_WITH(@p0, @p1)";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING))
+ .put("p1", Value.createSimpleNullValue(TypeKind.TYPE_STRING))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Boolean) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testEndsWithString() {
+ String sql = "SELECT STARTS_WITH('string1', 'ng0')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(false).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testEndsWithString2() {
+ String sql = "SELECT STARTS_WITH(@p0, @p1)";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING))
+ .put("p1", Value.createStringValue(""))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Boolean) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testEndsWithString3() {
+ String sql = "SELECT STARTS_WITH(@p0, @p1)";
+
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING))
+ .put("p1", Value.createSimpleNullValue(TypeKind.TYPE_STRING))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.BOOLEAN).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((Boolean) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Does not support DateTime literal.")
+ public void testDateTimeLiteral() {
+ String sql = "SELECT DATETIME '2018-01-01 05:30:00.334'";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ thrown.expect(RuntimeException.class);
+ thrown.expectMessage("Unsupported ResolvedLiteral type: DATETIME");
+ zetaSQLQueryPlanner.convertToBeamRel(sql);
+ }
+
+ @Test
+ public void testTimeStampLiteral() {
+ String sql = "SELECT TIMESTAMP '2016-12-25 05:30:00+00'";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addDateTimeField("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(parseTimestampWithUTCTimeZone("2016-12-25 05:30:00"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testTimeStampLiteralWithoutTimeZone() {
+ String sql = "SELECT TIMESTAMP '2016-12-25 05:30:00'";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addDateTimeField("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(parseTimestampWithUTCTimeZone("2016-12-25 05:30:00"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testTimeStampLiteralWithNonUTCTimeZone() {
+ String sql = "SELECT TIMESTAMP '2016-12-25 05:30:00+05'";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addDateTimeField("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(parseTimestampWithTimeZone("2016-12-25 05:30:00+05"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testConcatWithOneParameters() {
+ String sql = "SELECT concat('abc')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("abc").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testConcatWithTwoParameters() {
+ String sql = "SELECT concat('abc', 'def')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("abcdef").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testConcatWithThreeParameters() {
+ String sql = "SELECT concat('abc', 'def', 'xyz')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("abcdefxyz").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testConcatWithFourParameters() {
+ String sql = "SELECT concat('abc', 'def', ' ', 'xyz')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues("abcdef xyz").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testConcatWithFiveParameters() {
+ String sql = "SELECT concat('abc', 'def', ' ', 'xyz', 'kkk')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues("abcdef xyzkkk").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore(
+ "Calcite codegen does not support UDF with ... args."
+ + " See:https://jira.apache.org/jira/browse/CALCITE-2889")
+ public void testConcatWithSixParameters() {
+ String sql = "SELECT concat('abc', 'def', ' ', 'xyz', 'kkk', 'ttt')";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues("abcdef xyzkkkttt").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testConcatWithNull1() {
+ String sql = "SELECT CONCAT(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createStringValue(""),
+ "p1",
+ Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((String) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testConcatWithNull2() {
+ String sql = "SELECT CONCAT(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0",
+ Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p1",
+ Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((String) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testConcatParameterQuery() {
+ String sql = "SELECT CONCAT(@p0, @p1) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createStringValue(""), "p1", Value.createStringValue("A"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("A").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testReplace1() {
+ String sql = "SELECT REPLACE(@p0, @p1, @p2) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue(""),
+ "p1", Value.createStringValue(""),
+ "p2", Value.createStringValue("a"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testReplace2() {
+ String sql = "SELECT REPLACE(@p0, @p1, @p2) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("abc"),
+ "p1", Value.createStringValue(""),
+ "p2", Value.createStringValue("xyz"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("abc").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testReplace3() {
+ String sql = "SELECT REPLACE(@p0, @p1, @p2) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue(""),
+ "p1", Value.createStringValue(""),
+ "p2", Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((String) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testReplace4() {
+ String sql = "SELECT REPLACE(@p0, @p1, @p2) AS ColA";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p1", Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p2", Value.createStringValue(""));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((String) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testTrim1() {
+ String sql = "SELECT trim(@p0)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createStringValue(" a b c "));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("a b c").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testTrim2() {
+ String sql = "SELECT trim(@p0, @p1)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("abxyzab"), "p1", Value.createStringValue("ab"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("xyz").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testTrim3() {
+ String sql = "SELECT trim(@p0, @p1)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p1", Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((String) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testLTrim1() {
+ String sql = "SELECT ltrim(@p0)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createStringValue(" a b c "));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("a b c ").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testLTrim2() {
+ String sql = "SELECT ltrim(@p0, @p1)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("abxyzab"), "p1", Value.createStringValue("ab"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("xyzab").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testLTrim3() {
+ String sql = "SELECT ltrim(@p0, @p1)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p1", Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((String) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("types.size() != expressions.size()")
+ public void testRTrim1() {
+ String sql = "SELECT rtrim(@p0)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createStringValue(" a b c "));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(" a b c").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testRTrim2() {
+ String sql = "SELECT rtrim(@p0, @p1)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("abxyzab"), "p1", Value.createStringValue("ab"));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("abxyz").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testRTrim3() {
+ String sql = "SELECT rtrim(@p0, @p1)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createSimpleNullValue(TypeKind.TYPE_STRING),
+ "p1", Value.createSimpleNullValue(TypeKind.TYPE_STRING));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addNullableField("field1", FieldType.STRING).build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues((String) null).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("")
+ public void testCastBytesToString1() {
+ String sql = "SELECT CAST(@p0 AS STRING)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of("p0", Value.createBytesValue(ByteString.copyFromUtf8("`")));
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("`").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCastBytesToString2() {
+ String sql = "SELECT CAST(b'b' AS STRING)";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("b").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("")
+ public void testCastBytesToStringFromTable() {
+ String sql = "SELECT CAST(bytes_col AS STRING) FROM table_all_types";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("1").build(),
+ Row.withSchema(schema).addValues("2").build(),
+ Row.withSchema(schema).addValues("3").build(),
+ Row.withSchema(schema).addValues("4").build(),
+ Row.withSchema(schema).addValues("5").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCastStringToTS() {
+ String sql = "SELECT CAST('2019-01-15 13:21:03' AS TIMESTAMP)";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addDateTimeField("field_1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(parseTimestampWithUTCTimeZone("2019-01-15 13:21:03"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCastStringToString() {
+ String sql = "SELECT CAST(@p0 AS STRING)";
+ ImmutableMap<String, Value> params = ImmutableMap.of("p0", Value.createStringValue(""));
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testCastStringToInt64() {
+ String sql = "SELECT CAST(@p0 AS INT64)";
+ ImmutableMap<String, Value> params = ImmutableMap.of("p0", Value.createStringValue("123"));
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues(123L).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectConstant() {
+ String sql = "SELECT 'hi'";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("hi").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Does not support DATE_ADD.")
+ public void testDateAddWithParameter() {
+ String sql =
+ "SELECT "
+ + "DATE_ADD(@p0, INTERVAL @p1 DAY), "
+ + "DATE_ADD(@p2, INTERVAL @p3 DAY), "
+ + "DATE_ADD(@p4, INTERVAL @p5 YEAR), "
+ + "DATE_ADD(@p6, INTERVAL @p7 DAY), "
+ + "DATE_ADD(@p8, INTERVAL @p9 MONTH)";
+ // Value
+ ImmutableMap<String, Value> params =
+ ImmutableMap.<String, Value>builder()
+ .put("p0", Value.createDateValue(0)) // 1970-01-01
+ .put("p1", Value.createInt64Value(2L))
+ .put("p2", parseDateToValue("2019-01-01"))
+ .put("p3", Value.createInt64Value(2L))
+ .put("p4", Value.createSimpleNullValue(TypeKind.TYPE_DATE))
+ .put("p5", Value.createInt64Value(1L))
+ .put("p6", parseDateToValue("2000-02-29"))
+ .put("p7", Value.createInt64Value(-365L))
+ .put("p8", parseDateToValue("1999-03-31"))
+ .put("p9", Value.createInt64Value(-1L))
+ .build();
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema =
+ Schema.builder()
+ .addDateTimeField("field1")
+ .addDateTimeField("field2")
+ .addNullableField("field3", DATETIME)
+ .addDateTimeField("field4")
+ .addDateTimeField("field5")
+ .build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(
+ parseDate("1970-01-03"),
+ parseDate("2019-01-03"),
+ null,
+ parseDate("1999-03-01"),
+ parseDate("1999-02-28"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Does not support TIME_ADD.")
+ public void testTimeAddWithParameter() {
+ String sql = "SELECT TIME_ADD(@p0, INTERVAL @p1 SECOND)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", parseTimeToValue("12:13:14.123"),
+ "p1", Value.createInt64Value(1L));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addDateTimeField("field1").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(Row.withSchema(schema).addValues(parseTime("12:13:15.123")).build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("")
+ public void testTimestampAddWithParameter() {
+ String sql = "SELECT TIMESTAMP_ADD(@p0, INTERVAL @p1 MILLISECOND)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", parseTimestampWithTZToValue("2001-01-01 00:00:00+00"),
+ "p1", Value.createInt64Value(1L));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addDateTimeField("field1").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(parseTimestampWithTimeZone("2001-01-01 00:00:00.001+00"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testTimeStampAddWithParameter() {
+ String sql = "SELECT TIMESTAMP_ADD(@p0, INTERVAL @p1 MINUTE)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", parseTimestampWithTZToValue("2008-12-25 15:30:00+07:30"),
+ "p1", Value.createInt64Value(10L));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addDateTimeField("field1").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema)
+ .addValues(parseTimestampWithTimeZone("2008-12-25 15:40:00+07:30"))
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectFromTableWithMap() {
+ String sql = "SELECT row_field FROM table_with_map";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ Schema rowSchema = Schema.builder().addInt64Field("row_id").addStringField("data").build();
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(Schema.builder().addRowField("row_field", rowSchema).build())
+ .addValues(Row.withSchema(rowSchema).addValues(1L, "data1").build())
+ .build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSubQuery() {
+ String sql = "select sum(Key) from KeyValue\n" + "group by (select Key)";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("Does not support sub-queries");
+ zetaSQLQueryPlanner.convertToBeamRel(sql);
+ }
+
+ @Test
+ public void testSubstr() {
+ String sql = "SELECT substr(@p0, @p1, @p2)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("abc"),
+ "p1", Value.createInt64Value(-2L),
+ "p2", Value.createInt64Value(1L));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field1").build();
+ PAssert.that(stream).containsInAnyOrder(Row.withSchema(schema).addValues("b").build());
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSubstrWithLargeValueExpectException() {
+ String sql = "SELECT substr(@p0, @p1, @p2)";
+ ImmutableMap<String, Value> params =
+ ImmutableMap.of(
+ "p0", Value.createStringValue("abc"),
+ "p1", Value.createInt64Value(Integer.MAX_VALUE + 1L),
+ "p2", Value.createInt64Value(Integer.MIN_VALUE - 1L));
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql, params);
+ BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+ thrown.expect(RuntimeException.class);
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectAll() {
+ String sql = "SELECT ALL Key, Value FROM KeyValue;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").addStringField("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(14L, "KeyValue234").build(),
+ Row.withSchema(schema).addValues(15L, "KeyValue235").build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectDistinct() {
+ String sql = "SELECT DISTINCT Key FROM aggregate_test_table;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addInt64Field("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues(1L).build(),
+ Row.withSchema(schema).addValues(2L).build(),
+ Row.withSchema(schema).addValues(3L).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Bytes cannot be in UNION ALL")
+ public void testSelectDistinct2() {
+ String sql =
+ "SELECT DISTINCT val.BYTES\n"
+ + "from (select b\"BYTES\" BYTES union all\n"
+ + " select b\"bytes\" union all\n"
+ + " select b\"ByTeS\") val";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addByteArrayField("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("BYTES".getBytes(StandardCharsets.UTF_8)).build(),
+ Row.withSchema(schema).addValues("ByTeS".getBytes(StandardCharsets.UTF_8)).build(),
+ Row.withSchema(schema).addValues("bytes".getBytes(StandardCharsets.UTF_8)).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ @Ignore("Need to improve codegen's bytes support")
+ public void testSelectBytes() {
+ String sql = "SELECT b\"ByTes\"";
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addByteArrayField("field1").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("ByTes".getBytes(StandardCharsets.UTF_8)).build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectExcept() {
+ String sql = "SELECT * EXCEPT (Key, ts) FROM KeyValue;";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
+ final Schema schema = Schema.builder().addStringField("field2").build();
+
+ PAssert.that(stream)
+ .containsInAnyOrder(
+ Row.withSchema(schema).addValues("KeyValue234").build(),
+ Row.withSchema(schema).addValues("KeyValue235").build());
+
+ pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES));
+ }
+
+ @Test
+ public void testSelectReplace() {
+ String sql =
+ "WITH orders AS\n"
+ + " (SELECT 5 as order_id,\n"
+ + " \"sprocket\" as item_name,\n"
+ + " 200 as quantity)\n"
+ + "SELECT * REPLACE (\"widget\" AS item_name)\n"
+ + "FROM orders";
+
+ ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config);
+ BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql);
+ PCollection<Row> stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode);
+
... 308 lines suppressed ...