You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by so...@apache.org on 2018/11/02 21:20:02 UTC
[drill] 04/04: DRILL-6768: Improve to_date,
to_time and to_timestamp and corresponding cast functions to handle
empty string when option is enabled closes #1494
This is an automated email from the ASF dual-hosted git repository.
sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 56c8f0a7f97c3796ba90976fdbb6d90c5f2229a5
Author: Bohdan Kazydub <bo...@gmail.com>
AuthorDate: Wed Oct 3 20:07:46 2018 +0300
DRILL-6768: Improve to_date, to_time and to_timestamp and corresponding cast functions to handle empty string when option is enabled
closes #1494
---
exec/java-exec/src/main/codegen/data/Casts.tdd | 36 +++-
.../src/main/codegen/data/DateIntervalFunc.tdd | 14 ++
...VarCharDate.java => CastStringTypesToDate.java} | 51 +++--
.../templates/CastStringTypesToInterval.java | 111 ++++++++++
.../codegen/templates/CastVarCharInterval.java | 90 --------
.../SqlToDateTypeFunctions.java | 44 ++--
.../ToDateTypeFunctions.java | 75 ++++---
.../templates/Decimal/CastVarCharDecimal.java | 8 +-
.../java/org/apache/drill/exec/ExecConstants.java | 4 +-
.../exec/expr/ExpressionTreeMaterializer.java | 6 +-
.../expr/fn/FunctionImplementationRegistry.java | 12 +-
.../drill/exec/expr/fn/impl/MathFunctions.java | 94 ++++++++-
.../drill/exec/expr/stat/RangeExprEvaluator.java | 4 +-
.../physical/impl/project/ProjectRecordBatch.java | 4 +-
.../exec/planner/logical/DrillConstExecutor.java | 5 +-
.../exec/server/options/SystemOptionManager.java | 2 +-
.../exec/store/parquet/ParquetFilterBuilder.java | 4 +-
.../drill/exec/fn/impl/TestCastEmptyStrings.java | 151 ++++++++-----
.../exec/fn/impl/testing/TestDateConversions.java | 95 +++++++++
.../java/org/apache/drill/test/ConfigBuilder.java | 2 +-
.../java/org/apache/drill/test/TestBuilder.java | 19 ++
.../src/test/resources/dateWithEmptyStrings.json | 3 +
.../drill/common/expression/fn/CastFunctions.java | 227 --------------------
.../expression/fn/FunctionReplacementUtils.java | 233 +++++++++++++++++++++
24 files changed, 838 insertions(+), 456 deletions(-)
diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd
index e43572a..31eef19 100644
--- a/exec/java-exec/src/main/codegen/data/Casts.tdd
+++ b/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -61,10 +61,30 @@
{from: "VarChar", to: "TimeStamp", major: "VarCharDate", alias: "timestamptype"},
{from: "VarChar", to: "Time", major: "VarCharDate", alias: "timetype"},
+ {from: "VarChar", to: "NullableDate", major: "NullableVarCharDate"},
+ {from: "VarChar", to: "NullableTimeStamp", major: "NullableVarCharDate"},
+ {from: "VarChar", to: "NullableTime", major: "NullableVarCharDate"},
+
+ {from: "NullableVarChar", to: "NullableDate", major: "NullableVarCharDate"},
+ {from: "NullableVarChar", to: "NullableTimeStamp", major: "NullableVarCharDate"},
+ {from: "NullableVarChar", to: "NullableTime", major: "NullableVarCharDate"},
+
{from: "VarBinary", to: "Date", major: "VarBinaryDate", alias: "datetype"},
{from: "VarBinary", to: "TimeStamp", major: "VarBinaryDate", alias: "timestamptype"},
{from: "VarBinary", to: "Time", major: "VarBinaryDate", alias: "timetype"},
+ {from: "VarBinary", to: "NullableDate", major: "NullableVarCharDate"},
+ {from: "VarBinary", to: "NullableTimeStamp", major: "NullableVarCharDate"},
+ {from: "VarBinary", to: "NullableTime", major: "NullableVarCharDate"},
+
+ {from: "NullableVarBinary", to: "NullableDate", major: "NullableVarCharDate"},
+ {from: "NullableVarBinary", to: "NullableTimeStamp", major: "NullableVarCharDate"},
+ {from: "NullableVarBinary", to: "NullableTime", major: "NullableVarCharDate"},
+
+ {from: "NullableVar16Char", to: "NullableDate", major: "NullableVarCharDate"},
+ {from: "NullableVar16Char", to: "NullableTimeStamp", major: "NullableVarCharDate"},
+ {from: "NullableVar16Char", to: "NullableTime", major: "NullableVarCharDate"},
+
{from: "Date", to: "VarChar", major: "DateVarChar", bufferLength: "10"}
{from: "TimeStamp", to: "VarChar", major: "DateVarChar", bufferLength: "23"},
{from: "Time", to: "VarChar", major: "DateVarChar", bufferLength: "12"},
@@ -73,6 +93,14 @@
{from: "VarChar", to: "IntervalDay", major: "VarCharInterval"},
{from: "VarChar", to: "IntervalYear", major: "VarCharInterval"},
+ {from: "VarChar", to: "NullableInterval", major: "NullableVarCharInterval"},
+ {from: "VarChar", to: "NullableIntervalDay", major: "NullableVarCharInterval"},
+ {from: "VarChar", to: "NullableIntervalYear", major: "NullableVarCharInterval"},
+
+ {from: "NullableVarChar", to: "NullableInterval", major: "NullableVarCharInterval"},
+ {from: "NullableVarChar", to: "NullableIntervalDay", major: "NullableVarCharInterval"},
+ {from: "NullableVarChar", to: "NullableIntervalYear", major: "NullableVarCharInterval"},
+
{from: "Interval", to: "VarChar", major: "IntervalVarChar", bufferLength: "65"},
{from: "IntervalYear", to: "VarChar", major: "IntervalYearVarChar", bufferLength: "35"},
{from: "IntervalDay", to: "VarChar", major: "IntervalDayVarChar", bufferLength: "43"},
@@ -118,27 +146,27 @@
{from: "VarChar", to: "NullableFloat4", major: "EmptyString", javaType:"Float", parse:"Float"},
{from: "VarChar", to: "NullableFloat8", major: "EmptyString", javaType:"Double", parse:"Double"},
- {from: "VarChar", to: "NullableVarDecimal", major: "EmptyStringVarCharDecimalComplex"},
+ {from: "VarChar", to: "NullableVarDecimal", major: "NullableVarCharDecimalComplex"},
{from: "NullableVarChar", to: "NullableInt", major: "EmptyString", javaType:"Integer", primeType:"int"},
{from: "NullableVarChar", to: "NullableBigInt", major: "EmptyString", javaType: "Long", primeType: "long"},
{from: "NullableVarChar", to: "NullableFloat4", major: "EmptyString", javaType:"Float", parse:"Float"},
{from: "NullableVarChar", to: "NullableFloat8", major: "EmptyString", javaType:"Double", parse:"Double"},
- {from: "NullableVarChar", to: "NullableVarDecimal", major: "EmptyStringVarCharDecimalComplex"},
+ {from: "NullableVarChar", to: "NullableVarDecimal", major: "NullableVarCharDecimalComplex"},
{from: "NullableVar16Char", to: "NullableInt", major: "EmptyString", javaType:"Integer", primeType:"int"},
{from: "NullableVar16Char", to: "NullableBigInt", major: "EmptyString", javaType: "Long", primeType: "long"},
{from: "NullableVar16Char", to: "NullableFloat4", major: "EmptyString", javaType:"Float", parse:"Float"},
{from: "NullableVar16Char", to: "NullableFloat8", major: "EmptyString", javaType:"Double", parse:"Double"},
- {from: "NullableVar16Char", to: "NullableVarDecimal", major: "EmptyStringVarCharDecimalComplex"},
+ {from: "NullableVar16Char", to: "NullableVarDecimal", major: "NullableVarCharDecimalComplex"},
{from: "NullableVarBinary", to: "NullableInt", major: "EmptyString", javaType:"Integer", primeType:"int"},
{from: "NullableVarBinary", to: "NullableBigInt", major: "EmptyString", javaType: "Long", primeType: "long"},
{from: "NullableVarBinary", to: "NullableFloat4", major: "EmptyString", javaType:"Float", parse:"Float"},
{from: "NullableVarBinary", to: "NullableFloat8", major: "EmptyString", javaType:"Double", parse:"Double"},
- {from: "NullableVarBinary", to: "NullableVarDecimal", major: "EmptyStringVarCharDecimalComplex"},
+ {from: "NullableVarBinary", to: "NullableVarDecimal", major: "NullableVarCharDecimalComplex"},
]
}
diff --git a/exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd b/exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd
index 9f1ae37..96e1607 100644
--- a/exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd
+++ b/exec/java-exec/src/main/codegen/data/DateIntervalFunc.tdd
@@ -18,4 +18,18 @@
{intervals: ["Interval", "IntervalDay", "IntervalYear", "Int", "BigInt"] },
{truncInputTypes: ["Date", "TimeStamp", "Time", "Interval", "IntervalDay", "IntervalYear"] },
{truncUnits : ["Second", "Minute", "Hour", "Day", "Month", "Year", "Week", "Quarter", "Decade", "Century", "Millennium" ] },
+
+ {
+ varCharToDate: [
+ {from: "VarChar", to: "Date"},
+ {from: "VarChar", to: "NullableDate"},
+ {from: "NullableVarChar", to: "NullableDate"},
+ {from: "VarChar", to: "Time"},
+ {from: "VarChar", to: "NullableTime"},
+ {from: "NullableVarChar", to: "NullableTime"},
+ {from: "VarChar", to: "TimeStamp"},
+ {from: "VarChar", to: "NullableTimeStamp"},
+ {from: "NullableVarChar", to: "NullableTimeStamp"}
+ ]
+ }
}
diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java b/exec/java-exec/src/main/codegen/templates/CastStringTypesToDate.java
similarity index 53%
rename from exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
rename to exec/java-exec/src/main/codegen/templates/CastStringTypesToDate.java
index 8d1ab7d..c26f6e2 100644
--- a/exec/java-exec/src/main/codegen/templates/CastVarCharDate.java
+++ b/exec/java-exec/src/main/codegen/templates/CastStringTypesToDate.java
@@ -18,9 +18,13 @@
<@pp.dropOutputFile />
<#list cast.types as type>
-<#if type.major == "VarCharDate" || type.major == "VarBinaryDate"> <#-- Template to convert from VarChar/ VarBinary to Date, Time, TimeStamp -->
+<#if type.major == "VarCharDate" || type.major == "VarBinaryDate" || type.major == "NullableVarCharDate"> <#-- Template to convert from VarChar/ VarBinary to Date, Time, TimeStamp -->
+<#if type.major == "VarCharDate" || type.major == "VarBinaryDate">
<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}To${type.to}.java" />
+<#elseif type.major == "NullableVarCharDate">
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/GCast${type.from}To${type.to}.java" />
+</#if>
<#include "/@includes/license.ftl" />
@@ -46,9 +50,17 @@ import io.netty.buffer.DrillBuf;
* This class is generated using freemarker and the ${.template_name} template.
*/
@SuppressWarnings("unused")
+<#if type.major == "VarCharDate" || type.major == "VarBinaryDate">
@FunctionTemplate(names = {"cast${type.to?upper_case}", "${type.alias}"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL,
costCategory = FunctionCostCategory.COMPLEX)
public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
+<#elseif type.major == "NullableVarCharDate">
+@FunctionTemplate(name = "castEmptyString${type.from}To${type.to?upper_case}",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = NullHandling.INTERNAL,
+ isInternal = true)
+public class GCast${type.from}To${type.to} implements DrillSimpleFunc {
+</#if>
@Param ${type.from}Holder in;
@Output ${type.to}Holder out;
@@ -56,24 +68,31 @@ public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
public void setup() { }
public void eval() {
+ <#if type.major == "NullableVarCharDate">
+ if(<#if type.from == "NullableVarChar" || type.from == "NullableVar16Char" || type.from == "NullableVarBinary">in.isSet == 0 || </#if>in.end == in.start) {
+ out.isSet = 0;
+ return;
+ }
+ out.isSet = 1;
+ </#if>
- <#if type.to != "Date">
- byte[] buf = new byte[in.end - in.start];
- in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- String input = new String(buf, com.google.common.base.Charsets.UTF_8);
- </#if>
-
- <#if type.to == "Date">
- out.value = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getDate(in.buffer, in.start, in.end);
+ <#if type.to != "Date" && type.to != "NullableDate">
+ byte[] buf = new byte[in.end - in.start];
+ in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
+ String input = new String(buf, com.google.common.base.Charsets.UTF_8);
+ </#if>
- <#elseif type.to == "TimeStamp">
- java.time.LocalDateTime parsedDateTime = org.apache.drill.exec.expr.fn.impl.DateUtility.parseBest(input);
- out.value = parsedDateTime.toInstant(java.time.ZoneOffset.UTC).toEpochMilli();
+ <#if type.to == "Date" || type.to == "NullableDate">
+ out.value = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getDate(in.buffer, in.start, in.end);
- <#elseif type.to == "Time">
- java.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getTimeFormatter();
- out.value = (int) (java.time.LocalTime.parse(input, f).atDate(java.time.LocalDate.ofEpochDay(0)).toInstant(java.time.ZoneOffset.UTC).toEpochMilli());
- </#if>
+ <#elseif type.to == "TimeStamp" || type.to == "NullableTimeStamp">
+ java.time.LocalDateTime parsedDateTime = org.apache.drill.exec.expr.fn.impl.DateUtility.parseBest(input);
+ out.value = parsedDateTime.toInstant(java.time.ZoneOffset.UTC).toEpochMilli();
+
+ <#elseif type.to == "Time" || type.to == "NullableTime">
+ java.time.format.DateTimeFormatter f = org.apache.drill.exec.expr.fn.impl.DateUtility.getTimeFormatter();
+ out.value = (int) (java.time.LocalTime.parse(input, f).atDate(java.time.LocalDate.ofEpochDay(0)).toInstant(java.time.ZoneOffset.UTC).toEpochMilli());
+ </#if>
}
}
</#if> <#-- type.major -->
diff --git a/exec/java-exec/src/main/codegen/templates/CastStringTypesToInterval.java b/exec/java-exec/src/main/codegen/templates/CastStringTypesToInterval.java
new file mode 100644
index 0000000..196e860
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/CastStringTypesToInterval.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+<#list cast.types as type>
+<#if type.major == "VarCharInterval" || type.major == "NullableVarCharInterval"> <#-- Template to convert from VarChar to Interval, IntervalYear, IntervalDay -->
+
+<#if type.major == "VarCharInterval">
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}To${type.to}.java" />
+<#elseif type.major == "NullableVarCharInterval">
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/CastEmptyString${type.from}To${type.to}.java" />
+</#if>
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl.gcast;
+
+import io.netty.buffer.ByteBuf;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+import org.joda.time.MutableDateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.DateMidnight;
+import javax.inject.Inject;
+import io.netty.buffer.DrillBuf;
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+<#if type.major == "VarCharInterval">
+@FunctionTemplate(name = "cast${type.to?upper_case}",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = NullHandling.NULL_IF_NULL)
+public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
+<#elseif type.major == "NullableVarCharInterval">
+@FunctionTemplate(name = "castEmptyString${type.from}To${type.to?upper_case}",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = NullHandling.INTERNAL,
+ isInternal = true)
+public class CastEmptyString${type.from}To${type.to} implements DrillSimpleFunc {
+</#if>
+
+ @Param ${type.from}Holder in;
+ @Output ${type.to}Holder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ <#if type.major == "NullableVarCharInterval">
+ if (<#if type.from == "NullableVarChar" || type.from == "NullableVar16Char" || type.from == "NullableVarBinary">in.isSet == 0 || </#if>in.end == in.start) {
+ out.isSet = 0;
+ return;
+ }
+ out.isSet = 1;
+ </#if>
+
+ byte[] buf = new byte[in.end - in.start];
+ in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
+ String input = new String(buf, com.google.common.base.Charsets.UTF_8);
+
+ // Parse the ISO format
+ org.joda.time.Period period = org.joda.time.Period.parse(input);
+
+ <#if type.to == "Interval" || type.to == "NullableInterval">
+ out.months = (period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths) + period.getMonths();
+
+ out.days = period.getDays();
+
+ out.milliseconds = (period.getHours() * org.apache.drill.exec.vector.DateUtilities.hoursToMillis) +
+ (period.getMinutes() * org.apache.drill.exec.vector.DateUtilities.minutesToMillis) +
+ (period.getSeconds() * org.apache.drill.exec.vector.DateUtilities.secondsToMillis) +
+ (period.getMillis());
+
+ <#elseif type.to == "IntervalDay" || type.to == "NullableIntervalDay">
+ out.days = period.getDays();
+
+ out.milliseconds = (period.getHours() * org.apache.drill.exec.vector.DateUtilities.hoursToMillis) +
+ (period.getMinutes() * org.apache.drill.exec.vector.DateUtilities.minutesToMillis) +
+ (period.getSeconds() * org.apache.drill.exec.vector.DateUtilities.secondsToMillis) +
+ (period.getMillis());
+ <#elseif type.to == "IntervalYear" || type.to == "NullableIntervalYear">
+ out.value = (period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths) + period.getMonths();
+ </#if>
+ }
+}
+</#if> <#-- type.major -->
+</#list>
diff --git a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java b/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
deleted file mode 100644
index f9396a6..0000000
--- a/exec/java-exec/src/main/codegen/templates/CastVarCharInterval.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-<@pp.dropOutputFile />
-
-<#list cast.types as type>
-<#if type.major == "VarCharInterval"> <#-- Template to convert from VarChar to Interval, IntervalYear, IntervalDay -->
-
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}To${type.to}.java" />
-
-<#include "/@includes/license.ftl" />
-
-package org.apache.drill.exec.expr.fn.impl.gcast;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
-import org.apache.drill.exec.expr.annotations.Output;
-import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.annotations.Workspace;
-import org.apache.drill.exec.expr.holders.*;
-import org.apache.drill.exec.record.RecordBatch;
-import org.joda.time.MutableDateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.DateMidnight;
-import javax.inject.Inject;
-import io.netty.buffer.DrillBuf;
-
-/*
- * This class is generated using freemarker and the ${.template_name} template.
- */
-@SuppressWarnings("unused")
-@FunctionTemplate(name = "cast${type.to?upper_case}", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
-public class Cast${type.from}To${type.to} implements DrillSimpleFunc {
-
- @Param ${type.from}Holder in;
- @Output ${type.to}Holder out;
-
- public void setup() {
- }
-
- public void eval() {
-
- byte[] buf = new byte[in.end - in.start];
- in.buffer.getBytes(in.start, buf, 0, in.end - in.start);
- String input = new String(buf, com.google.common.base.Charsets.UTF_8);
-
- // Parse the ISO format
- org.joda.time.Period period = org.joda.time.Period.parse(input);
-
- <#if type.to == "Interval">
- out.months = (period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths) + period.getMonths();
-
- out.days = period.getDays();
-
- out.milliseconds = (period.getHours() * org.apache.drill.exec.vector.DateUtilities.hoursToMillis) +
- (period.getMinutes() * org.apache.drill.exec.vector.DateUtilities.minutesToMillis) +
- (period.getSeconds() * org.apache.drill.exec.vector.DateUtilities.secondsToMillis) +
- (period.getMillis());
-
- <#elseif type.to == "IntervalDay">
- out.days = period.getDays();
-
- out.milliseconds = (period.getHours() * org.apache.drill.exec.vector.DateUtilities.hoursToMillis) +
- (period.getMinutes() * org.apache.drill.exec.vector.DateUtilities.minutesToMillis) +
- (period.getSeconds() * org.apache.drill.exec.vector.DateUtilities.secondsToMillis) +
- (period.getMillis());
- <#elseif type.to == "IntervalYear">
- out.value = (period.getYears() * org.apache.drill.exec.vector.DateUtilities.yearsToMonths) + period.getMonths();
- </#if>
- }
-}
-</#if> <#-- type.major -->
-</#list>
\ No newline at end of file
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/SqlToDateTypeFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/SqlToDateTypeFunctions.java
index 32d824a..19c9351 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/SqlToDateTypeFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/SqlToDateTypeFunctions.java
@@ -19,9 +19,9 @@ import org.apache.drill.exec.expr.annotations.Workspace;
<@pp.dropOutputFile/>
-<#list dateIntervalFunc.dates as type>
+<#list dateIntervalFunc.varCharToDate as convert>
-<@pp.changeOutputFile name = "/org/apache/drill/exec/expr/fn/impl/SqlTo${type}.java"/>
+<@pp.changeOutputFile name = "/org/apache/drill/exec/expr/fn/impl/G${convert.from}SqlTo${convert.to}.java"/>
<#include "/@includes/license.ftl"/>
@@ -38,16 +38,22 @@ import org.apache.drill.exec.expr.holders.*;
/*
* This class is generated using freemarker and the ${.template_name} template.
*/
+<#if convert.to?contains("Nullable")>
+@FunctionTemplate(name = "convert${convert.from}SqlTo${convert.to}",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = NullHandling.INTERNAL,
+ isInternal = true)
+<#else>
+@FunctionTemplate(name = "sql_to_${convert.to?lower_case}",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = NullHandling.NULL_IF_NULL)
+</#if>
+public class G${convert.from}SqlTo${convert.to} implements DrillSimpleFunc {
-@FunctionTemplate(name = "sql_to_${type?lower_case}",
- scope = FunctionTemplate.FunctionScope.SIMPLE,
- nulls = NullHandling.NULL_IF_NULL)
-public class SqlTo${type} implements DrillSimpleFunc {
-
- @Param VarCharHolder left;
+ @Param ${convert.from}Holder left;
@Param VarCharHolder right;
@Workspace org.joda.time.format.DateTimeFormatter format;
- @Output ${type}Holder out;
+ @Output ${convert.to}Holder out;
public void setup() {
// Get the desired output format
@@ -57,27 +63,35 @@ public class SqlTo${type} implements DrillSimpleFunc {
format = org.joda.time.format.DateTimeFormat.forPattern(pattern);
} catch (IllegalArgumentException e) {
throw org.apache.drill.common.exceptions.UserException.functionError(e)
- .message("Error parsing formatter %s in %s function", formatString, "sql_to_${type?lower_case}")
+ .message("Error parsing formatter %s in %s function", formatString, "sql_to_${convert.to?lower_case}")
.build();
}
}
public void eval() {
+ <#if convert.to?contains("Nullable")>
+ if (<#if convert.from == "NullableVarChar">left.isSet == 0 || </#if>left.start == left.end) {
+ out.isSet = 0;
+ return;
+ }
+ out.isSet = 1;
+
+ </#if>
// Get the input
String input = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getStringFromVarCharHolder(left);
try {
- <#if type == "Date">
+ <#if convert.to == "Date" || convert.to == "NullableDate">
out.value = org.joda.time.DateMidnight.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis();
- <#elseif type == "TimeStamp">
+ <#elseif convert.to == "TimeStamp" || convert.to == "NullableTimeStamp">
out.value = org.joda.time.DateTime.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis();
- <#elseif type == "Time">
+ <#elseif convert.to == "Time" || convert.to == "NullableTime">
out.value = (int) format.parseDateTime(input).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis();
</#if>
} catch (IllegalArgumentException e) {
throw org.apache.drill.common.exceptions.UserException.functionError(e)
- .message("Error parsing date-time %s in %s function", input, "sql_to_${type?lower_case}")
+ .message("Error parsing date-time %s in %s function", input, "sql_to_${convert.to?lower_case}")
.build();
}
}
}
-</#list>
\ No newline at end of file
+</#list>
diff --git a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java
index 7c410e3..6ef196f 100644
--- a/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/DateIntervalFunctionTemplates/ToDateTypeFunctions.java
@@ -19,9 +19,9 @@ import org.apache.drill.exec.expr.annotations.Workspace;
<@pp.dropOutputFile />
-<#list dateIntervalFunc.dates as type>
+<#list dateIntervalFunc.varCharToDate as convert>
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/GTo${type}.java" />
+<@pp.changeOutputFile name = "/org/apache/drill/exec/expr/fn/impl/G${convert.from}To${convert.to}.java" />
<#include "/@includes/license.ftl" />
@@ -34,43 +34,56 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.holders.*;
-import org.apache.drill.exec.record.RecordBatch;
/*
* This class is generated using freemarker and the ${.template_name} template.
*/
+<#if convert.to?contains("Nullable")>
+@FunctionTemplate(name = "convert${convert.from}To${convert.to}",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = NullHandling.INTERNAL,
+ isInternal = true)
+<#else>
+@FunctionTemplate(name = "to_${convert.to?lower_case}",
+ scope = FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = NullHandling.NULL_IF_NULL)
+</#if>
+public class G${convert.from}To${convert.to} implements DrillSimpleFunc {
-@FunctionTemplate(name = "to_${type?lower_case}" , scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
-public class GTo${type} implements DrillSimpleFunc {
+ @Param ${convert.from}Holder left;
+ @Param VarCharHolder right;
+ @Workspace org.joda.time.format.DateTimeFormatter format;
+ @Output ${convert.to}Holder out;
+ public void setup() {
+ // Get the desired output format
+ byte[] buf = new byte[right.end - right.start];
+ right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
+ String formatString = new String(buf, com.google.common.base.Charsets.UTF_8);
+ format = org.joda.time.format.DateTimeFormat.forPattern(formatString);
+ }
- @Param VarCharHolder left;
- @Param VarCharHolder right;
- @Workspace org.joda.time.format.DateTimeFormatter format;
- @Output ${type}Holder out;
-
- public void setup() {
- // Get the desired output format
- byte[] buf = new byte[right.end - right.start];
- right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
- String formatString = new String(buf, com.google.common.base.Charsets.UTF_8);
- format = org.joda.time.format.DateTimeFormat.forPattern(formatString);
+ public void eval() {
+ <#if convert.to?contains("Nullable")>
+ if (<#if convert.from == "NullableVarChar">left.isSet == 0 || </#if>left.start == left.end) {
+ out.isSet = 0;
+ return;
}
+ out.isSet = 1;
- public void eval() {
-
- // Get the input
- byte[] buf1 = new byte[left.end - left.start];
- left.buffer.getBytes(left.start, buf1, 0, left.end - left.start);
- String input = new String(buf1, com.google.common.base.Charsets.UTF_8);
+ </#if>
+ // Get the input
+ byte[] buf1 = new byte[left.end - left.start];
+ left.buffer.getBytes(left.start, buf1, 0, left.end - left.start);
+ String input = new String(buf1, com.google.common.base.Charsets.UTF_8);
- <#if type == "Date">
- out.value = (org.joda.time.DateMidnight.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC)).getMillis();
- <#elseif type == "TimeStamp">
- out.value = org.joda.time.DateTime.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis();
- <#elseif type == "Time">
- out.value = (int) ((format.parseDateTime(input)).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis());
- </#if>
- }
+ <#if convert.to == "Date" || convert.to == "NullableDate">
+ out.value = (org.joda.time.DateMidnight.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC)).getMillis();
+ <#elseif convert.to == "TimeStamp" || convert.to == "NullableTimeStamp">
+ out.value = org.joda.time.DateTime.parse(input, format).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis();
+ <#elseif convert.to == "Time" || convert.to == "NullableTime">
+ out.value = (int) ((format.parseDateTime(input)).withZoneRetainFields(org.joda.time.DateTimeZone.UTC).getMillis());
+ </#if>
+ }
}
-</#list>
\ No newline at end of file
+</#list>
diff --git a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
index ac1fccb..3296a04 100644
--- a/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
+++ b/exec/java-exec/src/main/codegen/templates/Decimal/CastVarCharDecimal.java
@@ -19,11 +19,11 @@
<#list cast.types as type>
-<#if type.major == "VarCharDecimalComplex" || type.major == "EmptyStringVarCharDecimalComplex"> <#-- Cast function template for conversion from VarChar to VarDecimal -->
+<#if type.major == "VarCharDecimalComplex" || type.major == "NullableVarCharDecimalComplex"> <#-- Cast function template for conversion from VarChar to VarDecimal -->
<#if type.major == "VarCharDecimalComplex">
<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}${type.to}.java"/>
-<#elseif type.major == "EmptyStringVarCharDecimalComplex">
+<#elseif type.major == "NullableVarCharDecimalComplex">
<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/CastEmptyString${type.from}To${type.to}.java"/>
</#if>
@@ -58,7 +58,7 @@ import java.nio.ByteBuffer;
returnType = FunctionTemplate.ReturnType.DECIMAL_CAST,
nulls = NullHandling.NULL_IF_NULL)
public class Cast${type.from}${type.to} implements DrillSimpleFunc {
-<#elseif type.major == "EmptyStringVarCharDecimalComplex">
+<#elseif type.major == "NullableVarCharDecimalComplex">
@FunctionTemplate(name = "castEmptyString${type.from}To${type.to?upper_case}",
scope = FunctionTemplate.FunctionScope.SIMPLE,
returnType = FunctionTemplate.ReturnType.DECIMAL_CAST,
@@ -76,7 +76,7 @@ public class CastEmptyString${type.from}To${type.to} implements DrillSimpleFunc
}
public void eval() {
- <#if type.major == "EmptyStringVarCharDecimalComplex">
+ <#if type.major == "NullableVarCharDecimalComplex">
// Check if the input is null or empty string
if (<#if type.from == "NullableVarChar"> in.isSet == 0 || </#if> in.end == in.start) {
out.isSet = 0;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index cb0fc5c..eed4ff8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -499,8 +499,8 @@ public final class ExecConstants {
public static final PositiveLongValidator SLICE_TARGET_OPTION = new PositiveLongValidator(SLICE_TARGET, Long.MAX_VALUE,
new OptionDescription("The number of records manipulated within a fragment before Drill parallelizes operations."));
- public static final String CAST_TO_NULLABLE_NUMERIC = "drill.exec.functions.cast_empty_string_to_null";
- public static final BooleanValidator CAST_TO_NULLABLE_NUMERIC_OPTION = new BooleanValidator(CAST_TO_NULLABLE_NUMERIC,
+ public static final String CAST_EMPTY_STRING_TO_NULL = "drill.exec.functions.cast_empty_string_to_null";
+ public static final BooleanValidator CAST_EMPTY_STRING_TO_NULL_OPTION = new BooleanValidator(CAST_EMPTY_STRING_TO_NULL,
new OptionDescription("In a text file, treat empty fields as NULL values instead of empty string."));
/**
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 2a94d9b..4e35d6d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -60,7 +60,7 @@ import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
import org.apache.drill.common.expression.ValueExpressions.VarDecimalExpression;
-import org.apache.drill.common.expression.fn.CastFunctions;
+import org.apache.drill.common.expression.fn.FunctionReplacementUtils;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
import org.apache.drill.common.expression.visitors.ConditionalExprOptimizer;
import org.apache.drill.common.expression.visitors.ExpressionValidator;
@@ -198,7 +198,7 @@ public class ExpressionTreeMaterializer {
}
public static LogicalExpression addCastExpression(LogicalExpression fromExpr, MajorType toType, FunctionLookupContext functionLookupContext, ErrorCollector errorCollector, boolean exactResolver) {
- String castFuncName = CastFunctions.getCastFunc(toType.getMinorType());
+ String castFuncName = FunctionReplacementUtils.getCastFunc(toType.getMinorType());
List<LogicalExpression> castArgs = Lists.newArrayList();
castArgs.add(fromExpr); //input_expr
@@ -856,7 +856,7 @@ public class ExpressionTreeMaterializer {
MajorType type = e.getMajorType();
// Get the cast function name from the map
- String castFuncWithType = CastFunctions.getCastFunc(type.getMinorType());
+ String castFuncWithType = FunctionReplacementUtils.getCastFunc(type.getMinorType());
List<LogicalExpression> newArgs = Lists.newArrayList();
newArgs.add(input); //input_expr
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index 5621c44..137969a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -33,6 +33,7 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.drill.common.expression.fn.FunctionReplacementUtils;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
import org.apache.drill.shaded.guava.com.google.common.io.Files;
@@ -42,7 +43,6 @@ import org.apache.drill.common.config.CommonConstants;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.common.scanner.RunTimeScan;
import org.apache.drill.common.scanner.persistence.ScanResult;
@@ -204,16 +204,16 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
if (functionCall.args.size() == 0) {
return funcName;
}
- boolean castToNullableNumeric = optionManager != null &&
- optionManager.getOption(ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION);
- if (! castToNullableNumeric) {
+ boolean castEmptyStringToNull = optionManager != null &&
+ optionManager.getOption(ExecConstants.CAST_EMPTY_STRING_TO_NULL_OPTION);
+ if (!castEmptyStringToNull) {
return funcName;
}
MajorType majorType = functionCall.args.get(0).getMajorType();
DataMode dataMode = majorType.getMode();
MinorType minorType = majorType.getMinorType();
- if (CastFunctions.isReplacementNeeded(funcName, minorType)) {
- funcName = CastFunctions.getReplacingCastFunction(funcName, dataMode, minorType);
+ if (FunctionReplacementUtils.isReplacementNeeded(funcName, minorType)) {
+ funcName = FunctionReplacementUtils.getReplacingFunction(funcName, dataMode, minorType);
}
return funcName;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
index 59236ce..8470e1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
@@ -27,6 +27,8 @@ import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
import org.apache.drill.exec.expr.holders.VarCharHolder;
public class MathFunctions{
@@ -70,6 +72,7 @@ public class MathFunctions{
@Workspace int decimalDigits;
@Output Float8Holder out;
+ @Override
public void setup() {
byte[] buf = new byte[right.end - right.start];
right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
@@ -77,13 +80,14 @@ public class MathFunctions{
decimalDigits = inputFormat.getMaximumFractionDigits();
}
+ @Override
public void eval() {
byte[] buf1 = new byte[left.end - left.start];
left.buffer.getBytes(left.start, buf1, 0, left.end - left.start);
String input = new String(buf1);
try {
out.value = inputFormat.parse(input).doubleValue();
- } catch(java.text.ParseException e) {
+ } catch (java.text.ParseException e) {
throw new UnsupportedOperationException("Cannot parse input: " + input + " with pattern : " + inputFormat.toPattern());
}
@@ -93,6 +97,94 @@ public class MathFunctions{
}
}
+ @FunctionTemplate(name = "convertVarCharToNumber", scope = FunctionScope.SIMPLE, isInternal = true)
+ public static class ToNullableNumber implements DrillSimpleFunc {
+ @Param
+ VarCharHolder left;
+ @Param
+ VarCharHolder right;
+ @Workspace
+ java.text.DecimalFormat inputFormat;
+ @Workspace
+ int decimalDigits;
+ @Output
+ NullableFloat8Holder out;
+
+ @Override
+ public void setup() {
+ byte[] buf = new byte[right.end - right.start];
+ right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
+ inputFormat = new DecimalFormat(new String(buf));
+ decimalDigits = inputFormat.getMaximumFractionDigits();
+ }
+
+ @Override
+ public void eval() {
+ if (left.start == left.end) {
+ out.isSet = 0;
+ return;
+ }
+ out.isSet = 1;
+
+ byte[] buf1 = new byte[left.end - left.start];
+ left.buffer.getBytes(left.start, buf1, 0, left.end - left.start);
+ String input = new String(buf1);
+ try {
+ out.value = inputFormat.parse(input).doubleValue();
+ } catch (java.text.ParseException e) {
+ throw new UnsupportedOperationException("Cannot parse input: " + input + " with pattern : " + inputFormat.toPattern());
+ }
+
+ // Round the value
+ java.math.BigDecimal roundedValue = new java.math.BigDecimal(out.value);
+ out.value = (roundedValue.setScale(decimalDigits, java.math.BigDecimal.ROUND_HALF_UP)).doubleValue();
+ }
+ }
+
+ @FunctionTemplate(name = "convertNullableVarCharToNumber", scope = FunctionScope.SIMPLE, isInternal = true)
+ public static class ToNullableNumberNullableInput implements DrillSimpleFunc {
+ @Param
+ NullableVarCharHolder left;
+ @Param
+ VarCharHolder right;
+ @Workspace
+ java.text.DecimalFormat inputFormat;
+ @Workspace
+ int decimalDigits;
+ @Output
+ NullableFloat8Holder out;
+
+ @Override
+ public void setup() {
+ byte[] buf = new byte[right.end - right.start];
+ right.buffer.getBytes(right.start, buf, 0, right.end - right.start);
+ inputFormat = new DecimalFormat(new String(buf));
+ decimalDigits = inputFormat.getMaximumFractionDigits();
+ }
+
+ @Override
+ public void eval() {
+ if (left.isSet == 0 || left.start == left.end) {
+ out.isSet = 0;
+ return;
+ }
+ out.isSet = 1;
+
+ byte[] buf1 = new byte[left.end - left.start];
+ left.buffer.getBytes(left.start, buf1, 0, left.end - left.start);
+ String input = new String(buf1);
+ try {
+ out.value = inputFormat.parse(input).doubleValue();
+ } catch (java.text.ParseException e) {
+ throw new UnsupportedOperationException("Cannot parse input: " + input + " with pattern : " + inputFormat.toPattern());
+ }
+
+ // Round the value
+ java.math.BigDecimal roundedValue = new java.math.BigDecimal(out.value);
+ out.value = (roundedValue.setScale(decimalDigits, java.math.BigDecimal.ROUND_HALF_UP)).doubleValue();
+ }
+ }
+
@FunctionTemplate(name = "pi", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Pi implements DrillSimpleFunc {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
index 2b55e3d..48a7a90 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
@@ -23,7 +23,7 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.ValueExpressions;
-import org.apache.drill.common.expression.fn.CastFunctions;
+import org.apache.drill.common.expression.fn.FunctionReplacementUtils;
import org.apache.drill.common.expression.fn.FuncHolder;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
import org.apache.drill.common.types.TypeProtos;
@@ -145,7 +145,7 @@ public class RangeExprEvaluator<T extends Comparable<T>> extends AbstractExprVis
final String funcName = ((DrillSimpleFuncHolder) funcHolder).getRegisteredNames()[0];
- if (CastFunctions.isCastFunction(funcName)) {
+ if (FunctionReplacementUtils.isCastFunction(funcName)) {
Statistics stat = holderExpr.args.get(0).accept(this, null);
if (stat != null && ! stat.isEmpty()) {
return evalCastFunc(holderExpr, stat);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index a051d99..c9d97bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.physical.impl.project;
import com.carrotsearch.hppc.IntHashSet;
+import org.apache.drill.common.expression.fn.FunctionReplacementUtils;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
@@ -33,7 +34,6 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.PathSegment.NameSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions;
-import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.logical.data.NamedExpression;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
@@ -604,7 +604,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project> {
if (Types.isComplex(field.getType()) || Types.isRepeated(field.getType())) {
final LogicalExpression convertToJson = FunctionCallFactory.createConvert(ConvertExpression.CONVERT_TO, "JSON",
SchemaPath.getSimplePath(fieldName), ExpressionPosition.UNKNOWN);
- final String castFuncName = CastFunctions.getCastFunc(MinorType.VARCHAR);
+ final String castFuncName = FunctionReplacementUtils.getCastFunc(MinorType.VARCHAR);
final List<LogicalExpression> castArgs = Lists.newArrayList();
castArgs.add(convertToJson); //input_expr
// implicitly casting to varchar, since we don't know actual source length, cast to undefined length, which will preserve source length
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
index 8a74399..8991a7d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillConstExecutor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.planner.logical;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.drill.shaded.guava.com.google.common.base.Function;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import io.netty.buffer.DrillBuf;
@@ -159,7 +160,9 @@ public class DrillConstExecutor implements RexExecutor {
.message(message)
.build(logger);
}
- reducedValues.add(rexBuilder.makeNullLiteral(typeFactory.createSqlType(sqlTypeName)));
+
+ RelDataType type = TypeInferenceUtils.createCalciteTypeWithNullability(typeFactory, sqlTypeName, true);
+ reducedValues.add(rexBuilder.makeNullLiteral(type));
continue;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index a33d832..1d0bca0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -147,7 +147,7 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
new OptionDefinition(ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR), // for tuning
new OptionDefinition(ExecConstants.HASHAGG_USE_MEMORY_PREDICTION_VALIDATOR), // for testing
new OptionDefinition(ExecConstants.HASHAGG_FALLBACK_ENABLED_VALIDATOR), // for enable/disable unbounded HashAgg
- new OptionDefinition(ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION),
+ new OptionDefinition(ExecConstants.CAST_EMPTY_STRING_TO_NULL_OPTION),
new OptionDefinition(ExecConstants.OUTPUT_FORMAT_VALIDATOR),
new OptionDefinition(ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR),
new OptionDefinition(ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR),
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
index be93a56..ad38849 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
@@ -23,7 +23,7 @@ import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.ValueExpressions;
-import org.apache.drill.common.expression.fn.CastFunctions;
+import org.apache.drill.common.expression.fn.FunctionReplacementUtils;
import org.apache.drill.common.expression.fn.FuncHolder;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
import org.apache.drill.common.types.TypeProtos;
@@ -236,7 +236,7 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
return handleIsFunction(funcHolderExpr, value);
}
- if (CastFunctions.isCastFunction(funcName)) {
+ if (FunctionReplacementUtils.isCastFunction(funcName)) {
List<LogicalExpression> newArgs = generateNewExpressions(funcHolderExpr.args, value);
if (newArgs == null) {
return null;
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
index ab47b32..7f1e4b5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestCastEmptyStrings.java
@@ -17,79 +17,134 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.test.BaseTestQuery;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.categories.SqlFunctionTest;
import org.apache.drill.categories.UnlikelyTest;
import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.junit.AfterClass;
+import org.apache.drill.test.ClusterFixture;
+import org.apache.drill.test.ClusterFixtureBuilder;
+import org.apache.drill.test.ClusterTest;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+
@Category({UnlikelyTest.class, SqlFunctionTest.class})
-public class TestCastEmptyStrings extends BaseTestQuery {
- // enable decimal data type
+public class TestCastEmptyStrings extends ClusterTest {
+
@BeforeClass
- public static void enableDecimalDataType() throws Exception {
- test("alter session set `%s` = true", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY);
+ public static void setup() throws Exception {
+ ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher)
+ // enable decimal data type
+ .sessionOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY, true)
+ // Enable the new cast functions (cast empty string "" to null)
+ .systemOption(ExecConstants.CAST_EMPTY_STRING_TO_NULL, true);
+ startCluster(builder);
}
- @AfterClass
- public static void disableDecimalDataType() throws Exception {
- test("alter session set `%s` = false", PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY);
+ @Test // see DRILL-1874
+ public void testCastOptionalVarCharToNumeric() throws Exception {
+ testCastOptionalString("columns[0]", "int", "cp.`emptyStrings.csv`", null, 1, 2);
+ testCastOptionalString("columns[0]", "bigint", "cp.`emptyStrings.csv`", null, 1L, 2L);
+ testCastOptionalString("columns[0]", "float", "cp.`emptyStrings.csv`", null, 1.0f, 2.0f);
+ testCastOptionalString("columns[0]", "double", "cp.`emptyStrings.csv`", null, 1.0, 2.0);
}
@Test // see DRILL-1874
- public void testCastInputTypeNullableVarCharToNumeric() throws Exception {
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
-
- // Test Optional VarChar
- test("select cast(columns[0] as int) from cp.`emptyStrings.csv`");
- test("select cast(columns[0] as bigint) from cp.`emptyStrings.csv`");
- test("select cast(columns[0] as float) from cp.`emptyStrings.csv`");
- test("select cast(columns[0] as double) from cp.`emptyStrings.csv`");
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ public void testCastRequiredVarCharToNumeric() throws Exception {
+ testCastEmptyString("int");
+ testCastEmptyString("bigint");
+ testCastEmptyString("float");
+ testCastEmptyString("double");
}
@Test // see DRILL-1874
- public void testCastInputTypeNonNullableVarCharToNumeric() throws Exception {
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
- // Test Required VarChar
- test("select cast('' as int) from cp.`emptyStrings.csv`");
- test("select cast('' as bigint) from cp.`emptyStrings.csv`");
- test("select cast('' as float) from cp.`emptyStrings.csv`");
- test("select cast('' as double) from cp.`emptyStrings.csv`");
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ public void testCastOptionalVarCharToDecimal() throws Exception {
+ BigDecimal one = BigDecimal.valueOf(1L);
+ BigDecimal two = BigDecimal.valueOf(2L);
+ testCastOptionalString("columns[0]", "decimal", "cp.`emptyStrings.csv`", null, one, two);
+ testCastOptionalString("columns[0]", "decimal(9)", "cp.`emptyStrings.csv`", null, one, two);
+ testCastOptionalString("columns[0]", "decimal(18)", "cp.`emptyStrings.csv`", null, one, two);
+ testCastOptionalString("columns[0]", "decimal(28)", "cp.`emptyStrings.csv`", null, one, two);
+ testCastOptionalString("columns[0]", "decimal(38)", "cp.`emptyStrings.csv`", null, one, two);
}
@Test // see DRILL-1874
- public void testCastInputTypeNullableVarCharToDecimal() throws Exception {
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+ public void testCastRequiredVarCharToDecimal() throws Exception {
+ testCastEmptyString("decimal");
+ testCastEmptyString("decimal(18)");
+ testCastEmptyString("decimal(28)");
+ testCastEmptyString("decimal(38)");
+ }
- // Test Optional VarChar
- test("select cast(columns[0] as decimal) from cp.`emptyStrings.csv` where cast(columns[0] as decimal) is null");
- test("select cast(columns[0] as decimal(9)) from cp.`emptyStrings.csv`");
- test("select cast(columns[0] as decimal(18)) from cp.`emptyStrings.csv`");
- test("select cast(columns[0] as decimal(28)) from cp.`emptyStrings.csv`");
- test("select cast(columns[0] as decimal(38)) from cp.`emptyStrings.csv`");
+ @Test
+ public void testCastRequiredVarCharToDateTime() throws Exception {
+ testCastEmptyString("date");
+ testCastEmptyString("time");
+ testCastEmptyString("timestamp");
+ }
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ @Test
+ public void testCastOptionalVarCharToDateTime() throws Exception {
+ testCastOptionalString("dateCol", "date", "cp.`dateWithEmptyStrings.json`",
+ null, null, LocalDate.of(1997, 12, 10));
+ testCastOptionalString("timeCol", "time", "cp.`dateWithEmptyStrings.json`",
+ null, null, LocalTime.of(7, 21, 39));
+ testCastOptionalString("timestampCol", "timestamp", "cp.`dateWithEmptyStrings.json`",
+ null, null, LocalDateTime.of(2003, 9, 11, 10, 1, 37));
}
- @Test // see DRILL-1874
- public void testCastInputTypeNonNullableVarCharToDecimal() throws Exception {
- // Enable the new cast functions (cast empty string "" to null)
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = true;");
+ @Test
+ public void testCastRequiredVarCharToInterval() throws Exception {
+ testCastEmptyString("interval year");
+ testCastEmptyString("interval day");
+ testCastEmptyString("interval month");
+ }
+
+ private void testCastOptionalString(String column, String asType, String table,
+ Object... baselineValues) throws Exception {
+ String query = String.format("select cast(%s as %s) c from %s", column, asType, table);
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("c")
+ .baselineValuesForSingleColumn(baselineValues)
+ .go();
+ }
- // Test Required VarChar
- test("select cast('' as decimal) from cp.`emptyStrings.csv` where cast('' as decimal) is null");
- test("select cast('' as decimal(18)) from cp.`emptyStrings.csv`");
- test("select cast('' as decimal(28)) from cp.`emptyStrings.csv`");
- test("select cast('' as decimal(38)) from cp.`emptyStrings.csv`");
+ private void testCastEmptyString(String asType) throws Exception {
+ Object[] nullObj = new Object[] {null};
+ String query = String.format("select cast('' as %s) c from (values(1))", asType);
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("c")
+ .baselineValues(nullObj)
+ .go();
+ }
+
+ @Test
+ public void testCastOptionalVarCharToNumber() throws Exception {
+ testBuilder()
+ .sqlQuery("select to_number(columns[0], '#,##0.0') n from cp.`emptyStrings.csv`")
+ .unOrdered()
+ .baselineColumns("n")
+ .baselineValuesForSingleColumn(null, 1.0, 2.0)
+ .go();
+ }
- test("alter system set `drill.exec.functions.cast_empty_string_to_null` = false;");
+ @Test
+ public void testCastRequiredVarCharToNumber() throws Exception {
+ Object[] nullObj = new Object[] {null};
+ testBuilder()
+ .sqlQuery("select to_number('', '#,##0.0') n from (values(1))")
+ .unOrdered()
+ .baselineColumns("n")
+ .baselineValues(nullObj)
+ .go();
}
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
index 05d3831..e0121e6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/testing/TestDateConversions.java
@@ -30,12 +30,20 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+
import static org.hamcrest.CoreMatchers.startsWith;
import static org.junit.Assert.assertThat;
@RunWith(JMockit.class)
@Category({UnlikelyTest.class, SqlFunctionTest.class})
public class TestDateConversions extends BaseTestQuery {
+
+ private static final String ENABLE_CAST_EMPTY_STRING_AS_NULL_QUERY =
+ "alter system set `drill.exec.functions.cast_empty_string_to_null` = true;";
+
@BeforeClass
public static void generateTestFiles() throws IOException {
try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dirTestWatcher.getRootDir(), "joda_postgres_date.json")))) {
@@ -206,4 +214,91 @@ public class TestDateConversions extends BaseTestQuery {
throw e;
}
}
+
+ @Test
+ public void testToDateWithEmptyString() throws Exception {
+ String query = "SELECT to_date(dateCol, 'yyyy-MM-dd') d from cp.`dateWithEmptyStrings.json`";
+ testToDateTimeFunctionWithEmptyStringsAsNull(query, "d", null, null, LocalDate.of(1997, 12, 10));
+ }
+
+ @Test
+ public void testToTimeWithEmptyString() throws Exception {
+ String query = "SELECT to_time(timeCol, 'hh:mm:ss') t from cp.`dateWithEmptyStrings.json`";
+ testToDateTimeFunctionWithEmptyStringsAsNull(query, "t", null, null, LocalTime.of(7, 21, 39));
+ }
+
+ @Test
+ public void testToTimeStampWithEmptyString() throws Exception {
+ String query = "SELECT to_timestamp(timestampCol, 'yyyy-MM-dd hh:mm:ss') t from cp.`dateWithEmptyStrings.json`";
+ testToDateTimeFunctionWithEmptyStringsAsNull(query, "t", null, null, LocalDateTime.of(2003, 9, 11, 10, 1, 37));
+ }
+
+ @Test
+ public void testToDateWithLiteralEmptyString() throws Exception {
+ Object[] nullObj = new Object[] {null};
+ testToDateTimeFunctionWithEmptyStringsAsNull("SELECT to_date('', 'yyyy-MM-dd') d from (values(1))", "d", nullObj);
+ }
+
+ @Test
+ public void testToTimeWithLiteralEmptyString() throws Exception {
+ Object[] nullObj = new Object[] {null};
+ testToDateTimeFunctionWithEmptyStringsAsNull("SELECT to_time('', 'hh:mm:ss') d from (values(1))", "d", nullObj);
+ }
+
+ @Test
+ public void testToTimeStampWithLiteralEmptyString() throws Exception {
+ Object[] nullObj = new Object[] {null};
+ testToDateTimeFunctionWithEmptyStringsAsNull(
+ "SELECT to_timestamp('', 'yyyy-MM-dd hh:mm:ss') d from (values(1))", "d", nullObj);
+ }
+
+ @Test
+ public void testSqlToDateWithEmptyString() throws Exception {
+ String query = "SELECT sql_to_date(dateCol, 'yyyy-MM-dd') d from cp.`dateWithEmptyStrings.json`";
+ testToDateTimeFunctionWithEmptyStringsAsNull(query, "d", null, null, LocalDate.of(1997, 12, 10));
+ }
+
+ @Test
+ public void testSqlToTimeWithEmptyString() throws Exception {
+ String query = "SELECT sql_to_time(timeCol, 'HH24:MI:SS') t from cp.`dateWithEmptyStrings.json`";
+ testToDateTimeFunctionWithEmptyStringsAsNull(query, "t", null, null, LocalTime.of(7, 21, 39));
+ }
+
+ @Test
+ public void testSqlToTimeStampWithEmptyString() throws Exception {
+ String query = "SELECT sql_to_timestamp(timestampCol, 'yyyy-MM-dd HH24:MI:SS') t from cp.`dateWithEmptyStrings.json`";
+ testToDateTimeFunctionWithEmptyStringsAsNull(query, "t", null, null, LocalDateTime.of(2003, 9, 11, 10, 1, 37));
+ }
+
+ @Test
+ public void testSqlToDateWithLiteralEmptyString() throws Exception {
+ Object[] nullObj = new Object[] {null};
+ testToDateTimeFunctionWithEmptyStringsAsNull(
+ "SELECT sql_to_date('', 'yyyy-MM-dd') d from (values(1))", "d", nullObj);
+ }
+
+ @Test
+ public void testSqlToTimeWithLiteralEmptyString() throws Exception {
+ Object[] nullObj = new Object[] {null};
+ testToDateTimeFunctionWithEmptyStringsAsNull(
+ "SELECT sql_to_time('', 'HH24:MI:SS') d from (values(1))", "d", nullObj);
+ }
+
+ @Test
+ public void testSqlToTimeStampWithLiteralEmptyString() throws Exception {
+ Object[] nullObj = new Object[] {null};
+ testToDateTimeFunctionWithEmptyStringsAsNull(
+ "SELECT sql_to_timestamp('', 'yyyy-MM-dd HH24:MI:SS') d from (values(1))", "d", nullObj);
+ }
+
+ private void testToDateTimeFunctionWithEmptyStringsAsNull(
+ String query, String baselineColumn, Object... baselineValues) throws Exception {
+ testBuilder()
+ .optionSettingQueriesForTestQuery(ENABLE_CAST_EMPTY_STRING_AS_NULL_QUERY)
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns(baselineColumn)
+ .baselineValuesForSingleColumn(baselineValues)
+ .go();
+ }
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java
index 4df7af5..9f26b2e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/ConfigBuilder.java
@@ -142,7 +142,7 @@ public class ConfigBuilder {
private static Properties createDefaultProperties()
{
Properties properties = new Properties();
- properties.put(ExecConstants.CAST_TO_NULLABLE_NUMERIC, "false");
+ properties.put(ExecConstants.CAST_EMPTY_STRING_TO_NULL, "false");
properties.put(ExecConstants.USE_DYNAMIC_UDFS_KEY, "false");
properties.put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, "false");
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java
index 775f546..d45bd6f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestBuilder.java
@@ -25,6 +25,7 @@ import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -354,6 +355,24 @@ public class TestBuilder {
}
/**
+ * This method is used to pass in an array of values for records verification in case if
+ * {@link #baselineColumns(String...)} specifies one column only without
+ * the need to create a CSV or JSON file to store the baseline.
+ *
+ * This can be called repeatedly to pass an array of records to verify. It works for both ordered and unordered
+ * checks.
+ *
+ * @param baselineValues baseline values for a single column to validate
+ * @return {@code this} test builder
+ */
+ public TestBuilder baselineValuesForSingleColumn(Object... baselineValues) {
+ assertEquals("Only one column should be specified", 1, baselineColumns.length);
+ Arrays.stream(baselineValues)
+ .forEach(this::baselineValues);
+ return this;
+ }
+
+ /**
* This can be used in cases where we want to avoid issues with the assumptions made by the test framework.
* Most of the methods for verification in the framework run drill queries to generate the read baseline files or
* execute alternative baseline queries. This model relies on basic functionality of reading files with storage
diff --git a/exec/java-exec/src/test/resources/dateWithEmptyStrings.json b/exec/java-exec/src/test/resources/dateWithEmptyStrings.json
new file mode 100644
index 0000000..1762ee5
--- /dev/null
+++ b/exec/java-exec/src/test/resources/dateWithEmptyStrings.json
@@ -0,0 +1,3 @@
+{"dateCol": null, "timeCol": "", "timestampCol": "2003-09-11 10:01:37"}
+{"dateCol": "1997-12-10", "timeCol": null, "timestampCol": ""}
+{"dateCol": "", "timeCol": "07:21:39", "timestampCol": null}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java b/logical/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
deleted file mode 100644
index b5ed5b6..0000000
--- a/logical/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-public class CastFunctions {
-
- private static Map<MinorType, String> TYPE2FUNC = new HashMap<>();
- /** The cast functions that need to be replaced (if
- * "drill.exec.functions.cast_empty_string_to_null" is set to true). */
- private static Set<String> CAST_FUNC_REPLACEMENT_NEEDED = new HashSet<>();
- /** Map from the replaced functions to the new ones (for non-nullable VARCHAR). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR = new HashMap<>();
- /** Map from the replaced functions to the new ones (for non-nullable VAR16CHAR). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR = new HashMap<>();
- /** Map from the replaced functions to the new ones (for non-nullable VARBINARY). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY = new HashMap<>();
- /** Map from the replaced functions to the new ones (for nullable VARCHAR). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR = new HashMap<>();
- /** Map from the replaced functions to the new ones (for nullable VAR16CHAR). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR = new HashMap<>();
- /** Map from the replaced functions to the new ones (for nullable VARBINARY). */
- private static Map<String, String> CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY = new HashMap<>();
- static {
- TYPE2FUNC.put(MinorType.UNION, "castUNION");
- TYPE2FUNC.put(MinorType.BIGINT, "castBIGINT");
- TYPE2FUNC.put(MinorType.INT, "castINT");
- TYPE2FUNC.put(MinorType.BIT, "castBIT");
- TYPE2FUNC.put(MinorType.TINYINT, "castTINYINT");
- TYPE2FUNC.put(MinorType.FLOAT4, "castFLOAT4");
- TYPE2FUNC.put(MinorType.FLOAT8, "castFLOAT8");
- TYPE2FUNC.put(MinorType.VARCHAR, "castVARCHAR");
- TYPE2FUNC.put(MinorType.VAR16CHAR, "castVAR16CHAR");
- TYPE2FUNC.put(MinorType.VARBINARY, "castVARBINARY");
- TYPE2FUNC.put(MinorType.DATE, "castDATE");
- TYPE2FUNC.put(MinorType.TIME, "castTIME");
- TYPE2FUNC.put(MinorType.TIMESTAMP, "castTIMESTAMP");
- TYPE2FUNC.put(MinorType.TIMESTAMPTZ, "castTIMESTAMPTZ");
- TYPE2FUNC.put(MinorType.INTERVALDAY, "castINTERVALDAY");
- TYPE2FUNC.put(MinorType.INTERVALYEAR, "castINTERVALYEAR");
- TYPE2FUNC.put(MinorType.INTERVAL, "castINTERVAL");
- TYPE2FUNC.put(MinorType.DECIMAL9, "castDECIMAL9");
- TYPE2FUNC.put(MinorType.DECIMAL18, "castDECIMAL18");
- TYPE2FUNC.put(MinorType.DECIMAL28SPARSE, "castDECIMAL28SPARSE");
- TYPE2FUNC.put(MinorType.DECIMAL28DENSE, "castDECIMAL28DENSE");
- TYPE2FUNC.put(MinorType.DECIMAL38SPARSE, "castDECIMAL38SPARSE");
- TYPE2FUNC.put(MinorType.DECIMAL38DENSE, "castDECIMAL38DENSE");
- TYPE2FUNC.put(MinorType.VARDECIMAL, "castVARDECIMAL");
-
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.INT));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.BIGINT));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT4));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.FLOAT8));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL9));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL18));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE));
- CAST_FUNC_REPLACEMENT_NEEDED.add(TYPE2FUNC.get(MinorType.VARDECIMAL));
-
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringVarCharToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringVarCharToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringVarCharToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringVarCharToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringVarCharToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringVarCharToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringVarCharToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringVarCharToNullableDECIMAL38SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.VARDECIMAL), "castEmptyStringVarCharToNullableVARDECIMAL");
-
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringVar16CharToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringVar16CharToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringVar16CharToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringVar16CharToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringVar16CharToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringVar16CharToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringVar16CharToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringVar16CharToNullableDECIMAL38SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.VARDECIMAL), "castEmptyStringVar16CharToNullableVARDECIMAL");
-
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringVarBinaryToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringVarBinaryToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringVarBinaryToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringVarBinaryToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringVarBinaryToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringVarBinaryToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringVarBinaryToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringVarBinaryToNullableDECIMAL38SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.VARDECIMAL), "castEmptyStringVarBinaryToNullableVARDECIMAL");
-
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringNullableVarCharToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringNullableVarCharToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringNullableVarCharToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringNullableVarCharToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringNullableVarCharToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringNullableVarCharToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringNullableVarCharToNullableDECIMAL38SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(TYPE2FUNC.get(MinorType.VARDECIMAL), "castEmptyStringNullableVarCharToNullableVARDECIMAL");
-
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringNullableVar16CharToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringNullableVar16CharToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringNullableVar16CharToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringNullableVar16CharToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringNullableVar16CharToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringNullableVar16CharToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringNullableVar16CharToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringNullableVar16CharToNullableDECIMAL38SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(TYPE2FUNC.get(MinorType.VARDECIMAL), "castEmptyStringNullableVar16CharToNullableVARDECIMAL");
-
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.INT), "castEmptyStringNullableVarBinaryToNullableINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.BIGINT), "castEmptyStringNullableVarBinaryToNullableBIGINT");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.FLOAT4), "castEmptyStringNullableVarBinaryToNullableFLOAT4");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.FLOAT8), "castEmptyStringNullableVarBinaryToNullableFLOAT8");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL9), "castEmptyStringNullableVarBinaryToNullableDECIMAL9");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL18), "castEmptyStringNullableVarBinaryToNullableDECIMAL18");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL28SPARSE), "castEmptyStringNullableVarBinaryToNullableDECIMAL28SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.DECIMAL38SPARSE), "castEmptyStringNullableVarBinaryToNullableDECIMAL38SPARSE");
- CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(TYPE2FUNC.get(MinorType.VARDECIMAL), "castEmptyStringNullableVarBinaryToNullableVARDECIMAL");
- }
-
- /**
- * Given the target type, get the appropriate cast function
- * @param targetMinorType the target data type
- * @return the name of cast function
- */
- public static String getCastFunc(MinorType targetMinorType) {
- String func = TYPE2FUNC.get(targetMinorType);
- if (func != null) {
- return func;
- }
-
- throw new IllegalArgumentException(
- String.format("cast function for type %s is not defined", targetMinorType.name()));
- }
-
- /**
- * Get a replacing cast function for the original function, based on the specified data mode
- * @param originalCastFunction original cast function
- * @param dataMode data mode of the input data
- * @param inputType input (minor) type for cast
- * @return the name of replaced cast function
- */
- public static String getReplacingCastFunction(String originalCastFunction, DataMode dataMode, MinorType inputType) {
- if(dataMode == DataMode.OPTIONAL) {
- return getReplacingCastFunctionFromNullable(originalCastFunction, inputType);
- }
-
- if(dataMode == DataMode.REQUIRED) {
- return getReplacingCastFunctionFromNonNullable(originalCastFunction, inputType);
- }
-
- throw new RuntimeException(
- String.format("replacing cast function for datatype %s is not defined", dataMode));
- }
-
- /**
- * Check if a replacing cast function is available for the the original function
- * @param originalfunction original cast function
- * @param inputType input (minor) type for cast
- * @return true if replacement is needed, false - if isn't
- */
- public static boolean isReplacementNeeded(String originalfunction, MinorType inputType) {
- return (inputType == MinorType.VARCHAR || inputType == MinorType.VARBINARY || inputType == MinorType.VAR16CHAR) &&
- CAST_FUNC_REPLACEMENT_NEEDED.contains(originalfunction);
- }
-
- /**
- * Check if a funcName is one of the cast function.
- * @param funcName
- * @return
- */
- public static boolean isCastFunction(String funcName) {
- return TYPE2FUNC.values().contains(funcName);
- }
-
- private static String getReplacingCastFunctionFromNonNullable(String originalCastFunction, MinorType inputType) {
- if(inputType == MinorType.VARCHAR && CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARCHAR.get(originalCastFunction);
- }
- if(inputType == MinorType.VAR16CHAR && CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VAR16CHAR.get(originalCastFunction);
- }
- if(inputType == MinorType.VARBINARY && CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NONNULLABLE_VARBINARY.get(originalCastFunction);
- }
-
- throw new RuntimeException(
- String.format("replacing cast function for %s is not defined", originalCastFunction));
- }
-
- private static String getReplacingCastFunctionFromNullable(String originalCastFunction, MinorType inputType) {
- if(inputType == MinorType.VARCHAR && CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.get(originalCastFunction);
- }
- if(inputType == MinorType.VAR16CHAR && CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.get(originalCastFunction);
- }
- if(inputType == MinorType.VARBINARY && CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.containsKey(originalCastFunction)) {
- return CAST_FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.get(originalCastFunction);
- }
-
- throw new RuntimeException(
- String.format("replacing cast function for %s is not defined", originalCastFunction));
- }
-}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/fn/FunctionReplacementUtils.java b/logical/src/main/java/org/apache/drill/common/expression/fn/FunctionReplacementUtils.java
new file mode 100644
index 0000000..982423e
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/fn/FunctionReplacementUtils.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.fn;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public class FunctionReplacementUtils {
+
+ private static final Map<MinorType, String> TYPE_TO_CAST_FUNC = new HashMap<>();
+ // Maps function to supported input types for substitution
+ private static final Map<String, Set<MinorType>> FUNC_TO_INPUT_TYPES = new HashMap<>();
+ /**
+ * The functions that need to be replaced (if
+ * {@code "drill.exec.functions.cast_empty_string_to_null"} is set to {@code true}).
+ */
+ private static final Set<String> FUNC_REPLACEMENT_NEEDED = new HashSet<>();
+ /** Map from the replaced functions to the new ones (for non-nullable VARCHAR). */
+ private static final Map<String, String> FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARCHAR = new HashMap<>();
+ /** Map from the replaced functions to the new ones (for non-nullable VAR16CHAR). */
+ private static final Map<String, String> FUNC_REPLACEMENT_FROM_NON_NULLABLE_VAR16CHAR = new HashMap<>();
+ /** Map from the replaced functions to the new ones (for non-nullable VARBINARY). */
+ private static final Map<String, String> FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARBINARY = new HashMap<>();
+ /** Map from the replaced functions to the new ones (for nullable VARCHAR). */
+ private static final Map<String, String> FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR = new HashMap<>();
+ /** Map from the replaced functions to the new ones (for nullable VAR16CHAR). */
+ private static final Map<String, String> FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR = new HashMap<>();
+ /** Map from the replaced functions to the new ones (for nullable VARBINARY). */
+ private static final Map<String, String> FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY = new HashMap<>();
+
+ static {
+ initCastFunctionSubstitutions();
+ initToFunctionSubstitutions();
+ }
+
+ private static void initCastFunctionSubstitutions() {
+ TYPE_TO_CAST_FUNC.put(MinorType.UNION, "castUNION");
+ TYPE_TO_CAST_FUNC.put(MinorType.BIGINT, "castBIGINT");
+ TYPE_TO_CAST_FUNC.put(MinorType.INT, "castINT");
+ TYPE_TO_CAST_FUNC.put(MinorType.BIT, "castBIT");
+ TYPE_TO_CAST_FUNC.put(MinorType.TINYINT, "castTINYINT");
+ TYPE_TO_CAST_FUNC.put(MinorType.FLOAT4, "castFLOAT4");
+ TYPE_TO_CAST_FUNC.put(MinorType.FLOAT8, "castFLOAT8");
+ TYPE_TO_CAST_FUNC.put(MinorType.VARCHAR, "castVARCHAR");
+ TYPE_TO_CAST_FUNC.put(MinorType.VAR16CHAR, "castVAR16CHAR");
+ TYPE_TO_CAST_FUNC.put(MinorType.VARBINARY, "castVARBINARY");
+ TYPE_TO_CAST_FUNC.put(MinorType.DATE, "castDATE");
+ TYPE_TO_CAST_FUNC.put(MinorType.TIME, "castTIME");
+ TYPE_TO_CAST_FUNC.put(MinorType.TIMESTAMP, "castTIMESTAMP");
+ TYPE_TO_CAST_FUNC.put(MinorType.TIMESTAMPTZ, "castTIMESTAMPTZ");
+ TYPE_TO_CAST_FUNC.put(MinorType.INTERVALDAY, "castINTERVALDAY");
+ TYPE_TO_CAST_FUNC.put(MinorType.INTERVALYEAR, "castINTERVALYEAR");
+ TYPE_TO_CAST_FUNC.put(MinorType.INTERVAL, "castINTERVAL");
+ TYPE_TO_CAST_FUNC.put(MinorType.DECIMAL9, "castDECIMAL9");
+ TYPE_TO_CAST_FUNC.put(MinorType.DECIMAL18, "castDECIMAL18");
+ TYPE_TO_CAST_FUNC.put(MinorType.DECIMAL28SPARSE, "castDECIMAL28SPARSE");
+ TYPE_TO_CAST_FUNC.put(MinorType.DECIMAL28DENSE, "castDECIMAL28DENSE");
+ TYPE_TO_CAST_FUNC.put(MinorType.DECIMAL38SPARSE, "castDECIMAL38SPARSE");
+ TYPE_TO_CAST_FUNC.put(MinorType.DECIMAL38DENSE, "castDECIMAL38DENSE");
+ TYPE_TO_CAST_FUNC.put(MinorType.VARDECIMAL, "castVARDECIMAL");
+
+ // Numeric types
+ setupReplacementFunctionsForCast(MinorType.INT, "NullableINT");
+ setupReplacementFunctionsForCast(MinorType.BIGINT, "NullableBIGINT");
+ setupReplacementFunctionsForCast(MinorType.FLOAT4, "NullableFLOAT4");
+ setupReplacementFunctionsForCast(MinorType.FLOAT8, "NullableFLOAT8");
+ setupReplacementFunctionsForCast(MinorType.DECIMAL9, "NullableDECIMAL9");
+ setupReplacementFunctionsForCast(MinorType.DECIMAL18, "NullableDECIMAL18");
+ setupReplacementFunctionsForCast(MinorType.DECIMAL28SPARSE, "NullableDECIMAL28SPARSE");
+ setupReplacementFunctionsForCast(MinorType.DECIMAL38SPARSE, "NullableDECIMAL38SPARSE");
+ setupReplacementFunctionsForCast(MinorType.VARDECIMAL, "NullableVARDECIMAL");
+ // date/time types
+ setupReplacementFunctionsForCast(MinorType.DATE, "NULLABLEDATE");
+ setupReplacementFunctionsForCast(MinorType.TIME, "NULLABLETIME");
+ setupReplacementFunctionsForCast(MinorType.TIMESTAMP, "NULLABLETIMESTAMP");
+ // interval types
+ setupReplacementFunctionsForCast(MinorType.INTERVAL, "NullableINTERVAL");
+ setupReplacementFunctionsForCast(MinorType.INTERVALDAY, "NullableINTERVALDAY");
+ setupReplacementFunctionsForCast(MinorType.INTERVALYEAR, "NullableINTERVALYEAR");
+ }
+
+ private static void initToFunctionSubstitutions() {
+ setupReplacementFunctionsForTo("to_number", "ToNumber");
+
+ setupReplacementFunctionsForTo("to_date", "ToNullableDate");
+ setupReplacementFunctionsForTo("to_time", "ToNullableTime");
+ setupReplacementFunctionsForTo("to_timestamp", "ToNullableTimeStamp");
+
+ setupReplacementFunctionsForTo("sql_to_date", "SqlToNullableDate");
+ setupReplacementFunctionsForTo("sql_to_time", "SqlToNullableTime");
+ setupReplacementFunctionsForTo("sql_to_timestamp", "SqlToNullableTimeStamp");
+ }
+
+ private static void setupReplacementFunctionsForCast(MinorType type, String toSuffix) {
+ String functionName = TYPE_TO_CAST_FUNC.get(type);
+
+ FUNC_REPLACEMENT_NEEDED.add(functionName);
+ Set<MinorType> supportedInputTypes = new HashSet<>(
+ Arrays.asList(MinorType.VARCHAR, MinorType.VAR16CHAR, MinorType.VARBINARY));
+ FUNC_TO_INPUT_TYPES.put(functionName, supportedInputTypes);
+
+ FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARCHAR.put(functionName, "castEmptyStringVarCharTo" + toSuffix);
+ FUNC_REPLACEMENT_FROM_NON_NULLABLE_VAR16CHAR.put(functionName, "castEmptyStringVar16CharTo" + toSuffix);
+ FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARBINARY.put(functionName, "castEmptyStringVarBinaryTo" + toSuffix);
+
+ FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(functionName, "castEmptyStringNullableVarCharTo" + toSuffix);
+ FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.put(functionName, "castEmptyStringNullableVar16CharTo" + toSuffix);
+ FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.put(functionName, "castEmptyStringNullableVarBinaryTo" + toSuffix);
+ }
+
+ private static void setupReplacementFunctionsForTo(String functionName, String toSuffix) {
+ Set<MinorType> typeSet = Collections.singleton(MinorType.VARCHAR);
+ FUNC_TO_INPUT_TYPES.put(functionName, typeSet);
+ FUNC_REPLACEMENT_NEEDED.add(functionName);
+
+ FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARCHAR.put(functionName,"convertVarChar" + toSuffix);
+ FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.put(functionName, "convertNullableVarChar" + toSuffix);
+ }
+
+ /**
+ * Given the target type, get the appropriate cast function
+ * @param targetMinorType the target data type
+ * @return the name of cast function
+ */
+ public static String getCastFunc(MinorType targetMinorType) {
+ String func = TYPE_TO_CAST_FUNC.get(targetMinorType);
+ if (func != null) {
+ return func;
+ }
+
+ throw new IllegalArgumentException(
+ String.format("cast function for type %s is not defined", targetMinorType.name()));
+ }
+
+ /**
+ * Get a replacing function for the original function, based on the specified data mode
+ * @param functionName original function name
+ * @param dataMode data mode of the input data
+ * @param inputType input (minor) type
+ * @return the name of replaced function
+ */
+ public static String getReplacingFunction(String functionName, DataMode dataMode, MinorType inputType) {
+ if (dataMode == DataMode.OPTIONAL) {
+ return getReplacingFunctionFromNullable(functionName, inputType);
+ }
+
+ if (dataMode == DataMode.REQUIRED) {
+ return getReplacingFunctionFromNonNullable(functionName, inputType);
+ }
+
+ throw new DrillRuntimeException(
+ String.format("replacing function '%s' for datatype %s is not defined", functionName, dataMode));
+ }
+
+ /**
+ * Check if a replacing function is available for the the original function
+ * @param functionName original function name
+ * @param inputType input (minor) type
+ * @return {@code true} if replacement is needed, {@code false} otherwise
+ */
+ public static boolean isReplacementNeeded(String functionName, MinorType inputType) {
+ return FUNC_REPLACEMENT_NEEDED.contains(functionName)
+ && FUNC_TO_INPUT_TYPES.get(functionName).contains(inputType);
+ }
+
+ /**
+ * Check if a function is a cast function.
+ * @param functionName name of the function
+ * @return {@code true} if function is CAST function, {@code false} otherwise
+ */
+ public static boolean isCastFunction(String functionName) {
+ return TYPE_TO_CAST_FUNC.values().contains(functionName);
+ }
+
+ private static String getReplacingFunctionFromNonNullable(String functionName, MinorType inputType) {
+ if (inputType == MinorType.VARCHAR
+ && FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARCHAR.containsKey(functionName)) {
+ return FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARCHAR.get(functionName);
+ }
+ if (inputType == MinorType.VAR16CHAR
+ && FUNC_REPLACEMENT_FROM_NON_NULLABLE_VAR16CHAR.containsKey(functionName)) {
+ return FUNC_REPLACEMENT_FROM_NON_NULLABLE_VAR16CHAR.get(functionName);
+ }
+ if (inputType == MinorType.VARBINARY
+ && FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARBINARY.containsKey(functionName)) {
+ return FUNC_REPLACEMENT_FROM_NON_NULLABLE_VARBINARY.get(functionName);
+ }
+
+ throw new DrillRuntimeException(
+ String.format("replacing function for %s is not defined", functionName));
+ }
+
+ private static String getReplacingFunctionFromNullable(String functionName, MinorType inputType) {
+ if (inputType == MinorType.VARCHAR
+ && FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.containsKey(functionName)) {
+ return FUNC_REPLACEMENT_FROM_NULLABLE_VARCHAR.get(functionName);
+ }
+ if (inputType == MinorType.VAR16CHAR
+ && FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.containsKey(functionName)) {
+ return FUNC_REPLACEMENT_FROM_NULLABLE_VAR16CHAR.get(functionName);
+ }
+ if (inputType == MinorType.VARBINARY
+ && FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.containsKey(functionName)) {
+ return FUNC_REPLACEMENT_FROM_NULLABLE_VARBINARY.get(functionName);
+ }
+
+ throw new DrillRuntimeException(
+ String.format("replacing function for %s is not defined", functionName));
+ }
+}