You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2021/05/26 23:43:29 UTC
[asterixdb] 23/38: [ASTERIXDB-2897][FUN] Fix unknown handling in
PyUDF
This is an automated email from the ASF dual-hosted git repository.
imaxon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ec5195a4f58b2ce3971b537b48078f3bc696e0bf
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Tue May 4 09:35:21 2021 -0700
[ASTERIXDB-2897][FUN] Fix unknown handling in PyUDF
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Carry the fix from ASTERIXDB-2883 to Python UDFs,
and add a corresponding test for both this new
code in the non-batched execution, as well as
checking that it works the same in batched mode
Change-Id: I64ac16d094caafcc657a21650011ed41d0c3c6d6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11324
Reviewed-by: Ian Maxon <im...@uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
...3.query.sqlpp => type_validation.10.query.sqlpp} | 7 +++++--
...ion.4.ddl.sqlpp => type_validation.11.ddl.sqlpp} | 0
.../type_validation.2.ddl.sqlpp | 4 +++-
.../type_validation.3.query.sqlpp | 3 ++-
....3.query.sqlpp => type_validation.4.query.sqlpp} | 5 ++++-
....3.query.sqlpp => type_validation.5.query.sqlpp} | 5 +++--
....3.query.sqlpp => type_validation.6.query.sqlpp} | 7 +++++--
....3.query.sqlpp => type_validation.7.query.sqlpp} | 3 ++-
....3.query.sqlpp => type_validation.8.query.sqlpp} | 5 ++++-
....3.query.sqlpp => type_validation.9.query.sqlpp} | 5 +++--
.../type_validation.1.adm | 2 +-
.../type_validation.2.adm | 1 +
.../type_validation.3.adm | 1 +
.../type_validation.4.adm | 1 +
.../type_validation.5.adm | 1 +
.../type_validation.6.adm | 1 +
.../type_validation.7.adm | 1 +
.../type_validation.8.adm | 1 +
.../ExternalScalarPythonFunctionEvaluator.java | 21 ++++++++++++++++++---
19 files changed, 57 insertions(+), 17 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
index 0ae7d0c..69a0b25 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
index a8ba8a1..8041b5e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp
@@ -18,6 +18,8 @@
*/
use externallibtest;
-create function typeValidation(a, b, c, d, e, f, g)
+create function typeValidation(a, b, c, d, e, f, g, h)
as "roundtrip", "Tests.roundtrip" at testlib;
+create function typeValidationNullCall(a, b, c, d, e, f, g, h)
+ as "roundtrip", "Tests.roundtrip" at testlib with {"null-call": true};
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
index 0ae7d0c..f9aaf57 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
index 0ae7d0c..884f31f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
similarity index 87%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
index 0ae7d0c..fdd71ef 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
similarity index 83%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
index 0ae7d0c..b85e0d0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+set `rewrite_attempt_batch_assign` "false";
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
similarity index 96%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
index 0ae7d0c..fa76f1a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
index 0ae7d0c..30640fd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp
@@ -17,6 +17,9 @@
* under the License.
*/
use externallibtest;
+
+set `rewrite_attempt_batch_assign` "false";
+
typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
similarity index 86%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
index 0ae7d0c..810ba3c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp
@@ -17,6 +17,7 @@
* under the License.
*/
use externallibtest;
-typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
- unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")));
+
+typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")),
+ unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
index 93f8aec..19765bd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm
@@ -1 +1 @@
-[ 907, 9.07, "907", 9.07, true, 15706, 621173594 ]
+null
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm
new file mode 100644
index 0000000..19765bd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm
@@ -0,0 +1 @@
+null
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm
new file mode 100644
index 0000000..ec747fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm
@@ -0,0 +1 @@
+null
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm
new file mode 100644
index 0000000..53fd925
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm
@@ -0,0 +1 @@
+[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ]
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
index e664f47..eb87399 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.library;
+import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER;
import static org.msgpack.core.MessagePack.Code.FIXARRAY_PREFIX;
import java.io.DataOutput;
@@ -88,17 +89,31 @@ class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvalua
@Override
public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
argHolder.clear();
+ boolean nullCall = finfo.getNullCall();
+ boolean hasNullArg = false;
for (int i = 0, ln = argEvals.length; i < ln; i++) {
argEvals[i].evaluate(tuple, argValues[i]);
- if (!finfo.getNullCall() && PointableHelper.checkAndSetMissingOrNull(result, argValues[i])) {
- return;
+ if (!nullCall) {
+ byte[] argBytes = argValues[i].getByteArray();
+ int argStart = argValues[i].getStartOffset();
+ ATypeTag argType = ATYPETAGDESERIALIZER.deserialize(argBytes[argStart]);
+ if (argType == ATypeTag.MISSING) {
+ PointableHelper.setMissing(result);
+ return;
+ } else if (argType == ATypeTag.NULL) {
+ hasNullArg = true;
+ }
}
try {
- PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, finfo.getNullCall());
+ PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, nullCall);
} catch (IOException e) {
throw new HyracksDataException("Error evaluating Python UDF", e);
}
}
+ if (!nullCall && hasNullArg) {
+ PointableHelper.setNull(result);
+ return;
+ }
try {
ByteBuffer res = libraryEvaluator.callPython(fnId, argHolder, argTypes.length);
resultBuffer.reset();