You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2023/10/26 06:18:02 UTC
Change in asterixdb[master]: [NO ISSUE][EXT]: Handle type mismatch when computed field is not pass...
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17884 )
Change subject: [NO ISSUE][EXT]: Handle type mismatch when computed field is not passed in query
......................................................................
[NO ISSUE][EXT]: Handle type mismatch when computed field is not passed in query
Change-Id: I18e90d1dae8f61801d4467bcad02a0e13aed6c9d
---
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
5 files changed, 58 insertions(+), 0 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/84/17884/1
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp
new file mode 100644
index 0000000..afc8b4a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=10
+
+USE test;
+
+SELECT value count(*)
+FROM test t;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm
new file mode 100644
index 0000000..c227083
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm
@@ -0,0 +1 @@
+0
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index e252ab2..acf7693 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -288,6 +288,10 @@
<expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn>
<expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn>
<expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn>
+ <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 6d9de7a..7515996 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -243,6 +243,10 @@
<expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn>
<expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn>
<expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn>
+ <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes">
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 11de155..160641e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -19,6 +19,7 @@
package org.apache.asterix.external.util;
+import static org.apache.asterix.external.input.filter.ExternalFilterValueEvaluator.writeValue;
import static org.apache.asterix.external.util.ExternalDataConstants.COMPUTED_FIELD_PATTERN;
import static org.apache.asterix.external.util.ExternalDataConstants.PREFIX_DEFAULT_DELIMITER;
@@ -37,6 +38,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.external.IExternalFilterEvaluator;
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
@@ -47,7 +49,9 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.util.LogRedactionUtil;
+import org.apache.hyracks.util.string.UTF8StringWriter;
public final class ExternalDataPrefix implements Serializable {
private static final long serialVersionUID = -7612997190679310483L;
@@ -277,6 +281,12 @@
String computedFieldName = null;
IAType computedFieldType = null;
String computedFieldValue = null;
+
+ // TODO(htowaileb): this is temporary workaround, we should do something better.
+ // Added here to avoid serialization issues
+ ArrayBackedValueStorage arrayBackedStorage = new ArrayBackedValueStorage();
+ AStringSerializerDeserializer stringSerDer = new AStringSerializerDeserializer(new UTF8StringWriter(), null);
+
try {
for (int i = 0; i < computedFieldNames.size(); i++) {
computedFieldName = computedFieldNames.get(i);
@@ -285,6 +295,11 @@
if (evaluator.isComputedFieldUsed(i)) {
evaluator.setValue(i, computedFieldValue);
+ } else {
+ // TODO(htowaileb): This is temporary workaround, we should do something better
+ // although the computed field is not used, we need to validate since this will go to the value embedder
+ arrayBackedStorage.reset();
+ writeValue(computedFieldType.getTypeTag(), computedFieldValue, arrayBackedStorage, stringSerDer);
}
}
} catch (NumberFormatException ex) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17884
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I18e90d1dae8f61801d4467bcad02a0e13aed6c9d
Gerrit-Change-Number: 17884
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange