You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2023/12/05 03:37:34 UTC
(asterixdb) 02/07: [MULTIPLE ISSUES][COMP] Multiple array index fixes
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 08b9cdb0cf6ef9eaaa3ee44fff08c4c2022b8ef7
Author: Peeyush Gupta <pe...@couchbase.com>
AuthorDate: Wed Nov 29 14:46:18 2023 -0800
[MULTIPLE ISSUES][COMP] Multiple array index fixes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Fixes ASTERIXDB-3284 and ASTERIXDB-3285.
In the upsert pipeline, warning is returned instead of an error
when the indexed field is not an array. Elements of the
array having type different then the indexed type are not inserted
in the index. For the bulk load pipeline, on type mismatch the
index creation fails.
Change-Id: Icb7dd9ec54b4cf77963172adc93ed93704a2455d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17987
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Peeyush Gupta <pe...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
.../IntroduceSecondaryIndexInsertDeleteRule.java | 58 +++++++++++++++-------
.../optimizer/rules/am/AccessMethodUtils.java | 2 +-
.../rules/am/array/ArrayIndexStructureMatcher.java | 6 ++-
.../index-bad-fields-bulkload.000.ddl.sqlpp | 28 +++++++++++
.../index-bad-fields-bulkload.001.update.sqlpp | 28 +++++++++++
.../index-bad-fields-bulkload.002.ddl.sqlpp | 22 ++++++++
.../index-bad-fields-bulkload.003.ddl.sqlpp | 22 ++++++++
.../index-bad-fields-bulkload.004.ddl.sqlpp | 22 ++++++++
.../index-bad-fields-bulkload.005.ddl.sqlpp | 22 ++++++++
.../index-bad-fields-bulkload.006.query.sqlpp | 25 ++++++++++
.../index-bad-fields-bulkload.999.ddl.sqlpp | 20 ++++++++
.../index-bad-fields.000.ddl.sqlpp | 33 ++++++++++++
.../index-bad-fields.001.update.sqlpp | 39 +++++++++++++++
.../index-bad-fields.002.query.sqlpp | 22 ++++++++
.../index-bad-fields.003.query.sqlpp | 25 ++++++++++
.../index-bad-fields.004.query.sqlpp | 25 ++++++++++
.../index-bad-fields.005.query.sqlpp | 25 ++++++++++
.../index-bad-fields.006.query.sqlpp | 25 ++++++++++
.../index-bad-fields.007.query.sqlpp | 25 ++++++++++
.../index-bad-fields.999.ddl.sqlpp | 20 ++++++++
.../index-bad-fields.006.adm | 1 +
.../index-bad-fields/index-bad-fields.002.adm | 1 +
.../index-bad-fields/index-bad-fields.003.adm | 10 ++++
.../index-bad-fields/index-bad-fields.004.adm | 2 +
.../index-bad-fields/index-bad-fields.005.adm | 1 +
.../index-bad-fields/index-bad-fields.006.adm | 2 +
.../index-bad-fields/index-bad-fields.007.adm | 1 +
.../results/global-aggregate/q06/q06.3.adm | 10 ++++
.../group-by/sugar-01-negative/core-01.1.adm | 1 +
.../query-ASTERIXDB-2886.3.adm | 0
.../test/resources/runtimets/testsuite_sqlpp.xml | 29 ++++++++---
.../asterix/metadata/utils/ArrayIndexUtil.java | 16 +++---
.../SecondaryArrayIndexBTreeOperationsHelper.java | 18 ++++---
.../scalar/AbstractScalarAggregateDescriptor.java | 2 +-
.../std/ScanCollectionDescriptor.java | 14 ++++--
35 files changed, 555 insertions(+), 47 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 09ad4d1a9f..446c7791ff 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -690,7 +690,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
int sourceIndicatorForBaseRecord = arrayIndexDetails.getElementList().get(0).getSourceIndicator();
LogicalVariable sourceVarForBaseRecord = hasMetaPart
? ((sourceIndicatorForBaseRecord == Index.RECORD_INDICATOR) ? recordVar : metaVar) : recordVar;
- UnnestBranchCreator branchCreator = new UnnestBranchCreator(sourceVarForBaseRecord, unnestSourceOp);
+ UnnestBranchCreator branchCreator = new UnnestBranchCreator(index, sourceVarForBaseRecord, unnestSourceOp);
Set<LogicalVariable> secondaryKeyVars = new LinkedHashSet<>();
for (Index.ArrayIndexElement workingElement : arrayIndexDetails.getElementList()) {
@@ -712,7 +712,12 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
? getFieldAccessFunction(new MutableObject<>(varRef),
recordType.getFieldIndex(atomicFieldName.get(0)), atomicFieldName)
: getFieldAccessFunction(new MutableObject<>(varRef), -1, atomicFieldName);
-
+ IAType fieldType = recordType.getSubFieldType(atomicFieldName);
+ if (fieldType == null) {
+ newVarRef = castFunction(
+ index.isEnforced() ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX,
+ workingElement.getTypeList().get(0), newVarRef, sourceLoc);
+ }
// Add an assign on top to extract the atomic element.
AssignOperator newAssignOp = new AssignOperator(newVar, new MutableObject<>(newVarRef));
newAssignOp.setSourceLocation(sourceLoc);
@@ -728,16 +733,17 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
workingElement.getUnnestList(), workingElement.getProjectList().get(0));
List<Boolean> firstUnnestFlags = ArrayIndexUtil.getUnnestFlags(workingElement.getUnnestList(),
workingElement.getProjectList().get(0));
- ArrayIndexUtil.walkArrayPath(index, recordType, flatFirstFieldName, firstUnnestFlags, branchCreator);
+ ArrayIndexUtil.walkArrayPath(index, workingElement, recordType, flatFirstFieldName, firstUnnestFlags,
+ branchCreator);
secondaryKeyVars.add(branchCreator.lastFieldVars.get(0));
// For all other elements in the PROJECT list, add an assign.
for (int j = 1; j < workingElement.getProjectList().size(); j++) {
LogicalVariable newVar = context.newVar();
- AbstractFunctionCallExpression newVarRef =
+ ILogicalExpression newVarRef =
getFieldAccessFunction(new MutableObject<>(branchCreator.createLastRecordVarRef()), -1,
workingElement.getProjectList().get(j));
-
+ newVarRef = createCastExpressionForArrayIndex(newVarRef, recordType, index, workingElement, j);
AssignOperator newAssignOp = new AssignOperator(newVar, new MutableObject<>(newVarRef));
newAssignOp.setSourceLocation(sourceLoc);
branchCreator.currentTop = introduceNewOp(branchCreator.currentTop, newAssignOp, true);
@@ -933,7 +939,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
}
private ScalarFunctionCallExpression castFunction(FunctionIdentifier castFun, IAType requiredType,
- AbstractFunctionCallExpression inputExpr, SourceLocation sourceLoc) throws CompilationException {
+ ILogicalExpression inputExpr, SourceLocation sourceLoc) throws CompilationException {
BuiltinFunctionInfo castInfo = BuiltinFunctions.getBuiltinFunctionInfo(castFun);
ScalarFunctionCallExpression castExpr = new ScalarFunctionCallExpression(castInfo);
castExpr.setSourceLocation(sourceLoc);
@@ -957,6 +963,18 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
return constructorExpr;
}
+ private ILogicalExpression createCastExpressionForArrayIndex(ILogicalExpression varRef, ARecordType recordType,
+ Index index, Index.ArrayIndexElement workingElement, int fieldPos) throws AlgebricksException {
+ IAType fieldType = ArrayIndexUtil.getSubFieldType(recordType, workingElement.getUnnestList(),
+ workingElement.getProjectList().get(fieldPos));
+ if (fieldType != null) {
+ return varRef;
+ } else {
+ return castFunction(index.isEnforced() ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX,
+ workingElement.getTypeList().get(fieldPos), varRef, sourceLoc);
+ }
+ }
+
private ILogicalOperator introduceNewOp(ILogicalOperator currentTopOp, ILogicalOperator newOp, boolean afterOp)
throws AlgebricksException {
if (afterOp) {
@@ -1092,8 +1110,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
private final List<LogicalVariable> lastFieldVars;
private LogicalVariable lastRecordVar;
private ILogicalOperator currentTop, currentBottom = null;
+ private final Index index;
- public UnnestBranchCreator(LogicalVariable recordVar, ILogicalOperator sourceOperator) {
+ public UnnestBranchCreator(Index index, LogicalVariable recordVar, ILogicalOperator sourceOperator) {
+ this.index = index;
this.lastRecordVar = recordVar;
this.currentTop = sourceOperator;
this.lastFieldVars = new ArrayList<>();
@@ -1191,19 +1211,23 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException {
// If the final value is nested inside a record, add an additional ASSIGN.
+ ILogicalExpression accessToFinalVar;
if (!isNonArrayStep) {
- return;
+ accessToFinalVar = createCastExpressionForArrayIndex(createLastRecordVarRef(), baseRecordType, index,
+ workingElement, 0);
+ } else {
+ // Create the function to access our final field.
+ accessToFinalVar = (startingStepRecordType != null)
+ ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
+ startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
+ : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
+ accessToFinalVar =
+ createCastExpressionForArrayIndex(accessToFinalVar, baseRecordType, index, workingElement, 0);
}
-
- // Create the function to access our final field.
- AbstractFunctionCallExpression accessToFinalVar = (startingStepRecordType != null)
- ? getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()),
- startingStepRecordType.getFieldIndex(fieldName.get(0)), fieldName)
- : getFieldAccessFunction(new MutableObject<>(createLastRecordVarRef()), -1, fieldName);
-
LogicalVariable finalVar = context.newVar();
this.lastFieldVars.add(finalVar);
AssignOperator assignOperator = new AssignOperator(finalVar, new MutableObject<>(accessToFinalVar));
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 25d42dfbc4..b66a0fd199 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -3162,7 +3162,7 @@ public class AccessMethodUtils {
List<String> flatName = ArrayIndexUtil.getFlattenedKeyFieldNames(e.getUnnestList(), project);
List<Boolean> unnestFlags = ArrayIndexUtil.getUnnestFlags(e.getUnnestList(), project);
analysisCtx.getArrayIndexStructureMatcher().reset(assignVar, subTree);
- ArrayIndexUtil.walkArrayPath(index, subTree.getRecordType(), flatName, unnestFlags,
+ ArrayIndexUtil.walkArrayPath(index, e, subTree.getRecordType(), flatName, unnestFlags,
analysisCtx.getArrayIndexStructureMatcher());
LogicalVariable varAfterWalk = analysisCtx.getArrayIndexStructureMatcher().getEndVar();
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
index 62b266ae50..ddfdcacbe3 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/array/ArrayIndexStructureMatcher.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.ArrayIndexUtil;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
@@ -72,8 +73,9 @@ public class ArrayIndexStructureMatcher implements ArrayIndexUtil.TypeTrackerCom
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) {
if (isNonArrayStep) {
isStructureMatched = isStructureMatched && matchAssignVarAndFieldName(startingStepRecordType, fieldName);
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp
new file mode 100644
index 0000000000..efe6d6b957
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.000.ddl.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test array indexes where some tuples have malformed values in indexed fields
+ * Expected Res : Said tuples are not indexed
+ */
+
+CREATE DATAVERSE test;
+USE test;
+CREATE TYPE dt1 as {id:int};
+CREATE DATASET ds1(dt1) primary key id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp
new file mode 100644
index 0000000000..1822abea37
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.001.update.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+UPSERT INTO ds1 {"id": 1, "a": ["hello", 1]};
+UPSERT INTO ds1 {"id": 2, "b": ["hello", "hello"]};
+UPSERT INTO ds1 {"id": 3, "b": [11, 12]};
+UPSERT INTO ds1 {"id": 4, "c": [10, 10]};
+
+UPSERT INTO ds1 {"id": 5, "d": [{"x":1}, {"x":"A"}]};
+UPSERT INTO ds1 {"id": 6, "d": [{"x":1}, {"x":"A"}]};
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp
new file mode 100644
index 0000000000..fbe85d356b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.002.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp
new file mode 100644
index 0000000000..fbe85d356b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.003.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i2 ON ds1(UNNEST b:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp
new file mode 100644
index 0000000000..70119b54a6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.004.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i2 ON ds1(UNNEST d select x:string) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp
new file mode 100644
index 0000000000..ee3de29f18
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.005.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE INDEX i4 ON ds1(UNNEST c:int) EXCLUDE UNKNOWN KEY;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
new file mode 100644
index 0000000000..dcf23cf721
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.006.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i4") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp
new file mode 100644
index 0000000000..86a1b59399
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields-bulkload/index-bad-fields-bulkload.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp
new file mode 100644
index 0000000000..e8af202b59
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.000.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test array indexes where some tuples have malformed values in indexed fields
+ * Expected Res : Said tuples are not indexed
+ */
+
+CREATE DATAVERSE test;
+USE test;
+CREATE TYPE dt1 as {id:int};
+CREATE DATASET ds1(dt1) primary key id;
+
+CREATE INDEX i1 ON ds1(UNNEST a : string) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i2 ON ds1(UNNEST b SELECT x : int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i3 ON ds1(UNNEST b.x SELECT p: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i4 ON ds1(UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string, u: int) EXCLUDE UNKNOWN KEY;
+CREATE INDEX i5 ON ds1(p: int, (UNNEST b.c UNNEST d.e UNNEST t SELECT x.y : string, q.w: string), z.m: double) EXCLUDE UNKNOWN KEY;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp
new file mode 100644
index 0000000000..9c2b805b3f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.001.update.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+UPSERT INTO ds1 {"id": 1, "a": [10, "hello", 10, "hello"]};
+UPSERT INTO ds1 {"id": 2, "a": [10]};
+UPSERT INTO ds1 {"id": 3, "a": 94};
+UPSERT INTO ds1 {"id": 4, "a": {"x":1}};
+
+UPSERT INTO ds1 {"id": 5, "a": ["hello"], "b": [{"x":1}, {"x":"bb"}]};
+UPSERT INTO ds1 {"id": 6, "a": ["hello"], "b": [{"x":"aa"}]};
+UPSERT INTO ds1 {"id": 7, "a": 10, "b": [{"x":10}]};
+UPSERT INTO ds1 {"id": 8, "a": ["hello"], "b":100};
+UPSERT INTO ds1 {"id": 9, "b": 100};
+
+UPSERT INTO ds1 {"id": 10, "a": ["hello"], "b":{"x":[{"p":1}, {"k":1}]}};
+
+UPSERT INTO ds1 {"id": 11, "a": ["hello"], "b": {"c":[{"d": {"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":90}]}]}}]}};
+UPSERT INTO ds1 {"id": 12, "a": ["hello"], "b": {"c":[{"d": {"e":[{"t":{"x":{"y":"aab"}, "q":{"w":"10"}, "u":91}}]}}]}};
+UPSERT INTO ds1 {"id": 13, "a": ["hello"], "b": {"c":[{"d": {"e":{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":92}]}}}]}};
+
+UPSERT INTO ds1 {"id": 14, "a": ["hello"], "b":{"c":[{"d":{"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}, "u":93}]}]}}]}, "p":100, "z":{"m":100.10}};
+UPSERT INTO ds1 {"id": 15, "a": ["hello"], "b":{"c":[{"d":{"e":[{"t":[{"x":{"y":"aab"}, "q":{"w":"10"}}]}]}}]}, "p":"kk", "z":{"m":100.10}};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp
new file mode 100644
index 0000000000..e1bb086140
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.002.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT count(*) from ds1;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp
new file mode 100644
index 0000000000..18132bbdd7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.003.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i1") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp
new file mode 100644
index 0000000000..afb1d61013
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.004.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i2") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp
new file mode 100644
index 0000000000..57ad1d9c63
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.005.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i3") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp
new file mode 100644
index 0000000000..dcf23cf721
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.006.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i4") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp
new file mode 100644
index 0000000000..b070cbca8f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.007.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `import-private-functions` `true`;
+FROM DUMP_INDEX("test", "ds1", "i5") AS v
+SELECT VALUE v
+ORDER BY v.values;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp
new file mode 100644
index 0000000000..86a1b59399
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/array-index/index-bad-fields/index-bad-fields.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm
new file mode 100644
index 0000000000..917cc17047
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields-bulkload/index-bad-fields.006.adm
@@ -0,0 +1 @@
+{ "values": [ 10, 4 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm
new file mode 100644
index 0000000000..746d306753
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.002.adm
@@ -0,0 +1 @@
+{ "$1": 15 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm
new file mode 100644
index 0000000000..a4643809ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.003.adm
@@ -0,0 +1,10 @@
+{ "values": [ "hello", 1 ] }
+{ "values": [ "hello", 5 ] }
+{ "values": [ "hello", 6 ] }
+{ "values": [ "hello", 8 ] }
+{ "values": [ "hello", 10 ] }
+{ "values": [ "hello", 11 ] }
+{ "values": [ "hello", 12 ] }
+{ "values": [ "hello", 13 ] }
+{ "values": [ "hello", 14 ] }
+{ "values": [ "hello", 15 ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm
new file mode 100644
index 0000000000..3fdbd81ebe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.004.adm
@@ -0,0 +1,2 @@
+{ "values": [ 1, 5 ] }
+{ "values": [ 10, 7 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm
new file mode 100644
index 0000000000..6045d06eb7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.005.adm
@@ -0,0 +1 @@
+{ "values": [ 1, 10 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm
new file mode 100644
index 0000000000..4c44dbabd7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.006.adm
@@ -0,0 +1,2 @@
+{ "values": [ "aab", "10", 90, 11 ] }
+{ "values": [ "aab", "10", 93, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm
new file mode 100644
index 0000000000..a38bbc602f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/array-index/index-bad-fields/index-bad-fields.007.adm
@@ -0,0 +1 @@
+{ "values": [ 100, "aab", "10", 100.1, 14 ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm
new file mode 100644
index 0000000000..21c2ffaca7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/global-aggregate/q06/q06.3.adm
@@ -0,0 +1,10 @@
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
+{ "count": 0 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm
new file mode 100644
index 0000000000..cc15b264c4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/group-by/sugar-01-negative/core-01.1.adm
@@ -0,0 +1 @@
+{ "avgpay": null, "workers": [ { "name": "Bill", "salary": 2000 }, { "name": "Fred", "salary": 3000 } ], "deptno": "K55" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-2550/query-ASTERIXDB-2886.3.adm
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 0ab9672789..dc31f7e14a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -5801,8 +5801,7 @@
</test-case>
<test-case FilePath="global-aggregate">
<compilation-unit name="q06_error">
- <output-dir compare="Text">q01</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type string (in line 22, at column 8)</expected-error>
+ <output-dir compare="Text">q06</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="global-aggregate">
@@ -5939,8 +5938,7 @@
</test-case>
<test-case FilePath="group-by">
<compilation-unit name="sugar-01-negative">
- <output-dir compare="Text">core-01</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type bigint (in line 26, at column 26)</expected-error>
+ <output-dir compare="Text">sugar-01-negative</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="group-by">
@@ -7164,8 +7162,7 @@
</test-case>
<test-case FilePath="misc">
<compilation-unit name="query-ASTERIXDB-2550">
- <output-dir compare="Text">none</output-dir>
- <expected-error>ASX0037: Type mismatch: expected value of type array or multiset, but got the value of type object (in line 28, at column 2)</expected-error>
+ <output-dir compare="Text">query-ASTERIXDB-2550</output-dir>
</compilation-unit>
</test-case>
<test-case FilePath="misc">
@@ -8359,6 +8356,8 @@
<test-case FilePath="array-index/bulk-loading/on-index-creation/open">
<compilation-unit name="composite-atomic">
<output-dir compare="Text">composite-atomic</output-dir>
+ <expected-error>ASX0001: Field type string cannot be promoted to type bigint</expected-error>
+ <source-location>false</source-location>
</compilation-unit>
</test-case>
</test-group>
@@ -8656,6 +8655,24 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="array-index/index-bad-fields">
+ <test-case FilePath="array-index">
+ <compilation-unit name="index-bad-fields">
+ <output-dir compare="Text">index-bad-fields</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
+ <test-group name="array-index/index-bad-fields-bulkload">
+ <test-case FilePath="array-index">
+ <compilation-unit name="index-bad-fields-bulkload">
+ <output-dir compare="Text">index-bad-fields-bulkload</output-dir>
+ <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error>
+ <expected-error>ASX0018: Cannot convert the string type to the bigint type</expected-error>
+ <expected-error>ASX0001: Field type bigint cannot be promoted to type string</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
+ </test-group>
</test-group>
<test-group name="nestrecords">
<test-case FilePath="nestrecords">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
index 1abf30017f..7332dd0aa0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ArrayIndexUtil.java
@@ -240,8 +240,9 @@ public class ArrayIndexUtil {
* Traverse each distinct record path and invoke the appropriate commands for each scenario. Here, we keep track
* of the record/list type at each step and give this to each command.
*/
- public static void walkArrayPath(Index index, ARecordType baseRecordType, List<String> flattenedFieldName,
- List<Boolean> unnestFlags, TypeTrackerCommandExecutor commandExecutor) throws AlgebricksException {
+ public static void walkArrayPath(Index index, Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ List<String> flattenedFieldName, List<Boolean> unnestFlags, TypeTrackerCommandExecutor commandExecutor)
+ throws AlgebricksException {
ArrayPath arrayPath = new ArrayPath(flattenedFieldName, unnestFlags).invoke();
List<List<String>> fieldNamesPerArray = arrayPath.fieldNamesPerArray;
List<Boolean> unnestFlagsPerArray = arrayPath.unnestFlagsPerArray;
@@ -286,15 +287,15 @@ public class ArrayIndexUtil {
}
}
boolean isFirstArrayStep = i == 0;
- boolean isLastUnnestInIntermediateStep = i < fieldNamesPerArray.size() - 1;
+ boolean isLastUnnestInIntermediateStep = i <= fieldNamesPerArray.size() - 1;
commandExecutor.executeActionOnEachArrayStep(startingStepRecordType, workingType,
fieldNamesPerArray.get(i), isFirstArrayStep, isLastUnnestInIntermediateStep);
}
if (i == fieldNamesPerArray.size() - 1) {
boolean isNonArrayStep = !unnestFlagsPerArray.get(i);
- commandExecutor.executeActionOnFinalArrayStep(startingStepRecordType, fieldNamesPerArray.get(i),
- isNonArrayStep, requiresOnlyOneUnnest);
+ commandExecutor.executeActionOnFinalArrayStep(workingElement, baseRecordType, startingStepRecordType,
+ fieldNamesPerArray.get(i), isNonArrayStep, requiresOnlyOneUnnest);
}
}
}
@@ -341,8 +342,9 @@ public class ArrayIndexUtil {
List<String> fieldName, boolean isFirstArrayStep, boolean isLastUnnestInIntermediateStep)
throws AlgebricksException;
- void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException;
+ void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException;
}
private static class ArrayPath {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
index c8a7ee10d9..af0fd82ed2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryArrayIndexBTreeOperationsHelper.java
@@ -158,7 +158,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
int flattenedListPos = 0;
for (Index.ArrayIndexElement e : arrayIndexDetails.getElementList()) {
for (int i = 0; i < e.getProjectList().size(); i++) {
- addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : itemType, flattenedListPos, false);
+ addSKEvalFactories(isOverridingKeyFieldTypes ? enforcedItemType : itemType, flattenedListPos, false, e);
Pair<IAType, Boolean> keyTypePair = ArrayIndexUtil.getNonNullableOpenFieldType(e.getTypeList().get(i),
e.getUnnestList(), e.getProjectList().get(i), itemType);
IAType keyType = keyTypePair.first;
@@ -203,7 +203,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
ARecordType filterItemType =
((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
: metaType;
- addSKEvalFactories(itemType, numSecondaryKeys, true);
+ addSKEvalFactories(itemType, numSecondaryKeys, true, null);
Pair<IAType, Boolean> keyTypePair;
keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = keyTypePair.first;
@@ -230,8 +230,8 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
return fieldPermutation;
}
- protected void addSKEvalFactories(ARecordType recordType, int fieldPos, boolean isFilterField)
- throws AlgebricksException {
+ protected void addSKEvalFactories(ARecordType recordType, int fieldPos, boolean isFilterField,
+ Index.ArrayIndexElement workingElement) throws AlgebricksException {
if (isFilterField) {
addFilterFieldToBuilder(recordType);
return;
@@ -245,7 +245,8 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
} else {
EvalFactoryAndRecDescInvoker commandExecutor =
new EvalFactoryAndRecDescInvoker(!evalFactoryAndRecDescStackBuilder.isUnnestEvalPopulated());
- ArrayIndexUtil.walkArrayPath(index, recordType, flattenedFieldName, workingUnnestFlags, commandExecutor);
+ ArrayIndexUtil.walkArrayPath(index, workingElement, recordType, flattenedFieldName, workingUnnestFlags,
+ commandExecutor);
}
}
@@ -269,7 +270,7 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
sourceOp = targetOp;
if (arrayIndexDetails.isOverridingKeyFieldTypes() && !enforcedItemType.equals(itemType)) {
// If we have an enforced type, insert a "cast" after the primary index scan.
- targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ targetOp = createCastOp(spec, dataset.getDatasetType(), true);
spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
sourceOp = targetOp;
}
@@ -556,8 +557,9 @@ public class SecondaryArrayIndexBTreeOperationsHelper extends SecondaryTreeIndex
}
@Override
- public void executeActionOnFinalArrayStep(ARecordType startingStepRecordType, List<String> fieldName,
- boolean isNonArrayStep, boolean requiresOnlyOneUnnest) throws AlgebricksException {
+ public void executeActionOnFinalArrayStep(Index.ArrayIndexElement workingElement, ARecordType baseRecordType,
+ ARecordType startingStepRecordType, List<String> fieldName, boolean isNonArrayStep,
+ boolean requiresOnlyOneUnnest) throws AlgebricksException {
// If the final value is nested inside a record, add this SEF.
if (!isNonArrayStep) {
return;
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
index 064c861949..2fd065d539 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/scalar/AbstractScalarAggregateDescriptor.java
@@ -80,7 +80,7 @@ public abstract class AbstractScalarAggregateDescriptor extends AbstractScalarFu
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx) throws HyracksDataException {
// Use ScanCollection to iterate over list items.
ScanCollectionUnnestingFunctionFactory scanCollectionFactory =
- new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc);
+ new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc, getIdentifier());
return createScalarAggregateEvaluator(aggFuncFactory.createAggregateEvaluator(ctx),
scanCollectionFactory, ctx);
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
index ff6dd89bd4..6f9dd5c3b6 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/unnestingfunctions/std/ScanCollectionDescriptor.java
@@ -21,11 +21,11 @@ package org.apache.asterix.runtime.unnestingfunctions.std;
import java.io.IOException;
+import org.apache.asterix.om.exceptions.ExceptionUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.runtime.evaluators.common.ListAccessor;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
import org.apache.asterix.runtime.unnestingfunctions.base.AbstractUnnestingFunctionDynamicDescriptor;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
@@ -52,7 +52,7 @@ public class ScanCollectionDescriptor extends AbstractUnnestingFunctionDynamicDe
@Override
public IUnnestingEvaluatorFactory createUnnestingEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc);
+ return new ScanCollectionUnnestingFunctionFactory(args[0], sourceLoc, getIdentifier());
}
public static class ScanCollectionUnnestingFunctionFactory implements IUnnestingEvaluatorFactory {
@@ -60,10 +60,13 @@ public class ScanCollectionDescriptor extends AbstractUnnestingFunctionDynamicDe
private static final long serialVersionUID = 1L;
private IScalarEvaluatorFactory listEvalFactory;
private final SourceLocation sourceLoc;
+ private final FunctionIdentifier funID;
- public ScanCollectionUnnestingFunctionFactory(IScalarEvaluatorFactory arg, SourceLocation sourceLoc) {
+ public ScanCollectionUnnestingFunctionFactory(IScalarEvaluatorFactory arg, SourceLocation sourceLoc,
+ FunctionIdentifier funID) {
this.listEvalFactory = arg;
this.sourceLoc = sourceLoc;
+ this.funID = funID;
}
@Override
@@ -88,8 +91,9 @@ public class ScanCollectionDescriptor extends AbstractUnnestingFunctionDynamicDe
}
if (typeTag != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
&& typeTag != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- throw new TypeMismatchException(sourceLoc, typeTag, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG,
- ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ ExceptionUtil.warnTypeMismatch(ctx, sourceLoc, funID, typeTag, 0, ATypeTag.MULTISET);
+ metUnknown = true;
+ return;
}
listAccessor.reset(inputVal.getByteArray(), inputVal.getStartOffset());
itemIndex = 0;