You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2025/06/06 18:32:56 UTC
Change in asterixdb[ionic]: WIP: heterogenous index cast
From Ali Alsuliman <al...@gmail.com>:
Ali Alsuliman has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19877 )
Change subject: WIP: heterogenous index cast
......................................................................
WIP: heterogenous index cast
Change-Id: I99f46564bc0879f2a82b0cd7dfbbd95132be7512
---
M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
3 files changed, 81 insertions(+), 9 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/77/19877/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index df8d4cd..33c5baa 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -860,7 +860,7 @@
AbstractFunctionCallExpression theFieldAccessFunc;
LogicalVariable fieldVar = context.newVar();
- if (fieldType == null && !ATypeTag.ANY.equals(skType.getTypeTag())) {
+ if (fieldType == null) {
// Open field. must prevent inlining to maintain the cast before the primaryOp and
// make handling of records with incorrect value type for this field easier and cleaner
context.addNotToBeInlinedVar(fieldVar);
@@ -871,6 +871,13 @@
theFieldAccessFunc = createCastExpression(index, skType, fieldAccessFunc, sourceLoc,
indexFieldId.funId, indexFieldId.extraArg);
} else {
+ // sanity check. (heterogeneous) index keys with type ANY should not be on typed fields
+ ATypeTag skTag = skType.getTypeTag();
+ if (ATypeTag.ANY.equals(skTag)) {
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ "type mismatch for field " + indexFieldId.fieldName + ". secondary key type " + skTag
+ + ", field type " + fieldType.getTypeTag());
+ }
// Get the desired field position
int pos = indexFieldId.fieldName.size() > 1 ? -1
: sourceType.getFieldIndex(indexFieldId.fieldName.get(0));
@@ -903,12 +910,13 @@
private static IndexFieldId createIndexFieldId(Index index, List<String> skName, IAType skType, Integer skSrc,
ARecordType sourceType, SourceLocation srcLoc) throws AlgebricksException {
IAType fieldType = sourceType.getSubFieldType(skName);
- FunctionIdentifier skFun = null;
- IAObject fmtArg = null;
Pair<FunctionIdentifier, IAObject> castExpr;
if (ATypeTag.ANY.equals(skType.getTypeTag())) {
- return new IndexFieldId(skSrc, skName, skType.getTypeTag(), skFun, fmtArg);
+ // heterogeneous index needs to make sure the keys are in the open format (for records & lists values)
+ return new IndexFieldId(skSrc, skName, skType.getTypeTag(), BuiltinFunctions.CAST_TYPE, null);
}
+ FunctionIdentifier skFun = null;
+ IAObject fmtArg = null;
if (fieldType == null) {
// open field
castExpr = getCastExpression(index, skType, srcLoc);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index b4c9484..d82a8be 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -48,8 +48,10 @@
import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.typecomputer.base.TypeCastUtils;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
@@ -86,6 +88,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;
/**
@@ -348,6 +351,7 @@
// we made sure indexSubTree has datasource scan
AbstractDataSourceOperator dataSourceOp =
(AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
+ SourceLocation dataSrcLoc = dataSourceOp.getSourceLocation();
List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex);
int numSecondaryKeys = analysisCtx.getNumberOfMatchedKeys(chosenIndex);
@@ -643,7 +647,7 @@
if (!assignKeyVarList.isEmpty()) {
// Assign operator that sets the constant secondary-index search-key fields if necessary.
AssignOperator assignSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList);
- assignSearchKeys.setSourceLocation(dataSourceOp.getSourceLocation());
+ assignSearchKeys.setSourceLocation(dataSrcLoc);
if (probeSubTree == null) {
// We are optimizing a selection query.
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
@@ -674,6 +678,11 @@
inputOp = probeSubTree.getRoot();
}
+ // if a key is of type ANY, we need to cast the search value to ANY if it is record/list for proper comparison
+ if (chosenIndexKeyFieldTypes.stream().anyMatch(t -> t.getTypeTag() == ATypeTag.ANY)) {
+ inputOp = addCastAssignOp(context, chosenIndexKeyFieldTypes, jobGenParams, inputOp, dataSrcLoc);
+ }
+
// Creates an unnest-map for the secondary index search.
// The result: SK, PK, [Optional - the result of an instantTrylock on PK]
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
@@ -763,7 +772,7 @@
IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH);
UnnestingFunctionCallExpression primaryIndexSearchFunc =
new UnnestingFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs);
- primaryIndexSearchFunc.setSourceLocation(dataSourceOp.getSourceLocation());
+ primaryIndexSearchFunc.setSourceLocation(dataSrcLoc);
primaryIndexSearchFunc.setReturnsUniqueValues(true);
if (!leftOuterUnnestMapRequired) {
unnestMapOp = new UnnestMapOperator(scanVariables, new MutableObject<>(primaryIndexSearchFunc),
@@ -785,7 +794,7 @@
}
}
unnestMapOp.setExecutionMode(ExecutionMode.PARTITIONED);
- unnestMapOp.setSourceLocation(dataSourceOp.getSourceLocation());
+ unnestMapOp.setSourceLocation(dataSrcLoc);
unnestMapOp.getInputs().add(new MutableObject<>(inputOp));
context.computeAndSetTypeEnvironmentForOperator(unnestMapOp);
indexSearchOp = unnestMapOp;
@@ -800,6 +809,51 @@
return indexSearchOp;
}
+ private static ILogicalOperator addCastAssignOp(IOptimizationContext ctx, List<IAType> indexKeysTypes,
+ BTreeJobGenParams jobGenParams, ILogicalOperator inputOp, SourceLocation srcLoc)
+ throws AlgebricksException {
+ // cast the input values (low/high vars) if needed and update the jobGenParams
+ List<LogicalVariable> lowKeyVars = jobGenParams.getLowKeyVarList();
+ List<LogicalVariable> highKeyVars = jobGenParams.getHighKeyVarList();
+ List<LogicalVariable> castAssignVars = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> castAssignExprs = new ArrayList<>();
+ castInputValues(ctx, indexKeysTypes, lowKeyVars, inputOp, castAssignVars, castAssignExprs, srcLoc);
+ castInputValues(ctx, indexKeysTypes, highKeyVars, inputOp, castAssignVars, castAssignExprs, srcLoc);
+ if (castAssignVars.isEmpty()) {
+ return inputOp;
+ }
+ AssignOperator castAssignOp = new AssignOperator(castAssignVars, castAssignExprs);
+ castAssignOp.setSourceLocation(srcLoc);
+ castAssignOp.getInputs().add(new MutableObject<>(inputOp));
+ castAssignOp.setExecutionMode(inputOp.getExecutionMode());
+ return castAssignOp;
+ }
+
+ private static void castInputValues(IOptimizationContext ctx, List<IAType> indexKeyTypes,
+ List<LogicalVariable> inputVars, ILogicalOperator inputOp, List<LogicalVariable> castAssignVars,
+ List<Mutable<ILogicalExpression>> castAssignExprs, SourceLocation srcLoc) throws AlgebricksException {
+ for (int i = 0; i < inputVars.size(); i++) {
+ LogicalVariable inputVar = inputVars.get(i);
+ IVariableTypeEnvironment typeEnv = ctx.getOutputTypeEnvironment(inputOp);
+ IAType varType = (IAType) typeEnv.getVarType(inputVar);
+ IAType varActualType = TypeComputeUtils.getActualType(varType);
+ IAType indexKeyType = indexKeyTypes.get(i);
+ if (varActualType.getTypeTag().isDerivedType() && indexKeyType.getTypeTag() == ATypeTag.ANY) {
+ LogicalVariable newInputVar = ctx.newVar();
+ castAssignVars.add(newInputVar);
+ VariableReferenceExpression newInputVarRef = new VariableReferenceExpression(inputVar);
+ newInputVarRef.setSourceLocation(srcLoc);
+ ScalarFunctionCallExpression castFunc = new ScalarFunctionCallExpression(
+ BuiltinFunctions.getBuiltinFunctionInfo(BuiltinFunctions.CAST_TYPE),
+ new ArrayList<>(Collections.singletonList(new MutableObject<>(newInputVarRef))));
+ castFunc.setSourceLocation(srcLoc);
+ TypeCastUtils.setRequiredAndInputTypes(castFunc, BuiltinType.ANY, varType);
+ castAssignExprs.add(new MutableObject<>(castFunc));
+ inputVars.set(i, newInputVar);
+ }
+ }
+ }
+
private int createKeyVarsAndExprs(int numKeys, LimitType[] keyLimits, ILogicalExpression[] searchKeyExprs,
ArrayList<LogicalVariable> assignKeyVarList, ArrayList<Mutable<ILogicalExpression>> assignKeyExprList,
ArrayList<LogicalVariable> keyVarList, IOptimizationContext context, ILogicalExpression[] constExpressions,
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index cf4f8e0..7e1fd48 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -329,9 +329,10 @@
throws AlgebricksException {
IFunctionManager funManger = metadataProvider.getFunctionManager();
IDataFormat dataFormat = metadataProvider.getDataFormat();
- //if the target type is "BuiltinType.ANY" there is no need to cast. If not we have to cast.
if (ATypeTag.ANY.equals(targetType.getTypeTag())) {
- return fieldEvalFactory;
+ // this is to ensure records and lists values are in the open format
+ IScalarEvaluatorFactory[] castArg = new IScalarEvaluatorFactory[] { fieldEvalFactory };
+ return createCastFunction(targetType, BuiltinType.ANY, true, sourceLoc).createEvaluatorFactory(castArg);
}
// check IndexUtil.castDefaultNull(index), too, because we always want to cast even if the overriding type is
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19877
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: ionic
Gerrit-Change-Id: I99f46564bc0879f2a82b0cd7dfbbd95132be7512
Gerrit-Change-Number: 19877
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <al...@gmail.com>
Gerrit-MessageType: newchange