You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/09/12 21:13:49 UTC
[2/2] asterixdb git commit: ASTERIXDB-1451: Remove Record Casting for
insert/delete/upsert
ASTERIXDB-1451: Remove Record Casting for insert/delete/upsert
This change includes the following:
- Introduce cast function in case of delete operation
after the primary index to ensure types are passed
correctly to enforced indexes.
- Introduce cast function in case of upsert operation
before old secondary keys extraction to ensure types
are passed correctly to enforced indexes.
- Replace all record casts with open field casts for
insert/delete/upsert operations.
- Sonar-Qube fixes.
Change-Id: I6a80105798ea1c86a6a0eb69a79b9573b54931b7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1146
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/534d5892
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/534d5892
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/534d5892
Branch: refs/heads/master
Commit: 534d589297cc4fadbd61b4de70af5c406299814c
Parents: 11a0294
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Mon Sep 12 21:12:36 2016 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Mon Sep 12 14:13:03 2016 -0700
----------------------------------------------------------------------
...IntroduceSecondaryIndexInsertDeleteRule.java | 801 +++++++++----------
.../rules/ReplaceSinkOpWithCommitOpRule.java | 2 +-
.../rules/typecast/StaticTypeCastUtil.java | 3 +-
.../asterix/app/translator/QueryTranslator.java | 112 ++-
.../results/disjunction-to-join-delete-1.plan | 29 +-
.../results/disjunction-to-join-delete-2.plan | 41 +-
.../results/disjunction-to-join-delete-3.plan | 39 +-
.../insert-and-scan-dataset-with-index.plan | 25 +-
.../scan-delete-rtree-secondary-index.plan | 25 +-
.../results/scan-insert-secondary-index.plan | 20 +-
.../skip-ngram-index-search-in-delete.plan | 25 +-
.../skip-rtree-index-search-in-delete.plan | 25 +-
...-secondary-btree-index-search-in-delete.plan | 25 +-
.../skip-word-index-search-in-delete.plan | 25 +-
.../enforced-type-delete.1.ddl.aql | 33 +
.../enforced-type-delete.2.update.aql | 29 +
.../enforced-type-delete.3.ddl.aql | 27 +
.../enforced-type-delete.4.update.aql | 32 +
.../enforced-type-delete.5.query.aql | 29 +
.../enforced-type-upsert.1.ddl.aql | 33 +
.../enforced-type-upsert.2.update.aql | 29 +
.../enforced-type-upsert.3.ddl.aql | 27 +
.../enforced-type-upsert.4.update.aql | 29 +
.../enforced-type-upsert.5.query.aql | 29 +
.../enforced-type-delete.5.adm | 1 +
.../enforced-type-upsert.5.adm | 1 +
.../src/test/resources/runtimets/testsuite.xml | 12 +
.../apache/asterix/metadata/entities/Index.java | 2 +
.../om/typecomputer/base/TypeCastUtils.java | 18 +-
.../om/typecomputer/impl/CastTypeComputer.java | 16 +-
.../IndexInsertDeleteUpsertOperator.java | 5 +-
.../logical/InsertDeleteUpsertOperator.java | 6 +-
.../logical/visitors/UsedVariableVisitor.java | 7 +-
.../LogicalOperatorPrettyPrintVisitor.java | 12 +-
34 files changed, 952 insertions(+), 622 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/534d5892/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index c64258f..c487a96 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -20,10 +20,9 @@ package org.apache.asterix.optimizer.rules;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedHashMap;
+import java.util.HashMap;
import java.util.List;
-import java.util.Stack;
+import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
@@ -32,25 +31,22 @@ import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.AqlDataSource;
import org.apache.asterix.metadata.declared.AqlIndex;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.metadata.declared.DatasetDataSource;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AOrderedList;
import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
import org.apache.asterix.om.typecomputer.base.TypeCastUtils;
import org.apache.asterix.om.types.AOrderedListType;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.asterix.om.util.NonTaggedFormatUtil;
-import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -77,6 +73,12 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOpe
import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+/**
+ * This rule matches the pattern:
+ * assign --> insert-delete-upsert --> sink
+ * and produces
+ * assign --> insert-delete-upsert --> *(secondary indexes index-insert-delete-upsert) --> sink
+ */
public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewriteRule {
@Override
@@ -88,71 +90,38 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
@Override
public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
throws AlgebricksException {
- AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
- if (op0.getOperatorTag() != LogicalOperatorTag.SINK) {
+ AbstractLogicalOperator sinkOp = (AbstractLogicalOperator) opRef.getValue();
+ if (sinkOp.getOperatorTag() != LogicalOperatorTag.SINK) {
return false;
}
- AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getValue();
- if (op1.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) {
+ if (sinkOp.getInputs().get(0).getValue().getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT) {
return false;
}
-
- FunctionIdentifier fid = null;
/** find the record variable */
- InsertDeleteUpsertOperator insertOp = (InsertDeleteUpsertOperator) op1;
- boolean isBulkload = insertOp.isBulkload();
- ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue();
- List<Mutable<ILogicalExpression>> metaExprs = insertOp.getAdditionalNonFilteringExpressions();
- LogicalVariable recordVar = null;
- LogicalVariable metaVar = null;
- List<LogicalVariable> usedRecordVars = new ArrayList<>();
- /** assume the payload is always a single variable expression */
- recordExpr.getUsedVariables(usedRecordVars);
- if (usedRecordVars.size() == 1) {
- recordVar = usedRecordVars.get(0);
- }
- if (metaExprs != null) {
- List<LogicalVariable> metaVars = new ArrayList<>();
- for (Mutable<ILogicalExpression> expr : metaExprs) {
- expr.getValue().getUsedVariables(metaVars);
- }
- if (metaVars.size() > 1) {
- throw new AlgebricksException(
- "Number of meta fields can't be more than 1. Number of meta fields found = " + metaVars.size());
- }
- metaVar = metaVars.get(0);
- }
+ InsertDeleteUpsertOperator primaryIndexModificationOp =
+ (InsertDeleteUpsertOperator) sinkOp.getInputs().get(0).getValue();
+ boolean isBulkload = primaryIndexModificationOp.isBulkload();
+ ILogicalExpression newRecordExpr = primaryIndexModificationOp.getPayloadExpression().getValue();
+ List<Mutable<ILogicalExpression>> newMetaExprs =
+ primaryIndexModificationOp.getAdditionalNonFilteringExpressions();
+ LogicalVariable newRecordVar;
+ LogicalVariable newMetaVar = null;
/**
- * op2 is the assign operator which extracts primary keys from the input
+ * inputOp is the assign operator which extracts primary keys from the input
* variables (record or meta)
*/
- AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ AbstractLogicalOperator inputOp =
+ (AbstractLogicalOperator) primaryIndexModificationOp.getInputs().get(0).getValue();
- if (recordVar == null) {
- /**
- * For the case primary key-assignment expressions are constant
- * expressions, find assign op that creates record to be
- * inserted/deleted.
- */
- while (fid != AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR) {
- if (op2.getInputs().size() == 0) {
- return false;
- }
- op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
- if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
- continue;
- }
- AssignOperator assignOp = (AssignOperator) op2;
- ILogicalExpression assignExpr = assignOp.getExpressions().get(0).getValue();
- if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- ScalarFunctionCallExpression funcExpr =
- (ScalarFunctionCallExpression) assignOp.getExpressions().get(0).getValue();
- fid = funcExpr.getFunctionIdentifier();
- }
+ newRecordVar = getRecordVar(context, inputOp, newRecordExpr, 0);
+ if (newMetaExprs != null && !newMetaExprs.isEmpty()) {
+ if (newMetaExprs.size() > 1) {
+ throw new AlgebricksException(
+ "Number of meta records can't be more than 1. Number of meta records found = "
+ + newMetaExprs.size());
}
- AssignOperator assignOp2 = (AssignOperator) op2;
- recordVar = assignOp2.getVariables().get(0);
+ newMetaVar = getRecordVar(context, inputOp, newMetaExprs.get(0).getValue(), 1);
}
/*
@@ -160,10 +129,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
* Note: We have two operators:
* 1. An InsertDeleteOperator (primary)
* 2. An IndexInsertDeleteOperator (secondary)
- * The current insertOp is of the first type
+ * The current primaryIndexModificationOp is of the first type
*/
- AqlDataSource datasetSource = (AqlDataSource) insertOp.getDataSource();
+ AqlDataSource datasetSource = (AqlDataSource) primaryIndexModificationOp.getDataSource();
AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider();
String dataverseName = datasetSource.getId().getDataverseName();
String datasetName = datasetSource.getId().getDatasourceName();
@@ -175,7 +144,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
return false;
}
- // Create operators for secondary index insert/delete.
+ // Create operators for secondary index insert / delete.
String itemTypeName = dataset.getItemTypeName();
IAType itemType = mp.findType(dataset.getItemTypeDataverseName(), itemTypeName);
if (itemType.getTypeTag() != ATypeTag.RECORD) {
@@ -187,49 +156,35 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
if (dataset.hasMetaPart()) {
metaType = (ARecordType) mp.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
}
- // recType might be replaced with enforced record type and we want to keep a reference to the original record
- // type
- ARecordType originalRecType = recType;
List<Index> indexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
// Set the top operator pointer to the primary IndexInsertDeleteOperator
- ILogicalOperator currentTop = op1;
+ ILogicalOperator currentTop = primaryIndexModificationOp;
boolean hasSecondaryIndex = false;
// Put an n-gram or a keyword index in the later stage of index-update,
// since TokenizeOperator needs to be involved.
- Collections.sort(indexes, new Comparator<Index>() {
- @Override
- public int compare(Index o1, Index o2) {
- return o1.getIndexType().ordinal() - o2.getIndexType().ordinal();
- }
-
- });
-
- // Check whether multiple indexes exist
- int secondaryIndexTotalCnt = 0;
- for (Index index : indexes) {
- if (index.isSecondaryIndex()) {
- secondaryIndexTotalCnt++;
- }
- }
+ Collections.sort(indexes, (o1, o2) -> o1.getIndexType().ordinal() - o2.getIndexType().ordinal());
// At this point, we have the data type info, and the indexes info as well
+ int secondaryIndexTotalCnt = indexes.size() - 1;
if (secondaryIndexTotalCnt > 0) {
- op0.getInputs().clear();
+ sinkOp.getInputs().clear();
+ } else {
+ return false;
}
// Initialize inputs to the SINK operator Op0 (The SINK) is now without input
-
// Prepare filtering field information (This is the filter created using the "filter with" key word in the
// create dataset ddl)
List<String> filteringFields = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField();
- List<LogicalVariable> filteringVars = null;
+ List<LogicalVariable> filteringVars;
List<Mutable<ILogicalExpression>> filteringExpressions = null;
if (filteringFields != null) {
// The filter field var already exists. we can simply get it from the insert op
- filteringVars = new ArrayList<LogicalVariable>();
- filteringExpressions = new ArrayList<Mutable<ILogicalExpression>>();
- for (Mutable<ILogicalExpression> filteringExpression : insertOp.getAdditionalFilteringExpressions()) {
+ filteringVars = new ArrayList<>();
+ filteringExpressions = new ArrayList<>();
+ for (Mutable<ILogicalExpression> filteringExpression : primaryIndexModificationOp
+ .getAdditionalFilteringExpressions()) {
filteringExpression.getValue().getUsedVariables(filteringVars);
for (LogicalVariable var : filteringVars) {
filteringExpressions
@@ -237,51 +192,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
}
}
}
- LogicalVariable enforcedRecordVar = recordVar;
-
- /*
- * if the index is enforcing field types (For open indexes), We add a cast
- * operator to ensure type safety
- */
- if (insertOp.getOperation() == Kind.INSERT || insertOp.getOperation() == Kind.UPSERT) {
- try {
- DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource());
- ARecordType insertRecType = (ARecordType) ds.getItemType();
- // create the expected record type = the original + the optional open field
- ARecordType enforcedType = createEnforcedType(insertRecType, indexes);
- if (!enforcedType.equals(insertRecType)) {
- // A new variable which represents the casted record
- LogicalVariable castedRecVar = context.newVar();
- context.addNotToBeInlinedVar(castedRecVar);
- //introduce casting to enforced type
- AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression(
- FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CAST_TYPE));
- // The first argument is the record
- castFunc.getArguments()
- .add(new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue()));
- TypeCastUtils.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType);
- // AssignOperator puts in the cast var the casted record
- AssignOperator castedRecordAssignOperator =
- new AssignOperator(castedRecVar, new MutableObject<ILogicalExpression>(castFunc));
- // Connect the current top of the plan to the cast operator
- castedRecordAssignOperator.getInputs().addAll(currentTop.getInputs());
- currentTop.getInputs().clear();
- currentTop.getInputs().add(new MutableObject<>(castedRecordAssignOperator));
- enforcedRecordVar = castedRecVar;
- recType = enforcedType;
- context.computeAndSetTypeEnvironmentForOperator(castedRecordAssignOperator);
- context.computeAndSetTypeEnvironmentForOperator(currentTop);
- // We don't need to cast the old rec, we just need an assignment function that extracts the SK
- // and an expression which reference the new variables.
- }
- } catch (AsterixException e) {
- throw new AlgebricksException(e);
- }
- }
// Replicate Operator is applied only when doing the bulk-load.
- AbstractLogicalOperator replicateOp = null;
- if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) {
+ ReplicateOperator replicateOp = null;
+ if (secondaryIndexTotalCnt > 1 && primaryIndexModificationOp.isBulkload()) {
// Split the logical plan into "each secondary index update branch"
// to replicate each <PK,RECORD> pair.
replicateOp = new ReplicateOperator(secondaryIndexTotalCnt);
@@ -291,6 +205,50 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
currentTop = replicateOp;
}
+ /*
+ * The two maps are used to store variables to which [casted] field access is assigned.
+ * One for the beforeOp record and the other for the new record.
+ * There are two uses for these maps:
+ * 1. used for shared fields in indexes with overlapping keys.
+ * 2. used for setting variables of secondary keys for each secondary index operator.
+ */
+ Map<IndexFieldId, LogicalVariable> fieldVarsForBeforeOperation = new HashMap<>();
+ Map<IndexFieldId, LogicalVariable> fieldVarsForNewRecord = new HashMap<>();
+ /*
+ * if the index is enforcing field types (For open indexes), We add a cast
+ * operator to ensure type safety
+ */
+ try {
+ if (primaryIndexModificationOp.getOperation() == Kind.INSERT
+ || primaryIndexModificationOp.getOperation() == Kind.UPSERT
+ /* Actually, delete should not be here but it is now until issue
+ * https://issues.apache.org/jira/browse/ASTERIXDB-1507
+ * is solved
+ */
+ || primaryIndexModificationOp.getOperation() == Kind.DELETE) {
+ injectFieldAccessesForIndexes(context, dataset, indexes, fieldVarsForNewRecord, recType,
+ metaType, newRecordVar, newMetaVar, primaryIndexModificationOp, false);
+ if (replicateOp != null) {
+ context.computeAndSetTypeEnvironmentForOperator(replicateOp);
+ }
+ }
+ if (primaryIndexModificationOp.getOperation() == Kind.UPSERT
+ /* Actually, delete should be here but it is not until issue
+ * https://issues.apache.org/jira/browse/ASTERIXDB-1507
+ * is solved
+ */) {
+ List<LogicalVariable> beforeOpMetaVars =
+ primaryIndexModificationOp.getBeforeOpAdditionalNonFilteringVars();
+ LogicalVariable beforeOpMetaVar = beforeOpMetaVars == null ? null : beforeOpMetaVars.get(0);
+ currentTop =
+ injectFieldAccessesForIndexes(context, dataset, indexes, fieldVarsForBeforeOperation, recType,
+ metaType, primaryIndexModificationOp.getBeforeOpRecordVar(), beforeOpMetaVar,
+ currentTop, true);
+ }
+ } catch (AsterixException e) {
+ throw new AlgebricksException(e);
+ }
+
// Iterate each secondary index and applying Index Update operations.
// At first, op1 is the index insert op insertOp
for (Index index : indexes) {
@@ -300,88 +258,36 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
hasSecondaryIndex = true;
// Get the secondary fields names and types
List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
- List<IAType> secondaryKeyTypes = index.getKeyFieldTypes();
- List<LogicalVariable> secondaryKeyVars = new ArrayList<LogicalVariable>();
- List<Integer> indicators = index.getKeyFieldSourceIndicators();
- List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
- List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ List<LogicalVariable> secondaryKeyVars = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> beforeOpSecondaryExpressions = new ArrayList<>();
+ ILogicalOperator replicateOutput;
for (int i = 0; i < secondaryKeyFields.size(); i++) {
- List<String> secondaryKey = secondaryKeyFields.get(i);
- ARecordType sourceType = recType;
- LogicalVariable sourceVar = enforcedRecordVar;
- if (dataset.hasMetaPart()) {
- sourceType = indicators.get(i).intValue() == 0 ? recType : metaType;
- sourceVar = indicators.get(i).intValue() == 0 ? enforcedRecordVar : metaVar;
+ IndexFieldId indexFieldId =
+ new IndexFieldId(index.getKeyFieldSourceIndicators().get(i), secondaryKeyFields.get(i));
+ LogicalVariable skVar = fieldVarsForNewRecord.get(indexFieldId);
+ secondaryKeyVars.add(skVar);
+ secondaryExpressions.add(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(skVar)));
+ if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
+ beforeOpSecondaryExpressions.add(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(fieldVarsForBeforeOperation.get(indexFieldId))));
}
- prepareVarAndExpression(secondaryKey, sourceType.getFieldNames(), sourceVar, expressions,
- secondaryKeyVars, context);
- }
- // Used with upsert operation
- // in case of upsert, we need vars and expressions for the old SK as well.
- List<LogicalVariable> prevSecondaryKeyVars = null;
- List<Mutable<ILogicalExpression>> prevExpressions = null;
- List<Mutable<ILogicalExpression>> prevSecondaryExpressions = null;
- AssignOperator prevSecondaryKeyAssign = null;
- if (insertOp.getOperation() == Kind.UPSERT) {
- prevSecondaryKeyVars = new ArrayList<LogicalVariable>();
- prevExpressions = new ArrayList<Mutable<ILogicalExpression>>();
- prevSecondaryExpressions = new ArrayList<Mutable<ILogicalExpression>>();
- for (int i = 0; i < secondaryKeyFields.size(); i++) {
- List<String> secondaryKey = secondaryKeyFields.get(i);
- prepareVarAndExpression(secondaryKey,
- (indicators.get(i).intValue() == 0) ? originalRecType.getFieldNames()
- : metaType.getFieldNames(),
- (indicators.get(i).intValue() == 0) ? insertOp.getPrevRecordVar()
- : insertOp.getPrevAdditionalNonFilteringVars().get(0),
- prevExpressions, prevSecondaryKeyVars, context);
- }
- prevSecondaryKeyAssign = new AssignOperator(prevSecondaryKeyVars, prevExpressions);
- }
- AssignOperator assign = new AssignOperator(secondaryKeyVars, expressions);
- AssignOperator topAssign = assign;
- if (insertOp.getOperation() == Kind.UPSERT) {
- prevSecondaryKeyAssign.getInputs().add(new MutableObject<ILogicalOperator>(topAssign));
- topAssign = prevSecondaryKeyAssign;
- }
- // Only apply replicate operator when doing bulk-load
- if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) {
- assign.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp));
- } else {
- assign.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
}
- context.computeAndSetTypeEnvironmentForOperator(assign);
- if (insertOp.getOperation() == Kind.UPSERT) {
- context.computeAndSetTypeEnvironmentForOperator(prevSecondaryKeyAssign);
- }
- currentTop = topAssign;
-
- // in case of an Upsert operation, the currentTop is an assign which has the old secondary keys + the new secondary keys
- if (index.getIndexType() == IndexType.BTREE || index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
- || index.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX
- || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
+ IndexInsertDeleteUpsertOperator indexUpdate;
+ if (index.getIndexType() != IndexType.RTREE) {
// Create an expression per key
- for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
- secondaryExpressions.add(
- new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
- }
- Mutable<ILogicalExpression> filterExpression = null;
- if (insertOp.getOperation() == Kind.UPSERT) {
- for (LogicalVariable oldSecondaryKeyVar : prevSecondaryKeyVars) {
- prevSecondaryExpressions.add(new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(oldSecondaryKeyVar)));
- }
- } else {
- filterExpression = createFilterExpression(secondaryKeyVars,
- context.getOutputTypeEnvironment(currentTop), false);
- }
+ Mutable<ILogicalExpression> filterExpression =
+ (primaryIndexModificationOp.getOperation() == Kind.UPSERT) ? null
+ : createFilterExpression(secondaryKeyVars,
+ context.getOutputTypeEnvironment(currentTop), index.isEnforcingKeyFileds());
AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
// Introduce the TokenizeOperator only when doing bulk-load,
// and index type is keyword or n-gram.
- if (index.getIndexType() != IndexType.BTREE && insertOp.isBulkload()) {
+ if (index.getIndexType() != IndexType.BTREE && primaryIndexModificationOp.isBulkload()) {
// Note: Bulk load case, we don't need to take care of it for upsert operation
// Check whether the index is length-partitioned or not.
// If partitioned, [input variables to TokenizeOperator,
@@ -391,27 +297,25 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
// and fed into the IndexInsertDeleteOperator.
// Input variables are passed since TokenizeOperator is not an
// filtering operator.
- boolean isPartitioned = false;
- if (index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
- isPartitioned = true;
- }
+ boolean isPartitioned = index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
// Create a new logical variable - token
- List<LogicalVariable> tokenizeKeyVars = new ArrayList<LogicalVariable>();
- List<Mutable<ILogicalExpression>> tokenizeKeyExprs = new ArrayList<Mutable<ILogicalExpression>>();
+ List<LogicalVariable> tokenizeKeyVars = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> tokenizeKeyExprs = new ArrayList<>();
LogicalVariable tokenVar = context.newVar();
tokenizeKeyVars.add(tokenVar);
tokenizeKeyExprs
.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(tokenVar)));
// Check the field type of the secondary key.
- IAType secondaryKeyType = null;
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0), recType);
+ IAType secondaryKeyType;
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(
+ index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0),
+ recType);
secondaryKeyType = keyPairType.first;
- List<Object> varTypes = new ArrayList<Object>();
+ List<Object> varTypes = new ArrayList<>();
varTypes.add(NonTaggedFormatUtil.getTokenType(secondaryKeyType));
// If the index is a length-partitioned, then create
@@ -428,64 +332,53 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
// TokenizeOperator to tokenize [SK, PK] pairs
TokenizeOperator tokenUpdate = new TokenizeOperator(dataSourceIndex,
- insertOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars,
- filterExpression, insertOp.getOperation(), insertOp.isBulkload(), isPartitioned, varTypes);
- tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+ primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions,
+ tokenizeKeyVars,
+ filterExpression, primaryIndexModificationOp.getOperation(),
+ primaryIndexModificationOp.isBulkload(), isPartitioned, varTypes);
+ tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
-
- IndexInsertDeleteUpsertOperator indexUpdate =
- new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
- tokenizeKeyExprs, filterExpression, insertOp.getOperation(), insertOp.isBulkload(),
- insertOp.getAdditionalNonFilteringExpressions() == null ? 0
- : insertOp.getAdditionalNonFilteringExpressions().size());
+ replicateOutput = tokenUpdate;
+ indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
+ primaryIndexModificationOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
+ primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(),
+ primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0
+ : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
-
- context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
-
- currentTop = indexUpdate;
- op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
-
} else {
// When TokenizeOperator is not needed
- IndexInsertDeleteUpsertOperator indexUpdate =
- new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
- secondaryExpressions, filterExpression, insertOp.getOperation(),
- insertOp.isBulkload(), insertOp.getAdditionalNonFilteringExpressions() == null ? 0
- : insertOp.getAdditionalNonFilteringExpressions().size());
-
+ indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
+ primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions,
+ filterExpression,
+ primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(),
+ primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0
+ : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
+ replicateOutput = indexUpdate;
// We add the necessary expressions for upsert
- if (insertOp.getOperation() == Kind.UPSERT) {
- indexUpdate.setPrevSecondaryKeyExprs(prevSecondaryExpressions);
+ if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
+ indexUpdate.setBeforeOpSecondaryKeyExprs(beforeOpSecondaryExpressions);
if (filteringFields != null) {
- indexUpdate.setPrevAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(insertOp.getPrevFilterVar())));
+ indexUpdate.setBeforeOpAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(
+ primaryIndexModificationOp.getBeforeOpFilterVar())));
}
}
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
-
- currentTop = indexUpdate;
- context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
-
- if (insertOp.isBulkload()) {
- op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
- }
-
}
-
- } else if (index.getIndexType() == IndexType.RTREE) {
+ } else {
// Get type, dimensions and number of keys
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyFields.get(0), recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
+ secondaryKeyFields.get(0), recType);
IAType spatialType = keyPairType.first;
- boolean isPointMBR =
- spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
+ boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
+ || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numKeys = (isPointMBR && isBulkload) ? dimension : dimension * 2;
// Get variables and expressions
- List<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
- List<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
+ List<LogicalVariable> keyVarList = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
for (int i = 0; i < numKeys; i++) {
LogicalVariable keyVar = context.newVar();
keyVarList.add(keyVar);
@@ -499,6 +392,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
}
+ secondaryExpressions.clear();
for (LogicalVariable secondaryKeyVar : keyVarList) {
secondaryExpressions.add(
new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
@@ -514,11 +408,12 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
AssignOperator assignCoordinates = new AssignOperator(keyVarList, keyExprList);
assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
context.computeAndSetTypeEnvironmentForOperator(assignCoordinates);
+ replicateOutput = assignCoordinates;
Mutable<ILogicalExpression> filterExpression = null;
AssignOperator originalAssignCoordinates = null;
- // We do something similar for previous key if the operation is an upsert
- if (insertOp.getOperation() == Kind.UPSERT) {
- List<LogicalVariable> originalKeyVarList = new ArrayList<LogicalVariable>();
+ // We do something similar for beforeOp key if the operation is an upsert
+ if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
+ List<LogicalVariable> originalKeyVarList = new ArrayList<>();
List<Mutable<ILogicalExpression>> originalKeyExprList = new ArrayList<>();
// we don't do any filtering since nulls are expected here and there
for (int i = 0; i < numKeys; i++) {
@@ -526,26 +421,18 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
originalKeyVarList.add(keyVar);
AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CREATE_MBR));
- createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(prevSecondaryKeyVars.get(0))));
+ createMBR.getArguments().add(beforeOpSecondaryExpressions.get(0));
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
new ConstantExpression(new AsterixConstantValue(new AInt32(dimension)))));
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
originalKeyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
}
+ beforeOpSecondaryExpressions.clear();
for (LogicalVariable secondaryKeyVar : originalKeyVarList) {
- prevSecondaryExpressions.add(new MutableObject<ILogicalExpression>(
+ beforeOpSecondaryExpressions.add(new MutableObject<ILogicalExpression>(
new VariableReferenceExpression(secondaryKeyVar)));
}
- if (isPointMBR && isBulkload) {
- //for PointMBR optimization: see SecondaryRTreeOperationsHelper.buildLoadingJobSpec() and
- //createFieldPermutationForBulkLoadOp(int) for more details.
- for (LogicalVariable secondaryKeyVar : originalKeyVarList) {
- prevSecondaryExpressions.add(new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(secondaryKeyVar)));
- }
- }
originalAssignCoordinates = new AssignOperator(originalKeyVarList, originalKeyExprList);
originalAssignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
context.computeAndSetTypeEnvironmentForOperator(originalAssignCoordinates);
@@ -557,33 +444,35 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
}
AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
- IndexInsertDeleteUpsertOperator indexUpdate =
- new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
- secondaryExpressions, filterExpression, insertOp.getOperation(), insertOp.isBulkload(),
- insertOp.getAdditionalNonFilteringExpressions() == null ? 0
- : insertOp.getAdditionalNonFilteringExpressions().size());
+ indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
+ primaryIndexModificationOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
+ primaryIndexModificationOp.getOperation(), primaryIndexModificationOp.isBulkload(),
+ primaryIndexModificationOp.getAdditionalNonFilteringExpressions() == null ? 0
+ : primaryIndexModificationOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
- if (insertOp.getOperation() == Kind.UPSERT) {
- // set old secondary key expressions
+ if (primaryIndexModificationOp.getOperation() == Kind.UPSERT) {
+ // set before op secondary key expressions
if (filteringFields != null) {
- indexUpdate.setPrevAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(insertOp.getPrevFilterVar())));
+ indexUpdate.setBeforeOpAdditionalFilteringExpression(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(primaryIndexModificationOp.getBeforeOpFilterVar())));
}
// set filtering expressions
- indexUpdate.setPrevSecondaryKeyExprs(prevSecondaryExpressions);
- // assign --> assign previous values --> secondary index upsert
+ indexUpdate.setBeforeOpSecondaryKeyExprs(beforeOpSecondaryExpressions);
+ // assign --> assign beforeOp values --> secondary index upsert
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(originalAssignCoordinates));
} else {
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
}
+ }
+ context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
+ if (!primaryIndexModificationOp.isBulkload() || secondaryIndexTotalCnt == 1) {
currentTop = indexUpdate;
- context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
-
- if (insertOp.isBulkload()) {
- // For bulk load, we connect all fanned out insert operator to a single SINK operator
- op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
- }
-
+ } else {
+ replicateOp.getOutputs().add(new MutableObject<>(replicateOutput));
+ }
+ if (primaryIndexModificationOp.isBulkload()) {
+ // For bulk load, we connect all fanned out insert operator to a single SINK operator
+ sinkOp.getInputs().add(new MutableObject<ILogicalOperator>(indexUpdate));
}
}
@@ -591,183 +480,181 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
return false;
}
- if (!insertOp.isBulkload()) {
+ if (!primaryIndexModificationOp.isBulkload()) {
// If this is an upsert, we need to
// Remove the current input to the SINK operator (It is actually already removed above)
- op0.getInputs().clear();
+ sinkOp.getInputs().clear();
// Connect the last index update to the SINK
- op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+ sinkOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
}
return true;
}
- // Merges typed index fields with specified recordType, allowing indexed fields to be optional.
- // I.e. the type { "personId":int32, "name": string, "address" : { "street": string } } with typed indexes on age:int32, address.state:string
- // will be merged into type { "personId":int32, "name": string, "age": int32? "address" : { "street": string, "state": string? } }
- // Used by open indexes to enforce the type of an indexed record
- public static ARecordType createEnforcedType(ARecordType initialType, List<Index> indexes)
- throws AsterixException, AlgebricksException {
- ARecordType enforcedType = initialType;
+ private LogicalVariable getRecordVar(IOptimizationContext context, AbstractLogicalOperator inputOp,
+ ILogicalExpression recordExpr,
+ int expectedRecordIndex) throws AlgebricksException {
+ if (exprIsRecord(context.getOutputTypeEnvironment(inputOp), recordExpr)) {
+ return ((VariableReferenceExpression) recordExpr).getVariableReference();
+ } else {
+ /**
+ * For the case primary key-assignment expressions are constant
+ * expressions, find assign op that creates record to be
+ * inserted/deleted.
+ */
+ FunctionIdentifier fid = null;
+ AbstractLogicalOperator currentInputOp = inputOp;
+ while (fid != AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR) {
+ if (currentInputOp.getInputs().isEmpty()) {
+ return null;
+ }
+ currentInputOp = (AbstractLogicalOperator) currentInputOp.getInputs().get(0).getValue();
+ if (currentInputOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ continue;
+ }
+ AssignOperator assignOp = (AssignOperator) currentInputOp;
+ ILogicalExpression assignExpr = assignOp.getExpressions().get(expectedRecordIndex).getValue();
+ if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ ScalarFunctionCallExpression funcExpr = (ScalarFunctionCallExpression) assignOp.getExpressions()
+ .get(expectedRecordIndex).getValue();
+ fid = funcExpr.getFunctionIdentifier();
+ }
+ }
+ return ((AssignOperator) currentInputOp).getVariables().get(0);
+ }
+ }
+
+ private boolean exprIsRecord(IVariableTypeEnvironment typeEnvironment, ILogicalExpression recordExpr)
+ throws AlgebricksException {
+ if (recordExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ IAType type = (IAType) typeEnvironment.getType(recordExpr);
+ return type != null && type.getTypeTag() == ATypeTag.RECORD;
+ }
+ return false;
+ }
+
+ private ILogicalOperator injectFieldAccessesForIndexes(IOptimizationContext context, Dataset dataset,
+ List<Index> indexes, Map<IndexFieldId, LogicalVariable> fieldAccessVars, ARecordType recType,
+ ARecordType metaType, LogicalVariable recordVar, LogicalVariable metaVar, ILogicalOperator currentTop,
+ boolean afterOp) throws AlgebricksException {
+ List<LogicalVariable> vars = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
for (Index index : indexes) {
- if (!index.isSecondaryIndex() || !index.isEnforcingKeyFileds()) {
+ if (index.isPrimaryIndex()) {
continue;
}
- if (index.hasMetaFields()) {
- throw new AlgebricksException("Indexing an open field is only supported on the record part");
- }
+ List<IAType> skTypes = index.getKeyFieldTypes();
+ List<List<String>> skNames = index.getKeyFieldNames();
+ List<Integer> indicators = index.getKeyFieldSourceIndicators();
for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
- Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>();
- List<String> splits = index.getKeyFieldNames().get(i);
- ARecordType nestedFieldType = enforcedType;
- boolean openRecords = false;
- String bridgeName = nestedFieldType.getTypeName();
- int j;
- // Build the stack for the enforced type
- for (j = 1; j < splits.size(); j++) {
- nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
- bridgeName = nestedFieldType.getTypeName();
- nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
- if (nestedFieldType == null) {
- openRecords = true;
- break;
- }
+ IndexFieldId indexFieldId = new IndexFieldId(indicators.get(i), skNames.get(i));
+ if (fieldAccessVars.containsKey(indexFieldId)) {
+ // already handled in a different index
+ continue;
}
- if (openRecords == true) {
- // create the smallest record
- enforcedType = new ARecordType(splits.get(splits.size() - 2),
- new String[] { splits.get(splits.size() - 1) },
- new IAType[] { AUnionType.createUnknownableType(index.getKeyFieldTypes().get(i)) }, true);
- // create the open part of the nested field
- for (int k = splits.size() - 3; k > (j - 2); k--) {
- enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
- new IAType[] { AUnionType.createUnknownableType(enforcedType) }, true);
- }
- // Bridge the gap
- Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
- ARecordType parent = gapPair.first;
-
- IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
- new IAType[] { AUnionType.createUnknownableType(enforcedType) });
- enforcedType = new ARecordType(bridgeName,
- ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes,
- true);
+ ARecordType sourceType = dataset.hasMetaPart()
+ ? indicators.get(i).intValue() == Index.RECORD_INDICATOR ? recType : metaType : recType;
+ LogicalVariable sourceVar = dataset.hasMetaPart()
+ ? indicators.get(i).intValue() == Index.RECORD_INDICATOR ? recordVar : metaVar
+ : recordVar;
+ LogicalVariable fieldVar = context.newVar();
+ // create record variable ref
+ Mutable<ILogicalExpression> varRef =
+ new MutableObject<>(new VariableReferenceExpression(sourceVar));
+ IAType fieldType = sourceType.getSubFieldType(indexFieldId.fieldName);
+ AbstractFunctionCallExpression theFieldAccessFunc;
+ if (fieldType == null) {
+ // Open field. must prevent inlining to maintain the cast before the primaryOp and
+ // make handling of records with incorrect value type for this field easier and cleaner
+ context.addNotToBeInlinedVar(fieldVar);
+ // create field access
+ AbstractFunctionCallExpression fieldAccessFunc =
+ getOpenOrNestedFieldAccessFunction(varRef, indexFieldId.fieldName);
+ // create cast
+ theFieldAccessFunc = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CAST_TYPE));
+ // The first argument is the field
+ theFieldAccessFunc.getArguments()
+ .add(new MutableObject<ILogicalExpression>(fieldAccessFunc));
+ TypeCastUtils.setRequiredAndInputTypes(theFieldAccessFunc, skTypes.get(i),
+ BuiltinType.ANY);
} else {
- //Schema is closed all the way to the field
- //enforced fields are either null or strongly typed
- LinkedHashMap<String, IAType> recordNameTypesMap = createRecordNameTypeMap(nestedFieldType);
- // if a an enforced field already exists and the type is correct
- IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size() - 1));
- if (enforcedFieldType != null && enforcedFieldType.getTypeTag() == ATypeTag.UNION
- && ((AUnionType) enforcedFieldType).isUnknownableType()) {
- enforcedFieldType = ((AUnionType) enforcedFieldType).getActualType();
- }
- if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(),
- index.getKeyFieldTypes().get(i).getTypeTag())) {
- throw new AlgebricksException("Cannot enforce field " + index.getKeyFieldNames().get(i)
- + " to have type " + index.getKeyFieldTypes().get(i));
- }
- if (enforcedFieldType == null) {
- recordNameTypesMap.put(splits.get(splits.size() - 1),
- AUnionType.createUnknownableType(index.getKeyFieldTypes().get(i)));
- }
- enforcedType = new ARecordType(nestedFieldType.getTypeName(),
- recordNameTypesMap.keySet().toArray(new String[recordNameTypesMap.size()]),
- recordNameTypesMap.values().toArray(new IAType[recordNameTypesMap.size()]),
- nestedFieldType.isOpen());
- }
-
- // Create the enforcedtype for the nested fields in the schema, from the ground up
- if (nestedTypeStack.size() > 0) {
- while (!nestedTypeStack.isEmpty()) {
- Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
- ARecordType nestedRecType = nestedTypePair.first;
- IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
- nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedType;
- enforcedType = new ARecordType(nestedRecType.getTypeName() + "_enforced",
- nestedRecType.getFieldNames(), nestedRecTypeFieldTypes, nestedRecType.isOpen());
- }
+ // Get the desired field position
+ int pos = indexFieldId.fieldName.size() > 1 ? -1
+ : sourceType.getFieldIndex(indexFieldId.fieldName.get(0));
+ // Field not found --> This is either an open field or a nested field. it can't be accessed by index
+ theFieldAccessFunc =
+ (pos == -1) ? getOpenOrNestedFieldAccessFunction(varRef, indexFieldId.fieldName)
+ : getClosedFieldAccessFunction(varRef, pos);
}
+ vars.add(fieldVar);
+ exprs.add(new MutableObject<ILogicalExpression>(theFieldAccessFunc));
+ fieldAccessVars.put(indexFieldId, fieldVar);
}
}
- return enforcedType;
+ // AssignOperator assigns secondary keys to their vars
+ AssignOperator castedFieldAssignOperator = new AssignOperator(vars, exprs);
+ return introduceNewOp(context, currentTop, castedFieldAssignOperator, afterOp);
}
- private static LinkedHashMap<String, IAType> createRecordNameTypeMap(ARecordType nestedFieldType) {
- LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<>();
- for (int j = 0; j < nestedFieldType.getFieldNames().length; j++) {
- recordNameTypesMap.put(nestedFieldType.getFieldNames()[j], nestedFieldType.getFieldTypes()[j]);
+ private static ILogicalOperator introduceNewOp(IOptimizationContext context, ILogicalOperator currentTopOp,
+ ILogicalOperator newOp, boolean afterOp) throws AlgebricksException {
+ if (afterOp) {
+ newOp.getInputs().add(new MutableObject<>(currentTopOp));
+ context.computeAndSetTypeEnvironmentForOperator(newOp);
+ return newOp;
+ } else {
+ newOp.getInputs().addAll(currentTopOp.getInputs());
+ currentTopOp.getInputs().clear();
+ currentTopOp.getInputs().add(new MutableObject<>(newOp));
+ context.computeAndSetTypeEnvironmentForOperator(newOp);
+ context.computeAndSetTypeEnvironmentForOperator(currentTopOp);
+ return currentTopOp;
}
- return recordNameTypesMap;
}
- /***
- * This method takes a list of {fields}: a subset of {recordFields}, the original record variable
- * and populate expressions with expressions which evaluate to those fields (using field access functions) and
- * variables to represent them
- *
- * @param fields
- * desired fields
- * @param recordFields
- * all the record fields
- * @param recordVar
- * the record variable
- * @param expressions
- * @param vars
- * @param context
- * @throws AlgebricksException
- */
- @SuppressWarnings("unchecked")
- private void prepareVarAndExpression(List<String> fields, String[] recordFields, LogicalVariable recordVar,
- List<Mutable<ILogicalExpression>> expressions, List<LogicalVariable> vars, IOptimizationContext context)
- throws AlgebricksException {
- // Get a reference to the record variable
- Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(recordVar));
- // Get the desired field position
- int pos = -1;
- if (fields.size() == 1) {
- for (int j = 0; j < recordFields.length; j++) {
- if (recordFields[j].equals(fields.get(0))) {
- pos = j;
- break;
- }
- }
- }
- // Field not found --> This is either an open field or a nested field. it can't be accessed by index
- AbstractFunctionCallExpression func;
- if (pos == -1) {
- if (fields.size() > 1) {
- AOrderedList fieldList = new AOrderedList(new AOrderedListType(BuiltinType.ASTRING, null));
- for (int i = 0; i < fields.size(); i++) {
- fieldList.add(new AString(fields.get(i)));
- }
- Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(
- new ConstantExpression(new AsterixConstantValue(fieldList)));
- // Create an expression for the nested case
- func = new ScalarFunctionCallExpression(
- FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED), varRef, fieldRef);
- } else {
- Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(
- new ConstantExpression(new AsterixConstantValue(new AString(fields.get(0)))));
- // Create an expression for the open field case (By name)
- func = new ScalarFunctionCallExpression(
- FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME), varRef, fieldRef);
- }
+ private static AbstractFunctionCallExpression getClosedFieldAccessFunction(Mutable<ILogicalExpression> varRef,
+ int position) {
+ Mutable<ILogicalExpression> indexRef = new MutableObject<>(
+ new ConstantExpression(new AsterixConstantValue(new AInt32(position))));
+ return new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX), varRef, indexRef);
+ }
+
+ private static AbstractFunctionCallExpression getOpenOrNestedFieldAccessFunction(Mutable<ILogicalExpression> varRef,
+ List<String> fields) {
+ ScalarFunctionCallExpression func;
+ if (fields.size() > 1) {
+ IAObject fieldList = stringListToAOrderedList(fields);
+ Mutable<ILogicalExpression> fieldRef = constantToMutableLogicalExpression(fieldList);
+ // Create an expression for the nested case
+ func = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED), varRef, fieldRef);
} else {
- // Assumes the indexed field is in the closed portion of the type.
- Mutable<ILogicalExpression> indexRef = new MutableObject<ILogicalExpression>(
- new ConstantExpression(new AsterixConstantValue(new AInt32(pos))));
+ IAObject fieldList = new AString(fields.get(0));
+ Mutable<ILogicalExpression> fieldRef = constantToMutableLogicalExpression(fieldList);
+ // Create an expression for the open field case (By name)
func = new ScalarFunctionCallExpression(
- FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX), varRef, indexRef);
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME), varRef, fieldRef);
+ }
+ return func;
+ }
+
+ private static AOrderedList stringListToAOrderedList(List<String> fields) {
+ AOrderedList fieldList = new AOrderedList(new AOrderedListType(BuiltinType.ASTRING, null));
+ for (int i = 0; i < fields.size(); i++) {
+ fieldList.add(new AString(fields.get(i)));
}
- expressions.add(new MutableObject<ILogicalExpression>(func));
- LogicalVariable newVar = context.newVar();
- vars.add(newVar);
+ return fieldList;
+ }
+
+ private static Mutable<ILogicalExpression> constantToMutableLogicalExpression(IAObject constantObject) {
+ return new MutableObject<>(
+ new ConstantExpression(new AsterixConstantValue(constantObject)));
}
- @SuppressWarnings("unchecked")
private Mutable<ILogicalExpression> createFilterExpression(List<LogicalVariable> secondaryKeyVars,
IVariableTypeEnvironment typeEnv, boolean forceFilter) throws AlgebricksException {
- List<Mutable<ILogicalExpression>> filterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ List<Mutable<ILogicalExpression>> filterExpressions = new ArrayList<>();
// Add 'is not null' to all nullable secondary index keys as a filtering
// condition.
for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
@@ -775,26 +662,50 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
if (!NonTaggedFormatUtil.isOptional(secondaryKeyType) && !forceFilter) {
continue;
}
- ScalarFunctionCallExpression isUnknownFuncExpr =
- new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN),
- new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
- ScalarFunctionCallExpression notFuncExpr =
- new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT),
- new MutableObject<ILogicalExpression>(isUnknownFuncExpr));
+ ScalarFunctionCallExpression isUnknownFuncExpr = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN),
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
+ ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT),
+ new MutableObject<ILogicalExpression>(isUnknownFuncExpr));
filterExpressions.add(new MutableObject<ILogicalExpression>(notFuncExpr));
}
// No nullable secondary keys.
if (filterExpressions.isEmpty()) {
return null;
}
- Mutable<ILogicalExpression> filterExpression = null;
+ Mutable<ILogicalExpression> filterExpression;
if (filterExpressions.size() > 1) {
// Create a conjunctive condition.
- filterExpression = new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(
+ filterExpression = new MutableObject<>(new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.AND), filterExpressions));
} else {
filterExpression = filterExpressions.get(0);
}
return filterExpression;
}
+
+ private class IndexFieldId {
+ private int indicator;
+ private List<String> fieldName;
+
+ public IndexFieldId(int indicator, List<String> fieldName) {
+ this.indicator = indicator;
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public int hashCode() {
+ return 31 * indicator + fieldName.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof IndexFieldId) {
+ IndexFieldId oIndexFieldId = (IndexFieldId) o;
+ return indicator == oIndexFieldId.indicator && fieldName.equals(oIndexFieldId.fieldName);
+ }
+ return false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/534d5892/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
index 8aefd1a..0c36d0b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ReplaceSinkOpWithCommitOpRule.java
@@ -110,7 +110,7 @@ public class ReplaceSinkOpWithCommitOpRule implements IAlgebraicRewriteRule {
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_MISSING));
// argument is the previous record
isPrevMissingFunc.getArguments().add(new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(insertDeleteUpsertOperator.getPrevRecordVar())));
+ new VariableReferenceExpression(insertDeleteUpsertOperator.getBeforeOpRecordVar())));
orFunc.getArguments().add(new MutableObject<ILogicalExpression>(isPrevMissingFunc));
orFunc.getArguments().add(new MutableObject<ILogicalExpression>(isNewMissingFunc));
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/534d5892/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
index eac35cd..ba79534 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
@@ -506,9 +506,10 @@ public class StaticTypeCastUtil {
* the expression reference
* @param argExpr
* the original expression
+ * @throws AlgebricksException
*/
private static void injectCastFunction(IFunctionInfo funcInfo, IAType reqType, IAType inputType,
- Mutable<ILogicalExpression> exprRef, ILogicalExpression argExpr) {
+ Mutable<ILogicalExpression> exprRef, ILogicalExpression argExpr) throws AlgebricksException {
ScalarFunctionCallExpression cast = new ScalarFunctionCallExpression(funcInfo);
cast.getArguments().add(new MutableObject<ILogicalExpression>(argExpr));
exprRef.setValue(cast);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/534d5892/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index b430807..90bf6f6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -25,10 +25,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.rmi.RemoteException;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Date;
+import java.util.Deque;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -152,9 +155,10 @@ import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.metadata.utils.MetadataLockManager;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.types.TypeSignature;
-import org.apache.asterix.optimizer.rules.IntroduceSecondaryIndexInsertDeleteRule;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
import org.apache.asterix.runtime.util.AsterixAppContextInfo;
import org.apache.asterix.runtime.util.AsterixClusterProperties;
import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory;
@@ -175,6 +179,7 @@ import org.apache.asterix.translator.TypeTranslator;
import org.apache.asterix.translator.util.ValidateUtil;
import org.apache.asterix.util.FlushDatasetUtils;
import org.apache.asterix.util.JobUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.commons.lang3.mutable.MutableObject;
@@ -976,7 +981,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
ARecordType enforcedType = null;
if (stmtCreateIndex.isEnforced()) {
- enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType,
+ enforcedType = createEnforcedType(aRecordType,
Lists.newArrayList(index));
}
@@ -2473,7 +2478,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
dataverseName);
jobsToExecute.add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider));
ARecordType aRecordType = (ARecordType) dt.getDatatype();
- ARecordType enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(
+ ARecordType enforcedType = createEnforcedType(
aRecordType, indexes);
if (ds.getDatasetType() == DatasetType.INTERNAL) {
for (int j = 0; j < indexes.size(); j++) {
@@ -3124,4 +3129,105 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
rewriter.rewrite(stmt);
}
+ /*
+ * Merges typed index fields with specified recordType, allowing indexed fields to be optional.
+ * I.e. the type { "personId":int32, "name": string, "address" : { "street": string } } with typed indexes
+ * on age:int32, address.state:string will be merged into type { "personId":int32, "name": string,
+ * "age": int32? "address" : { "street": string, "state": string? } } Used by open indexes to enforce
+ * the type of an indexed record
+ */
+ private static ARecordType createEnforcedType(ARecordType initialType, List<Index> indexes)
+ throws AlgebricksException {
+ ARecordType enforcedType = initialType;
+ for (Index index : indexes) {
+ if (!index.isSecondaryIndex() || !index.isEnforcingKeyFileds()) {
+ continue;
+ }
+ if (index.hasMetaFields()) {
+ throw new AlgebricksException("Indexing an open field is only supported on the record part");
+ }
+ for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
+ Deque<Pair<ARecordType, String>> nestedTypeStack = new ArrayDeque<>();
+ List<String> splits = index.getKeyFieldNames().get(i);
+ ARecordType nestedFieldType = enforcedType;
+ boolean openRecords = false;
+ String bridgeName = nestedFieldType.getTypeName();
+ int j;
+ // Build the stack for the enforced type
+ for (j = 1; j < splits.size(); j++) {
+ nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
+ bridgeName = nestedFieldType.getTypeName();
+ nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
+ if (nestedFieldType == null) {
+ openRecords = true;
+ break;
+ }
+ }
+ if (openRecords) {
+ // create the smallest record
+ enforcedType = new ARecordType(splits.get(splits.size() - 2),
+ new String[] { splits.get(splits.size() - 1) },
+ new IAType[] { AUnionType.createUnknownableType(index.getKeyFieldTypes().get(i)) }, true);
+ // create the open part of the nested field
+ for (int k = splits.size() - 3; k > (j - 2); k--) {
+ enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
+ new IAType[] { AUnionType.createUnknownableType(enforcedType) }, true);
+ }
+ // Bridge the gap
+ Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
+ ARecordType parent = gapPair.first;
+
+ IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
+ new IAType[] { AUnionType.createUnknownableType(enforcedType) });
+ enforcedType = new ARecordType(bridgeName,
+ ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes,
+ true);
+ } else {
+ //Schema is closed all the way to the field
+ //enforced fields are either null or strongly typed
+ LinkedHashMap<String, IAType> recordNameTypesMap = createRecordNameTypeMap(nestedFieldType);
+ // if a an enforced field already exists and the type is correct
+ IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size() - 1));
+ if (enforcedFieldType != null && enforcedFieldType.getTypeTag() == ATypeTag.UNION
+ && ((AUnionType) enforcedFieldType).isUnknownableType()) {
+ enforcedFieldType = ((AUnionType) enforcedFieldType).getActualType();
+ }
+ if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(),
+ index.getKeyFieldTypes().get(i).getTypeTag())) {
+ throw new AlgebricksException("Cannot enforce field " + index.getKeyFieldNames().get(i)
+ + " to have type " + index.getKeyFieldTypes().get(i));
+ }
+ if (enforcedFieldType == null) {
+ recordNameTypesMap.put(splits.get(splits.size() - 1),
+ AUnionType.createUnknownableType(index.getKeyFieldTypes().get(i)));
+ }
+ enforcedType = new ARecordType(nestedFieldType.getTypeName(),
+ recordNameTypesMap.keySet().toArray(new String[recordNameTypesMap.size()]),
+ recordNameTypesMap.values().toArray(new IAType[recordNameTypesMap.size()]),
+ nestedFieldType.isOpen());
+ }
+
+ // Create the enforced type for the nested fields in the schema, from the ground up
+ if (!nestedTypeStack.isEmpty()) {
+ while (!nestedTypeStack.isEmpty()) {
+ Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
+ ARecordType nestedRecType = nestedTypePair.first;
+ IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
+ nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedType;
+ enforcedType = new ARecordType(nestedRecType.getTypeName() + "_enforced",
+ nestedRecType.getFieldNames(), nestedRecTypeFieldTypes, nestedRecType.isOpen());
+ }
+ }
+ }
+ }
+ return enforcedType;
+ }
+
+ private static LinkedHashMap<String, IAType> createRecordNameTypeMap(ARecordType nestedFieldType) {
+ LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<>();
+ for (int j = 0; j < nestedFieldType.getFieldNames().length; j++) {
+ recordNameTypesMap.put(nestedFieldType.getFieldNames()[j], nestedFieldType.getFieldTypes()[j]);
+ }
+ return recordNameTypesMap;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/534d5892/asterixdb/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-1.plan
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-1.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-1.plan
index c65c71c..4e6eef4 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-1.plan
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/disjunction-to-join-delete-1.plan
@@ -4,18 +4,17 @@
-- INDEX_INSERT_DELETE |PARTITIONED|
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
-- STREAM_PROJECT |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- INSERT_DELETE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- MATERIALIZE |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- ASSIGN |PARTITIONED|
- -- STREAM_PROJECT |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- BTREE_SEARCH |PARTITIONED|
- -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- -- STABLE_SORT [$$11(ASC)] |PARTITIONED|
- -- HASH_PARTITION_EXCHANGE [$$11] |PARTITIONED|
- -- UNNEST |UNPARTITIONED|
- -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- INSERT_DELETE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- MATERIALIZE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- BTREE_SEARCH |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$11(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$11] |PARTITIONED|
+ -- UNNEST |UNPARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |UNPARTITIONED|
\ No newline at end of file