You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2016/07/24 11:32:35 UTC
[3/3] asterixdb git commit: Cast Record Before Insert for Open Indexes
Cast Record Before Insert for Open Indexes
This change modifies the insert plan to perform cast for open
indexes before inserting to primary index. This avoids modification
of indexes and then doing complex work to try and undo the insert.
In addition, feeds can simply log and remove tuples which have
open fields of incompatible type.
Change-Id: I0eef5813ddbfe1b9c518cd7f92d37c95e8486914
Reviewed-on: https://asterix-gerrit.ics.uci.edu/997
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a3603ac9
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a3603ac9
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a3603ac9
Branch: refs/heads/master
Commit: a3603ac97f47d91ebc2e5ef5d49ebc18091c591c
Parents: 806f7d2
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Fri Jul 22 23:02:32 2016 +0300
Committer: Michael Blow <mb...@apache.org>
Committed: Sun Jul 24 04:31:53 2016 -0700
----------------------------------------------------------------------
...IntroduceSecondaryIndexInsertDeleteRule.java | 75 +-
.../data/external-parser/dropbox2/jobads3.txt | 7608 ++++++++++++++++++
.../api/http/servlet/QueryServiceServlet.java | 11 +-
...th-external-parser-with-open-index.1.ddl.aql | 33 +
...th-external-parser-with-open-index.2.lib.aql | 26 +
...th-external-parser-with-open-index.3.ddl.aql | 35 +
...external-parser-with-open-index.4.update.aql | 31 +
...-external-parser-with-open-index.5.query.aql | 30 +
...th-external-parser-with-open-index.6.lib.aql | 26 +
...th-external-parser-with-open-index.7.ddl.aql | 26 +
...ernal-parser-with-two-open-indexes.1.ddl.aql | 33 +
...ernal-parser-with-two-open-indexes.2.lib.aql | 26 +
...ernal-parser-with-two-open-indexes.3.ddl.aql | 36 +
...al-parser-with-two-open-indexes.4.update.aql | 31 +
...nal-parser-with-two-open-indexes.5.query.aql | 30 +
...ernal-parser-with-two-open-indexes.6.lib.aql | 26 +
...ernal-parser-with-two-open-indexes.7.ddl.aql | 26 +
...d-with-external-parser-with-open-index.1.adm | 1 +
...-external-parser-with-two-open-indexes.1.adm | 1 +
.../src/test/resources/runtimets/testsuite.xml | 12 +-
.../resources/runtimets/testsuite_sqlpp.xml | 14 +-
...erixLSMInsertDeleteOperatorNodePushable.java | 18 +-
.../asterix/common/exceptions/ErrorCode.java | 27 +
.../common/exceptions/FrameDataException.java | 38 -
.../feed/dataflow/FeedExceptionHandler.java | 8 +-
.../om/pointables/cast/ARecordCaster.java | 18 +-
.../rewriter/rules/InlineVariablesRule.java | 127 +-
.../operators/std/AssignRuntimeFactory.java | 53 +-
.../hyracks/api/exceptions/ErrorCode.java | 13 +-
.../api/exceptions/HyracksDataException.java | 76 +-
.../control/common/utils/ExceptionUtils.java | 25 +-
31 files changed, 8318 insertions(+), 222 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a3603ac9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index f5cfa2b..ae69093 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -146,8 +146,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
AssignOperator assignOp = (AssignOperator) op2;
ILogicalExpression assignExpr = assignOp.getExpressions().get(0).getValue();
if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- ScalarFunctionCallExpression funcExpr = (ScalarFunctionCallExpression) assignOp.getExpressions()
- .get(0).getValue();
+ ScalarFunctionCallExpression funcExpr =
+ (ScalarFunctionCallExpression) assignOp.getExpressions().get(0).getValue();
fid = funcExpr.getFunctionIdentifier();
}
}
@@ -214,10 +214,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
}
// At this point, we have the data type info, and the indexes info as well
- // Initialize inputs to the SINK operator Op0 (The SINK) is now without input
if (secondaryIndexTotalCnt > 0) {
op0.getInputs().clear();
}
+ // Initialize inputs to the SINK operator Op0 (The SINK) is now without input
// Prepare filtering field information (This is the filter created using the "filter with" key word in the
// create dataset ddl)
@@ -247,11 +247,12 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
try {
DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource());
ARecordType insertRecType = (ARecordType) ds.getItemType();
- // A new variable which represents the casted record
- LogicalVariable castedRecVar = context.newVar();
// create the expected record type = the original + the optional open field
ARecordType enforcedType = createEnforcedType(insertRecType, indexes);
if (!enforcedType.equals(insertRecType)) {
+ // A new variable which represents the casted record
+ LogicalVariable castedRecVar = context.newVar();
+ context.addNotToBeInlinedVar(castedRecVar);
//introduce casting to enforced type
AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CAST_RECORD));
@@ -260,14 +261,16 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
.add(new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue()));
TypeCastUtils.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType);
// AssignOperator puts in the cast var the casted record
- AssignOperator castedRecordAssignOperator = new AssignOperator(castedRecVar,
- new MutableObject<ILogicalExpression>(castFunc));
+ AssignOperator castedRecordAssignOperator =
+ new AssignOperator(castedRecVar, new MutableObject<ILogicalExpression>(castFunc));
// Connect the current top of the plan to the cast operator
- castedRecordAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
- currentTop = castedRecordAssignOperator;
+ castedRecordAssignOperator.getInputs().addAll(currentTop.getInputs());
+ currentTop.getInputs().clear();
+ currentTop.getInputs().add(new MutableObject<>(castedRecordAssignOperator));
enforcedRecordVar = castedRecVar;
recType = enforcedType;
context.computeAndSetTypeEnvironmentForOperator(castedRecordAssignOperator);
+ context.computeAndSetTypeEnvironmentForOperator(currentTop);
// We don't need to cast the old rec, we just need an assignment function that extracts the SK
// and an expression which reference the new variables.
}
@@ -404,8 +407,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
// Check the field type of the secondary key.
IAType secondaryKeyType = null;
- Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0),
- recType);
+ Pair<IAType, Boolean> keyPairType =
+ Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0), recType);
secondaryKeyType = keyPairType.first;
List<Object> varTypes = new ArrayList<Object>();
@@ -430,11 +433,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assign));
context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
- IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
- insertOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
- insertOp.getOperation(), insertOp.isBulkload(),
- insertOp.getAdditionalNonFilteringExpressions() == null ? 0
- : insertOp.getAdditionalNonFilteringExpressions().size());
+ IndexInsertDeleteUpsertOperator indexUpdate =
+ new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
+ tokenizeKeyExprs, filterExpression, insertOp.getOperation(), insertOp.isBulkload(),
+ insertOp.getAdditionalNonFilteringExpressions() == null ? 0
+ : insertOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
@@ -445,11 +448,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
} else {
// When TokenizeOperator is not needed
- IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
- insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
- insertOp.getOperation(), insertOp.isBulkload(),
- insertOp.getAdditionalNonFilteringExpressions() == null ? 0
- : insertOp.getAdditionalNonFilteringExpressions().size());
+ IndexInsertDeleteUpsertOperator indexUpdate =
+ new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
+ secondaryExpressions, filterExpression, insertOp.getOperation(),
+ insertOp.isBulkload(), insertOp.getAdditionalNonFilteringExpressions() == null ? 0
+ : insertOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
// We add the necessary expressions for upsert
@@ -473,11 +476,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
} else if (index.getIndexType() == IndexType.RTREE) {
// Get type, dimensions and number of keys
- Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
- secondaryKeyFields.get(0), recType);
+ Pair<IAType, Boolean> keyPairType =
+ Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyFields.get(0), recType);
IAType spatialType = keyPairType.first;
- boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
- || spatialType.getTypeTag() == ATypeTag.POINT3D;
+ boolean isPointMBR =
+ spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numKeys = (isPointMBR && isBulkload) ? dimension : dimension * 2;
// Get variables and expressions
@@ -554,11 +557,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
}
AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
- IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
- insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
- insertOp.getOperation(), insertOp.isBulkload(),
- insertOp.getAdditionalNonFilteringExpressions() == null ? 0
- : insertOp.getAdditionalNonFilteringExpressions().size());
+ IndexInsertDeleteUpsertOperator indexUpdate =
+ new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
+ secondaryExpressions, filterExpression, insertOp.getOperation(), insertOp.isBulkload(),
+ insertOp.getAdditionalNonFilteringExpressions() == null ? 0
+ : insertOp.getAdditionalNonFilteringExpressions().size());
indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
if (insertOp.getOperation() == Kind.UPSERT) {
// set old secondary key expressions
@@ -772,12 +775,12 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
if (!NonTaggedFormatUtil.isOptional(secondaryKeyType) && !forceFilter) {
continue;
}
- ScalarFunctionCallExpression isUnknownFuncExpr = new ScalarFunctionCallExpression(
- FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN),
- new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
- ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression(
- FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT),
- new MutableObject<ILogicalExpression>(isUnknownFuncExpr));
+ ScalarFunctionCallExpression isUnknownFuncExpr =
+ new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN),
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
+ ScalarFunctionCallExpression notFuncExpr =
+ new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT),
+ new MutableObject<ILogicalExpression>(isUnknownFuncExpr));
filterExpressions.add(new MutableObject<ILogicalExpression>(notFuncExpr));
}
// No nullable secondary keys.