You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/08/25 18:44:19 UTC
[31/51] [partial] incubator-asterixdb git commit: Change folder
structure for Java repackage
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
new file mode 100644
index 0000000..42b60d5
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -0,0 +1,598 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Stack;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.aql.util.FunctionUtils;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
+import edu.uci.ics.asterix.metadata.declared.AqlIndex;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.declared.DatasetDataSource;
+import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AOrderedList;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
+import edu.uci.ics.asterix.om.types.AOrderedListType;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.AUnionType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+ if (op0.getOperatorTag() != LogicalOperatorTag.SINK) {
+ return false;
+ }
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE) {
+ return false;
+ }
+
+ FunctionIdentifier fid = null;
+ /** find the record variable */
+ InsertDeleteOperator insertOp = (InsertDeleteOperator) op1;
+ ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue();
+ List<LogicalVariable> recordVar = new ArrayList<LogicalVariable>();
+ /** assume the payload is always a single variable expression */
+ recordExpr.getUsedVariables(recordVar);
+
+ /**
+ * op2 is the assign operator which extract primary keys from the record
+ * variable
+ */
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+
+ if (recordVar.size() == 0) {
+ /**
+ * For the case primary key-assignment expressions are constant
+ * expressions, find assign op that creates record to be
+ * inserted/deleted.
+ */
+ while (fid != AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR) {
+ if (op2.getInputs().size() == 0) {
+ return false;
+ }
+ op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ continue;
+ }
+ AssignOperator assignOp = (AssignOperator) op2;
+ ILogicalExpression assignExpr = assignOp.getExpressions().get(0).getValue();
+ if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ ScalarFunctionCallExpression funcExpr = (ScalarFunctionCallExpression) assignOp.getExpressions()
+ .get(0).getValue();
+ fid = funcExpr.getFunctionIdentifier();
+ }
+ }
+ AssignOperator assignOp2 = (AssignOperator) op2;
+ recordVar.addAll(assignOp2.getVariables());
+ }
+ AqlDataSource datasetSource = (AqlDataSource) insertOp.getDataSource();
+ AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider();
+ String dataverseName = datasetSource.getId().getDataverseName();
+ String datasetName = datasetSource.getId().getDatasourceName();
+ Dataset dataset = mp.findDataset(dataverseName, datasetName);
+ if (dataset == null) {
+ throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
+ }
+ if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+ return false;
+ }
+
+ // Create operators for secondary index insert/delete.
+ String itemTypeName = dataset.getItemTypeName();
+ IAType itemType = mp.findType(dataset.getDataverseName(), itemTypeName);
+ if (itemType.getTypeTag() != ATypeTag.RECORD) {
+ throw new AlgebricksException("Only record types can be indexed.");
+ }
+ ARecordType recType = (ARecordType) itemType;
+ List<Index> indexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
+ ILogicalOperator currentTop = op1;
+ boolean hasSecondaryIndex = false;
+
+ // Put an n-gram or a keyword index in the later stage of index-update,
+ // since TokenizeOperator needs to be involved.
+ Collections.sort(indexes, new Comparator<Index>() {
+ @Override
+ public int compare(Index o1, Index o2) {
+ return o1.getIndexType().ordinal() - o2.getIndexType().ordinal();
+ }
+
+ });
+
+ // Check whether multiple keyword or n-gram indexes exist
+ int secondaryIndexTotalCnt = 0;
+ for (Index index : indexes) {
+ if (index.isSecondaryIndex())
+ secondaryIndexTotalCnt++;
+ }
+
+ // Initialize inputs to the SINK operator
+ if (secondaryIndexTotalCnt > 0) {
+ op0.getInputs().clear();
+ }
+
+ // Replicate Operator is applied only when doing the bulk-load.
+ AbstractLogicalOperator replicateOp = null;
+
+ if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) {
+ // Split the logical plan into "each secondary index update branch"
+ // to replicate each <PK,RECORD> pair.
+ replicateOp = new ReplicateOperator(secondaryIndexTotalCnt);
+ replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+ replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
+ context.computeAndSetTypeEnvironmentForOperator(replicateOp);
+ currentTop = replicateOp;
+ }
+
+ // Prepare filtering field information
+ List<String> additionalFilteringField = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField();
+ List<LogicalVariable> additionalFilteringVars = null;
+ List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions = null;
+ List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
+ AssignOperator additionalFilteringAssign = null;
+
+ if (additionalFilteringField != null) {
+ additionalFilteringVars = new ArrayList<LogicalVariable>();
+ additionalFilteringAssignExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ additionalFilteringExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ prepareVarAndExpression(additionalFilteringField, recType.getFieldNames(), recordVar.get(0),
+ additionalFilteringAssignExpressions, additionalFilteringVars, context);
+ additionalFilteringAssign = new AssignOperator(additionalFilteringVars,
+ additionalFilteringAssignExpressions);
+ for (LogicalVariable var : additionalFilteringVars) {
+ additionalFilteringExpressions.add(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(var)));
+ }
+ }
+
+ // Iterate each secondary index and applying Index Update operations.
+ for (Index index : indexes) {
+ List<LogicalVariable> projectVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op1, projectVars);
+ if (!index.isSecondaryIndex()) {
+ continue;
+ }
+ LogicalVariable enforcedRecordVar = recordVar.get(0);
+ hasSecondaryIndex = true;
+ //if the index is enforcing field types
+ if (index.isEnforcingKeyFileds()) {
+ try {
+ DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource());
+ ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1];
+ LogicalVariable castVar = context.newVar();
+ ARecordType enforcedType = createEnforcedType(insertRecType, index);
+ //introduce casting to enforced type
+ AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CAST_RECORD));
+
+ castFunc.getArguments().add(
+ new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue()));
+ TypeComputerUtilities.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType);
+ AssignOperator newAssignOperator = new AssignOperator(castVar,
+ new MutableObject<ILogicalExpression>(castFunc));
+ newAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+ currentTop = newAssignOperator;
+ //project out casted record
+ projectVars.add(castVar);
+ enforcedRecordVar = castVar;
+ context.computeAndSetTypeEnvironmentForOperator(newAssignOperator);
+ context.computeAndSetTypeEnvironmentForOperator(currentTop);
+ recType = enforcedType;
+ } catch (AsterixException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
+ List<IAType> secondaryKeyTypes = index.getKeyFieldTypes();
+ List<LogicalVariable> secondaryKeyVars = new ArrayList<LogicalVariable>();
+ List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
+ List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+
+ for (List<String> secondaryKey : secondaryKeyFields) {
+ prepareVarAndExpression(secondaryKey, recType.getFieldNames(), enforcedRecordVar, expressions,
+ secondaryKeyVars, context);
+ }
+
+ AssignOperator assign = new AssignOperator(secondaryKeyVars, expressions);
+ ProjectOperator project = new ProjectOperator(projectVars);
+
+ if (additionalFilteringAssign != null) {
+ additionalFilteringAssign.getInputs().add(new MutableObject<ILogicalOperator>(project));
+ assign.getInputs().add(new MutableObject<ILogicalOperator>(additionalFilteringAssign));
+ } else {
+ assign.getInputs().add(new MutableObject<ILogicalOperator>(project));
+ }
+
+ // Only apply replicate operator when doing bulk-load
+ if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload())
+ project.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp));
+ else
+ project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+ context.computeAndSetTypeEnvironmentForOperator(project);
+
+ if (additionalFilteringAssign != null) {
+ context.computeAndSetTypeEnvironmentForOperator(additionalFilteringAssign);
+ }
+
+ context.computeAndSetTypeEnvironmentForOperator(assign);
+ currentTop = assign;
+
+ // BTree, Keyword, or n-gram index case
+ if (index.getIndexType() == IndexType.BTREE
+ || index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
+ || index.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX
+ || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
+ for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
+ secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+ secondaryKeyVar)));
+ }
+ Mutable<ILogicalExpression> filterExpression = createFilterExpression(secondaryKeyVars,
+ context.getOutputTypeEnvironment(currentTop), false);
+ AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
+
+ // Introduce the TokenizeOperator only when doing bulk-load,
+ // and index type is keyword or n-gram.
+ if (index.getIndexType() != IndexType.BTREE && insertOp.isBulkload()) {
+
+ // Check whether the index is length-partitioned or not.
+ // If partitioned, [input variables to TokenizeOperator,
+ // token, number of token] pairs will be generated and
+ // fed into the IndexInsertDeleteOperator.
+ // If not, [input variables, token] pairs will be generated
+ // and fed into the IndexInsertDeleteOperator.
+ // Input variables are passed since TokenizeOperator is not an
+ // filtering operator.
+ boolean isPartitioned = false;
+ if (index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)
+ isPartitioned = true;
+
+ // Create a new logical variable - token
+ List<LogicalVariable> tokenizeKeyVars = new ArrayList<LogicalVariable>();
+ List<Mutable<ILogicalExpression>> tokenizeKeyExprs = new ArrayList<Mutable<ILogicalExpression>>();
+ LogicalVariable tokenVar = context.newVar();
+ tokenizeKeyVars.add(tokenVar);
+ tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(tokenVar)));
+
+ // Check the field type of the secondary key.
+ IAType secondaryKeyType = null;
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0),
+ recType);
+ secondaryKeyType = keyPairType.first;
+
+ List<Object> varTypes = new ArrayList<Object>();
+ varTypes.add(NonTaggedFormatUtil.getTokenType(secondaryKeyType));
+
+ // If the index is a length-partitioned, then create
+ // additional variable - number of token.
+ // We use a special type for the length-partitioned index.
+ // The type is short, and this does not contain type info.
+ if (isPartitioned) {
+ LogicalVariable lengthVar = context.newVar();
+ tokenizeKeyVars.add(lengthVar);
+ tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+ lengthVar)));
+ varTypes.add(BuiltinType.SHORTWITHOUTTYPEINFO);
+ }
+
+ // TokenizeOperator to tokenize [SK, PK] pairs
+ TokenizeOperator tokenUpdate = new TokenizeOperator(dataSourceIndex,
+ insertOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars,
+ filterExpression, insertOp.getOperation(), insertOp.isBulkload(), isPartitioned, varTypes);
+ tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+ context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
+
+ IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
+ insertOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
+ insertOp.getOperation(), insertOp.isBulkload());
+ indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+ indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
+
+ context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
+
+ currentTop = indexUpdate;
+ op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+ } else {
+ // When TokenizeOperator is not needed
+ IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
+ insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
+ insertOp.getOperation(), insertOp.isBulkload());
+ indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+ indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+ currentTop = indexUpdate;
+ context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
+
+ if (insertOp.isBulkload())
+ op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+ }
+
+ } else if (index.getIndexType() == IndexType.RTREE) {
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
+ secondaryKeyFields.get(0), recType);
+ IAType spatialType = keyPairType.first;
+ int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
+ int numKeys = dimension * 2;
+ List<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
+ List<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
+ for (int i = 0; i < numKeys; i++) {
+ LogicalVariable keyVar = context.newVar();
+ keyVarList.add(keyVar);
+ AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CREATE_MBR));
+ createMBR.getArguments().add(
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVars
+ .get(0))));
+ createMBR.getArguments().add(
+ new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
+ new AInt32(dimension)))));
+ createMBR.getArguments().add(
+ new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
+ new AInt32(i)))));
+ keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
+ }
+ for (LogicalVariable secondaryKeyVar : keyVarList) {
+ secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+ secondaryKeyVar)));
+ }
+ AssignOperator assignCoordinates = new AssignOperator(keyVarList, keyExprList);
+ assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+ context.computeAndSetTypeEnvironmentForOperator(assignCoordinates);
+ // We must enforce the filter if the originating spatial type is
+ // nullable.
+ boolean forceFilter = keyPairType.second;
+ Mutable<ILogicalExpression> filterExpression = createFilterExpression(keyVarList,
+ context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
+ AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
+ IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
+ insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
+ insertOp.getOperation(), insertOp.isBulkload());
+ indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+ indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
+ currentTop = indexUpdate;
+ context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
+
+ if (insertOp.isBulkload())
+ op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+ }
+
+ }
+ if (!hasSecondaryIndex) {
+ return false;
+ }
+
+ if (!insertOp.isBulkload()) {
+ op0.getInputs().clear();
+ op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+ }
+ return true;
+ }
+
+ public static ARecordType createEnforcedType(ARecordType initialType, Index index) throws AsterixException,
+ AlgebricksException {
+ ARecordType enforcedType = initialType;
+ for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
+ try {
+ Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>();
+ List<String> splits = index.getKeyFieldNames().get(i);
+ ARecordType nestedFieldType = enforcedType;
+ boolean openRecords = false;
+ String bridgeName = nestedFieldType.getTypeName();
+ int j;
+ //Build the stack for the enforced type
+ for (j = 1; j < splits.size(); j++) {
+ nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
+ bridgeName = nestedFieldType.getTypeName();
+ nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
+ if (nestedFieldType == null) {
+ openRecords = true;
+ break;
+ }
+ }
+ if (openRecords == true) {
+ //create the smallest record
+ enforcedType = new ARecordType(splits.get(splits.size() - 2), new String[] { splits.get(splits
+ .size() - 1) }, new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes()
+ .get(i)) }, true);
+ //create the open part of the nested field
+ for (int k = splits.size() - 3; k > (j - 2); k--) {
+ enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
+ new IAType[] { AUnionType.createNullableType(enforcedType) }, true);
+ }
+ //Bridge the gap
+ Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
+ ARecordType parent = gapPair.first;
+
+ IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
+ new IAType[] { AUnionType.createNullableType(enforcedType) });
+ enforcedType = new ARecordType(bridgeName, ArrayUtils.addAll(parent.getFieldNames(),
+ enforcedType.getTypeName()), parentFieldTypes, true);
+
+ } else {
+ //Schema is closed all the way to the field
+ //enforced fields are either null or strongly typed
+ enforcedType = new ARecordType(nestedFieldType.getTypeName(), ArrayUtils.addAll(
+ nestedFieldType.getFieldNames(), splits.get(splits.size() - 1)), ArrayUtils.addAll(
+ nestedFieldType.getFieldTypes(),
+ AUnionType.createNullableType(index.getKeyFieldTypes().get(i))), nestedFieldType.isOpen());
+ }
+
+ //Create the enforcedtype for the nested fields in the schema, from the ground up
+ if (nestedTypeStack.size() > 0) {
+ while (!nestedTypeStack.isEmpty()) {
+ Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
+ ARecordType nestedRecType = nestedTypePair.first;
+ IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
+ nestedRecTypeFieldTypes[nestedRecType.findFieldPosition(nestedTypePair.second)] = enforcedType;
+ enforcedType = new ARecordType(nestedRecType.getTypeName(), nestedRecType.getFieldNames(),
+ nestedRecTypeFieldTypes, nestedRecType.isOpen());
+ }
+ }
+
+ } catch (AsterixException e) {
+ throw new AlgebricksException("Cannot enforce typed fields "
+ + StringUtils.join(index.getKeyFieldNames()), e);
+ } catch (IOException e) {
+ throw new AsterixException(e);
+ }
+ }
+ return enforcedType;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void prepareVarAndExpression(List<String> field, String[] fieldNames, LogicalVariable recordVar,
+ List<Mutable<ILogicalExpression>> expressions, List<LogicalVariable> vars, IOptimizationContext context)
+ throws AlgebricksException {
+ Mutable<ILogicalExpression> varRef = new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+ recordVar));
+ int pos = -1;
+ if (field.size() == 1) {
+ for (int j = 0; j < fieldNames.length; j++) {
+ if (fieldNames[j].equals(field.get(0))) {
+ pos = j;
+ break;
+ }
+ }
+ }
+ if (pos == -1) {
+ AbstractFunctionCallExpression func;
+ if (field.size() > 1) {
+ AOrderedList fieldList = new AOrderedList(new AOrderedListType(BuiltinType.ASTRING, null));
+ for (int i = 0; i < field.size(); i++) {
+ fieldList.add(new AString(field.get(i)));
+ }
+ Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
+ new AsterixConstantValue(fieldList)));
+ //Create an expression for the nested case
+ func = new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED), varRef, fieldRef);
+ } else {
+ Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
+ new AsterixConstantValue(new AString(field.get(0)))));
+ //Create an expression for the open field case (By name)
+ func = new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME), varRef, fieldRef);
+ }
+ expressions.add(new MutableObject<ILogicalExpression>(func));
+ LogicalVariable newVar = context.newVar();
+ vars.add(newVar);
+ } else {
+ // Assumes the indexed field is in the closed portion of the type.
+ Mutable<ILogicalExpression> indexRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
+ new AsterixConstantValue(new AInt32(pos))));
+ AbstractFunctionCallExpression func = new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX), varRef, indexRef);
+ expressions.add(new MutableObject<ILogicalExpression>(func));
+ LogicalVariable newVar = context.newVar();
+ vars.add(newVar);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private Mutable<ILogicalExpression> createFilterExpression(List<LogicalVariable> secondaryKeyVars,
+ IVariableTypeEnvironment typeEnv, boolean forceFilter) throws AlgebricksException {
+ List<Mutable<ILogicalExpression>> filterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ // Add 'is not null' to all nullable secondary index keys as a filtering
+ // condition.
+ for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
+ IAType secondaryKeyType = (IAType) typeEnv.getVarType(secondaryKeyVar);
+ if (!NonTaggedFormatUtil.isOptional(secondaryKeyType) && !forceFilter) {
+ continue;
+ }
+ ScalarFunctionCallExpression isNullFuncExpr = new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL),
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
+ ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NOT), new MutableObject<ILogicalExpression>(
+ isNullFuncExpr));
+ filterExpressions.add(new MutableObject<ILogicalExpression>(notFuncExpr));
+ }
+ // No nullable secondary keys.
+ if (filterExpressions.isEmpty()) {
+ return null;
+ }
+ Mutable<ILogicalExpression> filterExpression = null;
+ if (filterExpressions.size() > 1) {
+ // Create a conjunctive condition.
+ filterExpression = new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.AND), filterExpressions));
+ } else {
+ filterExpression = filterExpressions.get(0);
+ }
+ return filterExpression;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java
new file mode 100644
index 0000000..6419fc1
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
+import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.optimizer.rules.typecast.StaticTypeCastUtil;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Statically cast a constant from its type to a specified required type, in a
+ * recursive way. It enables: 1. bag-based fields in a record, 2. bidirectional
+ * cast of an open field and a matched closed field, and 3. put in null fields
+ * when necessary. It should be fired before the constant folding rule.
+ * This rule is not responsible for type casting between primitive types.
+ * Here is an example: A record { "hobby": {{"music", "coding"}}, "id": "001",
+ * "name": "Person Three"} which confirms to closed type ( id: string, name:
+ * string, hobby: {{string}}? ) can be cast to an open type (id: string ), or
+ * vice versa.
+ * Implementation wise: first, we match the record's type and its target dataset
+ * type to see if it is "cast-able"; second, if the types are cast-able, we
+ * embed the required type into the original producer expression. If the types
+ * are not cast-able, we throw a compile time exception.
+ * Then, at runtime (not in this rule), the corresponding record/list
+ * constructors know what to do by checking the required output type.
+ * TODO: right now record/list constructor of the cast result is not done in the
+ * ConstantFoldingRule and has to go to the runtime, because the
+ * ConstantFoldingRule uses ARecordSerializerDeserializer which seems to have
+ * some problem.
+ */
+public class IntroduceStaticTypeCastForInsertRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ /**
+ * pattern match: sink/insert/assign record type is propagated from
+ * insert data source to the record-constructor expression
+ */
+ if (context.checkIfInDontApplySet(this, opRef.getValue()))
+ return false;
+ context.addToDontApplySet(this, opRef.getValue());
+
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ List<LogicalVariable> producedVariables = new ArrayList<LogicalVariable>();
+ LogicalVariable oldRecordVariable;
+
+ if (op1.getOperatorTag() != LogicalOperatorTag.SINK)
+ return false;
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE)
+ return false;
+ InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op2;
+ if (insertDeleteOp.getOperation() == InsertDeleteOperator.Kind.DELETE)
+ return false;
+ /**
+ * get required record type
+ */
+ InsertDeleteOperator insertDeleteOperator = (InsertDeleteOperator) op2;
+ AqlDataSource dataSource = (AqlDataSource) insertDeleteOperator.getDataSource();
+ IAType[] schemaTypes = (IAType[]) dataSource.getSchemaTypes();
+ IAType requiredRecordType = schemaTypes[schemaTypes.length - 1];
+
+ List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+ insertDeleteOperator.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+
+ // the used variable should contain the record that will be inserted
+ // but it will not fail in many cases even if the used variable set is
+ // empty
+ if (usedVariables.size() == 0)
+ return false;
+
+ oldRecordVariable = usedVariables.get(0);
+ LogicalVariable inputRecordVar = usedVariables.get(0);
+ IVariableTypeEnvironment env = insertDeleteOperator.computeOutputTypeEnvironment(context);
+ IAType inputRecordType = (IAType) env.getVarType(inputRecordVar);
+
+ AbstractLogicalOperator currentOperator = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+ /**
+ * find the assign operator for the "input record" to the insert_delete
+ * operator
+ */
+ do {
+ context.addToDontApplySet(this, currentOperator);
+ if (currentOperator.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assignOp = (AssignOperator) currentOperator;
+ producedVariables.clear();
+ VariableUtilities.getProducedVariables(currentOperator, producedVariables);
+ int position = producedVariables.indexOf(oldRecordVariable);
+
+ /**
+ * set the top-down propagated type
+ */
+ if (position >= 0) {
+ AssignOperator originalAssign = (AssignOperator) currentOperator;
+ List<Mutable<ILogicalExpression>> expressionRefs = originalAssign.getExpressions();
+ ILogicalExpression expr = expressionRefs.get(position).getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+ // that expression has been rewritten, and it will not
+ // fail but just return false
+ if (TypeComputerUtilities.getRequiredType(funcExpr) != null) {
+ context.computeAndSetTypeEnvironmentForOperator(assignOp);
+ return false;
+ }
+ IVariableTypeEnvironment assignEnv = assignOp.computeOutputTypeEnvironment(context);
+ StaticTypeCastUtil.rewriteFuncExpr(funcExpr, requiredRecordType, inputRecordType, assignEnv);
+ }
+ context.computeAndSetTypeEnvironmentForOperator(originalAssign);
+ }
+ }
+ if (currentOperator.getInputs().size() > 0)
+ currentOperator = (AbstractLogicalOperator) currentOperator.getInputs().get(0).getValue();
+ else
+ break;
+ } while (currentOperator != null);
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java
new file mode 100644
index 0000000..3984fe8
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IntroduceTransactionCommitByAssignOpRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+ SelectOperator selectOperator = (SelectOperator) op;
+
+ Mutable<ILogicalOperator> childOfSelect = selectOperator.getInputs().get(0);
+
+ //[Direction] SelectOp(cond1)<--ChildOps... ==> SelectOp(booleanValue of cond1)<--NewAssignOp(cond1)<--ChildOps...
+ //#. Create an assign-operator with a new local variable and the condition of the select-operator.
+ //#. Set the input(child operator) of the new assign-operator to input(child operator) of the select-operator.
+ // (Later, the newly created assign-operator will apply the condition on behalf of the select-operator,
+ // and set the variable of the assign-operator to a boolean value according to the condition evaluation.)
+ //#. Give the select-operator the result boolean value created by the newly created child assign-operator.
+
+ //create an assignOp with a variable and the condition of the select-operator.
+ LogicalVariable v = context.newVar();
+ AssignOperator assignOperator = new AssignOperator(v, new MutableObject<ILogicalExpression>(selectOperator
+ .getCondition().getValue()));
+
+ //set the input of the new assign-operator to the input of the select-operator.
+ assignOperator.getInputs().add(childOfSelect);
+
+ //set the result value of the assign-operator to the condition of the select-operator
+ selectOperator.getCondition().setValue(new VariableReferenceExpression(v));//scalarFunctionCallExpression);
+ selectOperator.getInputs().set(0, new MutableObject<ILogicalOperator>(assignOperator));
+
+ context.computeAndSetTypeEnvironmentForOperator(assignOperator);
+
+ //Once this rule is fired, don't apply again.
+ context.addToDontApplySet(this, selectOperator);
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
new file mode 100644
index 0000000..050a2cd
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * @author kereno, ecarm002, ildar.absalyamov
+ * Generates a union operator and puts it instead of "assign <- [function-call: asterix:union]"
+ * Before rule:
+ * ============
+ * assign [var] <- [asterix:union(left_branch, right_branch)]
+ * join (TRUE)
+ * left_branch
+ * right_branch
+ * After rule:
+ * ============
+ * union (left_branch, right_branch, result_var)
+ * left_branch
+ * right_branch
+ */
+public class IntroduceUnionRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+
+ if (!opRef.getValue().getOperatorTag().equals(LogicalOperatorTag.ASSIGN)) {
+ return false;
+ }
+
+ AssignOperator assignUnion = (AssignOperator) opRef.getValue();
+
+ if (assignUnion.getExpressions().get(0).getValue().getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL)
+ return false;
+
+ AbstractFunctionCallExpression u = (AbstractFunctionCallExpression) assignUnion.getExpressions().get(0)
+ .getValue();
+ if (!AsterixBuiltinFunctions.UNION.equals(u.getFunctionIdentifier())) {
+ return false;
+ }
+
+ //Retrieving the logical variables for the union from the two aggregates which are inputs to the join
+ Mutable<ILogicalOperator> join = assignUnion.getInputs().get(0);
+
+ LogicalOperatorTag tag1 = join.getValue().getOperatorTag();
+ if (tag1 != LogicalOperatorTag.INNERJOIN && tag1 != LogicalOperatorTag.LEFTOUTERJOIN) {
+ return false;
+ }
+ AbstractBinaryJoinOperator join1 = (AbstractBinaryJoinOperator) join.getValue();
+ ILogicalExpression cond1 = join1.getCondition().getValue();
+ // don't try to push a product down
+ if (!OperatorPropertiesUtil.isAlwaysTrueCond(cond1)) {
+ return false;
+ }
+
+ List<Mutable<ILogicalOperator>> joinInputs = join.getValue().getInputs();
+
+ Mutable<ILogicalOperator> left_branch = joinInputs.get(0);
+ Mutable<ILogicalOperator> right_branch = joinInputs.get(1);
+
+ List<LogicalVariable> input1Var = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(left_branch.getValue(), input1Var);
+
+ List<LogicalVariable> input2Var = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(right_branch.getValue(), input2Var);
+
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>(
+ 1);
+ Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
+ input1Var.get(0), input2Var.get(0), assignUnion.getVariables().get(0));
+ varMap.add(triple);
+ UnionAllOperator unionOp = new UnionAllOperator(varMap);
+
+ unionOp.getInputs().add(left_branch);
+ unionOp.getInputs().add(right_branch);
+
+ context.computeAndSetTypeEnvironmentForOperator(unionOp);
+
+ opRef.setValue(unionOp);
+
+ return true;
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java
new file mode 100644
index 0000000..5943911
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.aql.util.FunctionUtils;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule introduces a unnest operator for the collection-to-sequence function (if the input to the function is a collection).
+ *
+ * @author yingyib
+ */
+public class IntroduceUnnestForCollectionToSequenceRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+ AssignOperator assign = (AssignOperator) op;
+ List<Mutable<ILogicalExpression>> exprs = assign.getExpressions();
+ if (exprs.size() != 1) {
+ return false;
+ }
+ ILogicalExpression expr = exprs.get(0).getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression func = (AbstractFunctionCallExpression) expr;
+ if (func.getFunctionIdentifier() != AsterixBuiltinFunctions.COLLECTION_TO_SEQUENCE) {
+ return false;
+ }
+
+ IVariableTypeEnvironment env = assign.computeInputTypeEnvironment(context);
+ ILogicalExpression argExpr = func.getArguments().get(0).getValue();
+ IAType outerExprType = (IAType) env.getType(expr);
+ IAType innerExprType = (IAType) env.getType(argExpr);
+ if (outerExprType.equals(innerExprType)) {
+ /** nothing is changed with the collection-to-sequence function, remove the collection-sequence function call */
+ assign.getExpressions().set(0, new MutableObject<ILogicalExpression>(argExpr));
+ return true;
+ }
+ /** change the assign operator to an unnest operator */
+ LogicalVariable var = assign.getVariables().get(0);
+ @SuppressWarnings("unchecked")
+ UnnestOperator unnest = new UnnestOperator(var, new MutableObject<ILogicalExpression>(
+ new UnnestingFunctionCallExpression(
+ FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.SCAN_COLLECTION),
+ new MutableObject<ILogicalExpression>(argExpr))));
+ unnest.getInputs().addAll(assign.getInputs());
+ opRef.setValue(unnest);
+ context.computeAndSetTypeEnvironmentForOperator(unnest);
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
new file mode 100644
index 0000000..54e470c
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
@@ -0,0 +1,368 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.algebra.base.AsterixOperatorAnnotations;
+import edu.uci.ics.asterix.common.exceptions.AsterixRuntimeException;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.optimizer.base.AnalysisUtil;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
+
+ private ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (context.checkIfInDontApplySet(this, op1)) {
+ return false;
+ }
+
+ if (op1.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator a1 = (AssignOperator) op1;
+ ILogicalExpression expr = getFirstExpr(a1);
+ if (AnalysisUtil.isAccessToFieldRecord(expr)) {
+ boolean res = findAndEliminateRedundantFieldAccess(a1);
+ context.addToDontApplySet(this, op1);
+ return res;
+ }
+ }
+ exprVisitor.setTopOp(op1);
+ exprVisitor.setContext(context);
+ boolean res = op1.acceptExpressionTransform(exprVisitor);
+ if (!res) {
+ context.addToDontApplySet(this, op1);
+ }
+ if (res && op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ // checking if we can annotate a Selection as using just one field
+ // access
+ SelectOperator sigma = (SelectOperator) op1;
+ LinkedList<LogicalVariable> vars = new LinkedList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(sigma, vars);
+ if (vars.size() == 1) {
+ // we can annotate Selection
+ AssignOperator assign1 = (AssignOperator) op1.getInputs().get(0).getValue();
+ AbstractLogicalExpression expr1 = (AbstractLogicalExpression) getFirstExpr(assign1);
+ if (expr1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr1;
+ // f should be a call to a field/data access kind of
+ // function
+ sigma.getAnnotations().put(AsterixOperatorAnnotations.FIELD_ACCESS, f.getArguments().get(0));
+ }
+ }
+ }
+
+ // TODO: avoid having to recompute type env. here
+ if (res) {
+ OperatorPropertiesUtil.typeOpRec(opRef, context);
+ }
+ return res;
+ }
+
+ private static boolean pushFieldLoads(Mutable<ILogicalExpression> exprRef, AbstractLogicalOperator topOp,
+ IOptimizationContext context) throws AlgebricksException {
+ ILogicalExpression expr = exprRef.getValue();
+ if (expr == null) {
+ return false;
+ }
+ switch (expr.getExpressionTag()) {
+ case FUNCTION_CALL: {
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
+ FunctionIdentifier fi = f.getFunctionIdentifier();
+ if (AlgebricksBuiltinFunctions.isComparisonFunction(fi)) {
+ boolean b1 = pushFieldLoads(f.getArguments().get(0), topOp, context);
+ boolean b2 = pushFieldLoads(f.getArguments().get(1), topOp, context);
+ return b1 || b2;
+ }
+ if (fi.equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
+ if (AnalysisUtil.numberOfVarsInExpr(f) == 0) {
+ return false;
+ }
+ // create an assign
+ LogicalVariable v = context.newVar();
+ AssignOperator a2 = new AssignOperator(v, new MutableObject<ILogicalExpression>(f));
+ pushFieldAssign(a2, topOp, context);
+ context.computeAndSetTypeEnvironmentForOperator(a2);
+ ILogicalExpression arg = f.getArguments().get(0).getValue();
+ if (arg.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression ref = (VariableReferenceExpression) arg;
+ LogicalVariable var = ref.getVariableReference();
+ List<LogicalVariable> keys = context.findPrimaryKey(var);
+ if (keys != null) {
+ List<LogicalVariable> tail = new ArrayList<LogicalVariable>();
+ tail.add(v);
+ FunctionalDependency pk = new FunctionalDependency(keys, tail);
+ context.addPrimaryKey(pk);
+ }
+ }
+ exprRef.setValue(new VariableReferenceExpression(v));
+ return true;
+ } else {
+ boolean pushed = false;
+ for (Mutable<ILogicalExpression> argRef : f.getArguments()) {
+ if (pushFieldLoads(argRef, topOp, context)) {
+ pushed = true;
+ }
+ }
+ return pushed;
+ }
+ }
+ case CONSTANT:
+ case VARIABLE: {
+ return false;
+ }
+ default: {
+ assert false;
+ throw new IllegalArgumentException();
+ }
+ }
+ }
+
+ private static void pushFieldAssign(AssignOperator a2, AbstractLogicalOperator topOp, IOptimizationContext context)
+ throws AlgebricksException {
+ if (topOp.getInputs().size() == 1 && !topOp.hasNestedPlans()) {
+ Mutable<ILogicalOperator> topChild = topOp.getInputs().get(0);
+ // plugAccessAboveOp(a2, topChild, context);
+ List<Mutable<ILogicalOperator>> a2InptList = a2.getInputs();
+ a2InptList.clear();
+ a2InptList.add(topChild);
+ // and link it as child in the op. tree
+ topOp.getInputs().set(0, new MutableObject<ILogicalOperator>(a2));
+ findAndEliminateRedundantFieldAccess(a2);
+ } else { // e.g., a join
+ LinkedList<LogicalVariable> usedInAccess = new LinkedList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(a2, usedInAccess);
+
+ LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(topOp, produced2);
+
+ if (OperatorPropertiesUtil.disjoint(produced2, usedInAccess)) {
+ for (Mutable<ILogicalOperator> inp : topOp.getInputs()) {
+ HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
+ VariableUtilities.getLiveVariables(inp.getValue(), v2);
+ if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
+ pushAccessAboveOpRef(a2, inp, context);
+ return;
+ }
+ }
+ if (topOp.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) topOp;
+ for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+ HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
+ VariableUtilities.getLiveVariables(root.getValue(), v2);
+ if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
+ pushAccessAboveOpRef(a2, root, context);
+ return;
+ }
+ }
+ }
+ }
+ throw new AsterixRuntimeException("Field access " + getFirstExpr(a2)
+ + " does not correspond to any input of operator " + topOp);
+ }
+ }
+ }
+
+ /**
+ * Pushes one field-access assignment above toPushThroughChildRef
+ *
+ * @param toPush
+ * @param toPushThroughChildRef
+ */
+ private static void pushAccessAboveOpRef(AssignOperator toPush, Mutable<ILogicalOperator> toPushThroughChildRef,
+ IOptimizationContext context) throws AlgebricksException {
+ List<Mutable<ILogicalOperator>> tpInpList = toPush.getInputs();
+ tpInpList.clear();
+ tpInpList.add(new MutableObject<ILogicalOperator>(toPushThroughChildRef.getValue()));
+ toPushThroughChildRef.setValue(toPush);
+ findAndEliminateRedundantFieldAccess(toPush);
+ }
+
+ /**
+ * Rewrite
+ * assign $x := field-access($y, "field")
+ * assign $y := record-constructor { "field": Expr, ... }
+ * into
+ * assign $x := Expr
+ * assign $y := record-constructor { "field": Expr, ... }
+ *
+ * @param toPush
+ */
+ private static boolean findAndEliminateRedundantFieldAccess(AssignOperator assign) throws AlgebricksException {
+ ILogicalExpression expr = getFirstExpr(assign);
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
+ ILogicalExpression arg0 = f.getArguments().get(0).getValue();
+ if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+ VariableReferenceExpression vre = (VariableReferenceExpression) arg0;
+ LogicalVariable recordVar = vre.getVariableReference();
+ ILogicalExpression arg1 = f.getArguments().get(1).getValue();
+ if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return false;
+ }
+ ConstantExpression ce = (ConstantExpression) arg1;
+ if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
+ String fldName = ((AString) ((AsterixConstantValue) ce.getValue()).getObject()).getStringValue();
+ ILogicalExpression fldExpr = findFieldExpression(assign, recordVar, fldName);
+
+ if (fldExpr != null) {
+ // check the liveness of the new expression
+ List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+ fldExpr.getUsedVariables(usedVariables);
+ List<LogicalVariable> liveInputVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(assign, liveInputVars);
+ usedVariables.removeAll(liveInputVars);
+ if (usedVariables.size() == 0) {
+ assign.getExpressions().get(0).setValue(fldExpr);
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ } else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX)) {
+ // int fldIdx = ((IntegerLiteral) ce.getValue()).getValue();
+ // TODO
+ return false;
+ } else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED)) {
+ return false;
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+
+ private static ILogicalExpression findFieldExpression(AbstractLogicalOperator op, LogicalVariable recordVar,
+ String fldName) {
+ for (Mutable<ILogicalOperator> child : op.getInputs()) {
+ AbstractLogicalOperator opChild = (AbstractLogicalOperator) child.getValue();
+ if (opChild.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator op2 = (AssignOperator) opChild;
+ int i = 0;
+ for (LogicalVariable var : op2.getVariables()) {
+ if (var == recordVar) {
+ AbstractLogicalExpression constr = (AbstractLogicalExpression) op2.getExpressions().get(i)
+ .getValue();
+ if (constr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return null;
+ }
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) constr;
+ if (!fce.getFunctionIdentifier().equals(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)
+ && !fce.getFunctionIdentifier().equals(
+ AsterixBuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR)) {
+ return null;
+ }
+ Iterator<Mutable<ILogicalExpression>> fldIter = fce.getArguments().iterator();
+ while (fldIter.hasNext()) {
+ ILogicalExpression fldExpr = fldIter.next().getValue();
+ if (fldExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ ConstantExpression ce = (ConstantExpression) fldExpr;
+ String f2 = ((AString) ((AsterixConstantValue) ce.getValue()).getObject())
+ .getStringValue();
+ if (fldName.equals(f2)) {
+ return fldIter.next().getValue();
+ }
+ }
+ fldIter.next();
+ }
+ return null;
+ }
+ i++;
+ }
+ } else if (opChild.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ NestedTupleSourceOperator nts = (NestedTupleSourceOperator) opChild;
+ AbstractLogicalOperator opBelowNestedPlan = (AbstractLogicalOperator) nts.getDataSourceReference()
+ .getValue().getInputs().get(0).getValue();
+ ILogicalExpression expr1 = findFieldExpression(opBelowNestedPlan, recordVar, fldName);
+ if (expr1 != null) {
+ return expr1;
+ }
+ }
+ ILogicalExpression expr2 = findFieldExpression(opChild, recordVar, fldName);
+ if (expr2 != null) {
+ return expr2;
+ }
+ }
+ return null;
+ }
+
+ private final class ExtractFieldLoadExpressionVisitor implements ILogicalExpressionReferenceTransform {
+
+ private AbstractLogicalOperator topOp;
+ private IOptimizationContext context;
+
+ public void setTopOp(AbstractLogicalOperator topOp) {
+ this.topOp = topOp;
+ }
+
+ public void setContext(IOptimizationContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+ return pushFieldLoads(exprRef, topOp, context);
+ }
+
+ }
+
+ private static ILogicalExpression getFirstExpr(AssignOperator assign) {
+ return assign.getExpressions().get(0).getValue();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java
new file mode 100644
index 0000000..3ae35bc
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class NestGroupByRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ SubplanOperator subplan = (SubplanOperator) op1;
+ if (subplan.getNestedPlans().size() != 1) {
+ return false;
+ }
+ ILogicalPlan p = subplan.getNestedPlans().get(0);
+ if (p.getRoots().size() != 1) {
+ return false;
+ }
+
+ Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
+ if (free.size() != 1) {
+ return false;
+ }
+ LogicalVariable fVar = null;
+ for (LogicalVariable v : free) {
+ fVar = v;
+ break;
+ }
+
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ GroupByOperator gby = (GroupByOperator) op2;
+ if (gby.getNestedPlans().size() != 1) {
+ return false;
+ }
+ ILogicalPlan p2 = gby.getNestedPlans().get(0);
+ if (p2.getRoots().size() != 1) {
+ return false;
+ }
+ Mutable<ILogicalOperator> r2 = p2.getRoots().get(0);
+ AbstractLogicalOperator opr2 = (AbstractLogicalOperator) r2.getValue();
+ if (opr2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AggregateOperator aggOuter = (AggregateOperator) opr2;
+ int posInAggList = aggOuter.getVariables().indexOf(fVar);
+ if (posInAggList < 0) {
+ return false;
+ }
+ AbstractLogicalOperator outerAggSon = (AbstractLogicalOperator) aggOuter.getInputs().get(0).getValue();
+ if (outerAggSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ return false;
+ }
+ ILogicalExpression eAgg = aggOuter.getExpressions().get(posInAggList).getValue();
+ if (eAgg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression listifyCall = (AbstractFunctionCallExpression) eAgg;
+ if (listifyCall.getFunctionIdentifier() != AsterixBuiltinFunctions.LISTIFY) {
+ return false;
+ }
+ ILogicalExpression argListify = listifyCall.getArguments().get(0).getValue();
+ if (argListify.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> r = p.getRoots().get(0);
+ AbstractLogicalOperator opInS = (AbstractLogicalOperator) r.getValue();
+ if (opInS.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AggregateOperator aggInner = (AggregateOperator) opInS;
+ do {
+ opInS = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
+ } while (opInS.getOperatorTag() == LogicalOperatorTag.ASSIGN);
+ if (opInS.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ AbstractLogicalOperator unnestParent = opInS;
+ AbstractLogicalOperator opUnder = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
+ // skip Assigns
+ while (opUnder.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ unnestParent = opUnder;
+ opUnder = (AbstractLogicalOperator) opUnder.getInputs().get(0).getValue();
+ }
+ if (opUnder.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+ return false;
+ }
+ UnnestOperator unnest = (UnnestOperator) opUnder;
+ AbstractLogicalOperator unnestSon = (AbstractLogicalOperator) unnest.getInputs().get(0).getValue();
+ if (unnestSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ return false;
+ }
+ NestedTupleSourceOperator innerNts = (NestedTupleSourceOperator) unnestSon;
+
+ ILogicalExpression eUnnest = unnest.getExpressionRef().getValue();
+ if (eUnnest.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression uf = (AbstractFunctionCallExpression) eUnnest;
+ if (uf.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) {
+ return false;
+ }
+ ILogicalExpression scanArg = uf.getArguments().get(0).getValue();
+ if (scanArg.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+ if (((VariableReferenceExpression) scanArg).getVariableReference() != fVar) {
+ return false;
+ }
+ LogicalVariable uVar = unnest.getVariable();
+ GroupByOperator innerGby = (GroupByOperator) opInS;
+ Set<LogicalVariable> freeInInnerGby = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSubplans(innerGby, freeInInnerGby);
+ for (LogicalVariable v : freeInInnerGby) {
+ if (v != uVar) {
+ return false;
+ }
+ }
+
+ unnestParent.getInputs().get(0).setValue(innerNts);
+ LogicalVariable listifiedVar = ((VariableReferenceExpression) argListify).getVariableReference();
+ substInSubplan(aggInner, uVar, listifiedVar, context);
+ gby.getNestedPlans().add(p);
+ innerNts.getDataSourceReference().setValue(gby);
+ opRef.setValue(gby);
+ OperatorPropertiesUtil.typePlan(p, context);
+ OperatorPropertiesUtil.typePlan(p2, context);
+ context.computeAndSetTypeEnvironmentForOperator(gby);
+ return true;
+
+ }
+
+ private void substInSubplan(AggregateOperator aggInner, LogicalVariable v1, LogicalVariable v2,
+ IOptimizationContext context) throws AlgebricksException {
+ ILogicalOperator op = aggInner;
+ while (op.getInputs().size() == 1) {
+ VariableUtilities.substituteVariables(op, v1, v2, context);
+ op = op.getInputs().get(0).getValue();
+ }
+ }
+}