You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/08/25 18:44:30 UTC
[42/51] [partial] incubator-asterixdb git commit: Change folder
structure for Java repackage
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
deleted file mode 100644
index b6d07a3..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ /dev/null
@@ -1,747 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules.am;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.dataflow.data.common.AqlExpressionTypeComputer;
-import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
-import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
-import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
-import edu.uci.ics.asterix.metadata.entities.Index;
-import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
-import edu.uci.ics.asterix.om.base.AInt32;
-import edu.uci.ics.asterix.om.base.AOrderedList;
-import edu.uci.ics.asterix.om.base.AString;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.BuiltinType;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.types.hierachy.ATypeHierarchy;
-import edu.uci.ics.asterix.optimizer.rules.am.OptimizableOperatorSubTree.DataSourceType;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Class that embodies the commonalities between rewrite rules for access
- * methods.
- */
-public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRewriteRule {
-
- private AqlMetadataProvider metadataProvider;
-
- public abstract Map<FunctionIdentifier, List<IAccessMethod>> getAccessMethods();
-
- protected static void registerAccessMethod(IAccessMethod accessMethod,
- Map<FunctionIdentifier, List<IAccessMethod>> accessMethods) {
- List<FunctionIdentifier> funcs = accessMethod.getOptimizableFunctions();
- for (FunctionIdentifier funcIdent : funcs) {
- List<IAccessMethod> l = accessMethods.get(funcIdent);
- if (l == null) {
- l = new ArrayList<IAccessMethod>();
- accessMethods.put(funcIdent, l);
- }
- l.add(accessMethod);
- }
- }
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
- return false;
- }
-
- protected void setMetadataDeclarations(IOptimizationContext context) {
- metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
- }
-
- protected void fillSubTreeIndexExprs(OptimizableOperatorSubTree subTree,
- Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs, IOptimizationContext context)
- throws AlgebricksException {
- Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
- // Check applicability of indexes by access method type.
- while (amIt.hasNext()) {
- Map.Entry<IAccessMethod, AccessMethodAnalysisContext> entry = amIt.next();
- AccessMethodAnalysisContext amCtx = entry.getValue();
- // For the current access method type, map variables to applicable
- // indexes.
- fillAllIndexExprs(subTree, amCtx, context);
- }
- }
-
- protected void pruneIndexCandidates(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs)
- throws AlgebricksException {
- Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
- // Check applicability of indexes by access method type.
- while (amIt.hasNext()) {
- Map.Entry<IAccessMethod, AccessMethodAnalysisContext> entry = amIt.next();
- AccessMethodAnalysisContext amCtx = entry.getValue();
- pruneIndexCandidates(entry.getKey(), amCtx);
- // Remove access methods for which there are definitely no
- // applicable indexes.
- if (amCtx.indexExprsAndVars.isEmpty()) {
- amIt.remove();
- }
- }
- }
-
- /**
- * Simply picks the first index that it finds. TODO: Improve this decision
- * process by making it more systematic.
- */
- protected Pair<IAccessMethod, Index> chooseIndex(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
- Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
- while (amIt.hasNext()) {
- Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry = amIt.next();
- AccessMethodAnalysisContext analysisCtx = amEntry.getValue();
- Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt = analysisCtx.indexExprsAndVars.entrySet()
- .iterator();
- if (indexIt.hasNext()) {
- Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry = indexIt.next();
- // To avoid a case where the chosen access method and a chosen
- // index type is different.
- // Allowed Case: [BTreeAccessMethod , IndexType.BTREE],
- // [RTreeAccessMethod , IndexType.RTREE],
- // [InvertedIndexAccessMethod,
- // IndexType.SINGLE_PARTITION_WORD_INVIX ||
- // SINGLE_PARTITION_NGRAM_INVIX ||
- // LENGTH_PARTITIONED_WORD_INVIX ||
- // LENGTH_PARTITIONED_NGRAM_INVIX]
- IAccessMethod chosenAccessMethod = amEntry.getKey();
- Index chosenIndex = indexEntry.getKey();
- boolean isKeywordOrNgramIndexChosen = false;
- if (chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX
- || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
- || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX)
- isKeywordOrNgramIndexChosen = true;
- if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE && chosenIndex.getIndexType() != IndexType.BTREE)
- || (chosenAccessMethod == RTreeAccessMethod.INSTANCE && chosenIndex.getIndexType() != IndexType.RTREE)
- || (chosenAccessMethod == InvertedIndexAccessMethod.INSTANCE && !isKeywordOrNgramIndexChosen)) {
- continue;
- }
- return new Pair<IAccessMethod, Index>(chosenAccessMethod, chosenIndex);
- }
- }
- return null;
- }
-
- /**
- * Removes irrelevant access methods candidates, based on whether the
- * expressions in the query match those in the index. For example, some
- * index may require all its expressions to be matched, and some indexes may
- * only require a match on a prefix of fields to be applicable. This methods
- * removes all index candidates indexExprs that are definitely not
- * applicable according to the expressions involved.
- *
- * @throws AlgebricksException
- */
- public void pruneIndexCandidates(IAccessMethod accessMethod, AccessMethodAnalysisContext analysisCtx)
- throws AlgebricksException {
- Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexExprAndVarIt = analysisCtx.indexExprsAndVars
- .entrySet().iterator();
- // Used to keep track of matched expressions (added for prefix search)
- int numMatchedKeys = 0;
- ArrayList<Integer> matchedExpressions = new ArrayList<Integer>();
- while (indexExprAndVarIt.hasNext()) {
- Map.Entry<Index, List<Pair<Integer, Integer>>> indexExprAndVarEntry = indexExprAndVarIt.next();
- Index index = indexExprAndVarEntry.getKey();
- boolean allUsed = true;
- int lastFieldMatched = -1;
- boolean foundKeyField = false;
- matchedExpressions.clear();
- numMatchedKeys = 0;
-
- // Remove the candidate if the dataset is a metadata dataset and the index is secondary
- // TODO: fix the way secondary metadata indexes are implemented and remove this check
- if (accessMethod.matchPrefixIndexExprs()) {
- if (index.getDataverseName().equals(MetadataConstants.METADATA_DATAVERSE_NAME)
- && !index.isPrimaryIndex()) {
- indexExprAndVarIt.remove();
- continue;
- }
- }
- for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
- List<String> keyField = index.getKeyFieldNames().get(i);
- final IAType keyType = index.getKeyFieldTypes().get(i);
- Iterator<Pair<Integer, Integer>> exprsAndVarIter = indexExprAndVarEntry.getValue().iterator();
- while (exprsAndVarIter.hasNext()) {
- final Pair<Integer, Integer> exprAndVarIdx = exprsAndVarIter.next();
- final IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(exprAndVarIdx.first);
- // If expr is not optimizable by concrete index then remove
- // expr and continue.
- if (!accessMethod.exprIsOptimizable(index, optFuncExpr)) {
- exprsAndVarIter.remove();
- continue;
- }
- boolean typeMatch = true;
- //Prune indexes based on field types
- List<IAType> indexedTypes = new ArrayList<IAType>();
- //retrieve types of expressions joined/selected with an indexed field
- for (int j = 0; j < optFuncExpr.getNumLogicalVars(); j++)
- if (j != exprAndVarIdx.second)
- indexedTypes.add(optFuncExpr.getFieldType(j));
- //add constants in case of select
- if (indexedTypes.size() < 2 && optFuncExpr.getNumLogicalVars() == 1) {
- indexedTypes.add((IAType) AqlExpressionTypeComputer.INSTANCE.getType(new ConstantExpression(
- optFuncExpr.getConstantVal(0)), null, null));
- }
- //infer type of logicalExpr based on index keyType
- indexedTypes.add((IAType) AqlExpressionTypeComputer.INSTANCE.getType(
- optFuncExpr.getLogicalExpr(exprAndVarIdx.second), null, new IVariableTypeEnvironment() {
-
- @Override
- public Object getVarType(LogicalVariable var) throws AlgebricksException {
- if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second)))
- return keyType;
- throw new IllegalArgumentException();
- }
-
- @Override
- public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
- List<List<LogicalVariable>> correlatedNullableVariableLists)
- throws AlgebricksException {
- if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second)))
- return keyType;
- throw new IllegalArgumentException();
- }
-
- @Override
- public void setVarType(LogicalVariable var, Object type) {
- throw new IllegalArgumentException();
- }
-
- @Override
- public Object getType(ILogicalExpression expr) throws AlgebricksException {
- return AqlExpressionTypeComputer.INSTANCE.getType(expr, null, this);
- }
-
- @Override
- public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2)
- throws AlgebricksException {
- throw new IllegalArgumentException();
- }
- }));
-
- //for the case when jaccard similarity is measured between ordered & unordered lists
- boolean jaccardSimilarity = optFuncExpr.getFuncExpr().getFunctionIdentifier().getName()
- .startsWith("similarity-jaccard-check");
-
- for (int j = 0; j < indexedTypes.size(); j++)
- for (int k = j + 1; k < indexedTypes.size(); k++)
- typeMatch &= isMatched(indexedTypes.get(j), indexedTypes.get(k), jaccardSimilarity);
-
- // Check if any field name in the optFuncExpr matches.
- if (optFuncExpr.findFieldName(keyField) != -1) {
- foundKeyField = typeMatch
- && optFuncExpr.getOperatorSubTree(exprAndVarIdx.second).hasDataSourceScan();
- if (foundKeyField) {
- matchedExpressions.add(exprAndVarIdx.first);
- numMatchedKeys++;
- if (lastFieldMatched == i - 1) {
- lastFieldMatched = i;
- }
- break;
- }
- }
- }
- if (!foundKeyField) {
- allUsed = false;
- // if any expression was matched, remove the non-matched expressions, otherwise the index is unusable
- if (lastFieldMatched >= 0) {
- exprsAndVarIter = indexExprAndVarEntry.getValue().iterator();
- while (exprsAndVarIter.hasNext()) {
- if (!matchedExpressions.contains(exprsAndVarIter.next().first)) {
- exprsAndVarIter.remove();
- }
- }
- }
- break;
- }
- }
- // If the access method requires all exprs to be matched but they
- // are not, remove this candidate.
- if (!allUsed && accessMethod.matchAllIndexExprs()) {
- indexExprAndVarIt.remove();
- continue;
- }
- // A prefix of the index exprs may have been matched.
- if (accessMethod.matchPrefixIndexExprs()) {
- if (lastFieldMatched < 0) {
- indexExprAndVarIt.remove();
- continue;
- }
- }
- analysisCtx.indexNumMatchedKeys.put(index, new Integer(numMatchedKeys));
- }
- }
-
- private boolean isMatched(IAType type1, IAType type2, boolean useListDomain) throws AlgebricksException {
- if (ATypeHierarchy.isSameTypeDomain(Index.getNonNullableType(type1).first.getTypeTag(),
- Index.getNonNullableType(type2).first.getTypeTag(), useListDomain))
- return true;
- return ATypeHierarchy.canPromote(Index.getNonNullableType(type1).first.getTypeTag(),
- Index.getNonNullableType(type2).first.getTypeTag());
- }
-
- /**
- * Analyzes the given selection condition, filling analyzedAMs with
- * applicable access method types. At this point we are not yet consulting
- * the metadata whether an actual index exists or not.
- */
- protected boolean analyzeCondition(ILogicalExpression cond, List<AbstractLogicalOperator> assignsAndUnnests,
- Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
- AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) cond;
- FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
- // Don't consider optimizing a disjunctive condition with an index (too
- // complicated for now).
- if (funcIdent == AlgebricksBuiltinFunctions.OR) {
- return false;
- }
- boolean found = analyzeFunctionExpr(funcExpr, assignsAndUnnests, analyzedAMs);
- for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
- ILogicalExpression argExpr = arg.getValue();
- if (argExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- continue;
- }
- AbstractFunctionCallExpression argFuncExpr = (AbstractFunctionCallExpression) argExpr;
- boolean matchFound = analyzeFunctionExpr(argFuncExpr, assignsAndUnnests, analyzedAMs);
- found = found || matchFound;
- }
- return found;
- }
-
- /**
- * Finds applicable access methods for the given function expression based
- * on the function identifier, and an analysis of the function's arguments.
- * Updates the analyzedAMs accordingly.
- */
- protected boolean analyzeFunctionExpr(AbstractFunctionCallExpression funcExpr,
- List<AbstractLogicalOperator> assignsAndUnnests, Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
- FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
- if (funcIdent == AlgebricksBuiltinFunctions.AND) {
- return false;
- }
- // Retrieves the list of access methods that are relevant based on the
- // funcIdent.
- List<IAccessMethod> relevantAMs = getAccessMethods().get(funcIdent);
- if (relevantAMs == null) {
- return false;
- }
- boolean atLeastOneMatchFound = false;
- // Place holder for a new analysis context in case we need one.
- AccessMethodAnalysisContext newAnalysisCtx = new AccessMethodAnalysisContext();
- for (IAccessMethod accessMethod : relevantAMs) {
- AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(accessMethod);
- // Use the current place holder.
- if (analysisCtx == null) {
- analysisCtx = newAnalysisCtx;
- }
- // Analyzes the funcExpr's arguments to see if the accessMethod is
- // truly applicable.
- boolean matchFound = accessMethod.analyzeFuncExprArgs(funcExpr, assignsAndUnnests, analysisCtx);
- if (matchFound) {
- // If we've used the current new context placeholder, replace it
- // with a new one.
- if (analysisCtx == newAnalysisCtx) {
- analyzedAMs.put(accessMethod, analysisCtx);
- newAnalysisCtx = new AccessMethodAnalysisContext();
- }
- atLeastOneMatchFound = true;
- }
- }
- return atLeastOneMatchFound;
- }
-
- /**
- * Finds secondary indexes whose keys include fieldName, and adds a mapping
- * in analysisCtx.indexEsprs from that index to the a corresponding
- * optimizable function expression.
- *
- * @return true if a candidate index was added to foundIndexExprs, false
- * otherwise
- * @throws AlgebricksException
- */
- protected boolean fillIndexExprs(List<Index> datasetIndexes, List<String> fieldName, IAType fieldType,
- IOptimizableFuncExpr optFuncExpr, int matchedFuncExprIndex, int varIdx,
- OptimizableOperatorSubTree matchedSubTree, AccessMethodAnalysisContext analysisCtx)
- throws AlgebricksException {
- List<Index> indexCandidates = new ArrayList<Index>();
- // Add an index to the candidates if one of the indexed fields is
- // fieldName
- for (Index index : datasetIndexes) {
- // Need to also verify the index is pending no op
- if (index.getKeyFieldNames().contains(fieldName) && index.getPendingOp() == IMetadataEntity.PENDING_NO_OP) {
- indexCandidates.add(index);
- if (optFuncExpr.getFieldType(varIdx) == BuiltinType.ANULL
- || optFuncExpr.getFieldType(varIdx) == BuiltinType.ANY)
- optFuncExpr.setFieldType(varIdx,
- index.getKeyFieldTypes().get(index.getKeyFieldNames().indexOf(fieldName)));
- analysisCtx.addIndexExpr(matchedSubTree.dataset, index, matchedFuncExprIndex, varIdx);
- }
- }
- // No index candidates for fieldName.
- if (indexCandidates.isEmpty()) {
- return false;
- }
- return true;
- }
-
- protected void fillAllIndexExprs(OptimizableOperatorSubTree subTree, AccessMethodAnalysisContext analysisCtx,
- IOptimizationContext context) throws AlgebricksException {
- int optFuncExprIndex = 0;
- List<Index> datasetIndexes = new ArrayList<Index>();
- if (subTree.dataSourceType != DataSourceType.COLLECTION_SCAN)
- datasetIndexes = metadataProvider.getDatasetIndexes(subTree.dataset.getDataverseName(),
- subTree.dataset.getDatasetName());
- for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
- // Try to match variables from optFuncExpr to assigns or unnests.
- for (int assignOrUnnestIndex = 0; assignOrUnnestIndex < subTree.assignsAndUnnests.size(); assignOrUnnestIndex++) {
- AbstractLogicalOperator op = subTree.assignsAndUnnests.get(assignOrUnnestIndex);
- if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- AssignOperator assignOp = (AssignOperator) op;
- List<LogicalVariable> varList = assignOp.getVariables();
- for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
- LogicalVariable var = varList.get(varIndex);
- int optVarIndex = optFuncExpr.findLogicalVar(var);
- // No matching var in optFuncExpr.
- if (optVarIndex == -1) {
- continue;
- }
- // At this point we have matched the optimizable func
- // expr at optFuncExprIndex to an assigned variable.
- // Remember matching subtree.
- optFuncExpr.setOptimizableSubTree(optVarIndex, subTree);
- List<String> fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex,
- varIndex, subTree.recordType, optVarIndex, optFuncExpr.getFuncExpr().getArguments()
- .get(optVarIndex).getValue());
- if (fieldName == null) {
- continue;
- }
- IAType fieldType = (IAType) context.getOutputTypeEnvironment(assignOp).getType(
- optFuncExpr.getLogicalExpr(optVarIndex));
- // Set the fieldName in the corresponding matched
- // function expression.
- optFuncExpr.setFieldName(optVarIndex, fieldName);
- optFuncExpr.setFieldType(optVarIndex, fieldType);
-
- setTypeTag(context, subTree, optFuncExpr, optVarIndex);
- if (subTree.hasDataSource()) {
- fillIndexExprs(datasetIndexes, fieldName, fieldType, optFuncExpr, optFuncExprIndex,
- optVarIndex, subTree, analysisCtx);
- }
- }
- } else {
- UnnestOperator unnestOp = (UnnestOperator) op;
- LogicalVariable var = unnestOp.getVariable();
- int funcVarIndex = optFuncExpr.findLogicalVar(var);
- // No matching var in optFuncExpr.
- if (funcVarIndex == -1) {
- continue;
- }
- // At this point we have matched the optimizable func expr
- // at optFuncExprIndex to an unnest variable.
- // Remember matching subtree.
- optFuncExpr.setOptimizableSubTree(funcVarIndex, subTree);
- List<String> fieldName = null;
- if (subTree.dataSourceType == DataSourceType.COLLECTION_SCAN) {
- optFuncExpr.setLogicalExpr(funcVarIndex, new VariableReferenceExpression(var));
- } else {
- fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, 0,
- subTree.recordType, funcVarIndex,
- optFuncExpr.getFuncExpr().getArguments().get(funcVarIndex).getValue());
- if (fieldName == null) {
- continue;
- }
- }
- IAType fieldType = (IAType) context.getOutputTypeEnvironment(unnestOp).getType(
- optFuncExpr.getLogicalExpr(funcVarIndex));
- // Set the fieldName in the corresponding matched function
- // expression.
- optFuncExpr.setFieldName(funcVarIndex, fieldName);
- optFuncExpr.setFieldType(funcVarIndex, fieldType);
-
- setTypeTag(context, subTree, optFuncExpr, funcVarIndex);
- if (subTree.hasDataSource()) {
- fillIndexExprs(datasetIndexes, fieldName, fieldType, optFuncExpr, optFuncExprIndex,
- funcVarIndex, subTree, analysisCtx);
- }
- }
- }
-
- // Try to match variables from optFuncExpr to datasourcescan if not
- // already matched in assigns.
- List<LogicalVariable> dsVarList = subTree.getDataSourceVariables();
- for (int varIndex = 0; varIndex < dsVarList.size(); varIndex++) {
- LogicalVariable var = dsVarList.get(varIndex);
- int funcVarIndex = optFuncExpr.findLogicalVar(var);
- // No matching var in optFuncExpr.
- if (funcVarIndex == -1) {
- continue;
- }
- // The variable value is one of the partitioning fields.
- List<String> fieldName = DatasetUtils.getPartitioningKeys(subTree.dataset).get(varIndex);
- IAType fieldType = (IAType) context.getOutputTypeEnvironment(subTree.dataSourceRef.getValue())
- .getVarType(var);
- // Set the fieldName in the corresponding matched function
- // expression, and remember matching subtree.
- optFuncExpr.setFieldName(funcVarIndex, fieldName);
- optFuncExpr.setOptimizableSubTree(funcVarIndex, subTree);
- optFuncExpr.setSourceVar(funcVarIndex, var);
- optFuncExpr.setLogicalExpr(funcVarIndex, new VariableReferenceExpression(var));
- setTypeTag(context, subTree, optFuncExpr, funcVarIndex);
- if (subTree.hasDataSourceScan()) {
- fillIndexExprs(datasetIndexes, fieldName, fieldType, optFuncExpr, optFuncExprIndex, funcVarIndex,
- subTree, analysisCtx);
- }
- }
- optFuncExprIndex++;
- }
- }
-
- private void setTypeTag(IOptimizationContext context, OptimizableOperatorSubTree subTree,
- IOptimizableFuncExpr optFuncExpr, int funcVarIndex) throws AlgebricksException {
- // Set the typeTag if the type is not null
- IAType type = (IAType) context.getOutputTypeEnvironment(subTree.root).getVarType(
- optFuncExpr.getLogicalVar(funcVarIndex));
- optFuncExpr.setFieldType(funcVarIndex, type);
- }
-
- /**
- * Returns the field name corresponding to the assigned variable at
- * varIndex. Returns null if the expr at varIndex does not yield to a field
- * access function after following a set of allowed functions.
- *
- * @throws AlgebricksException
- */
- protected List<String> getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr,
- OptimizableOperatorSubTree subTree, int opIndex, int assignVarIndex, ARecordType recordType,
- int funcVarIndex, ILogicalExpression parentFuncExpr) throws AlgebricksException {
- // Get expression corresponding to opVar at varIndex.
- AbstractLogicalExpression expr = null;
- AbstractFunctionCallExpression childFuncExpr = null;
- AbstractLogicalOperator op = subTree.assignsAndUnnests.get(opIndex);
- if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- AssignOperator assignOp = (AssignOperator) op;
- expr = (AbstractLogicalExpression) assignOp.getExpressions().get(assignVarIndex).getValue();
- childFuncExpr = (AbstractFunctionCallExpression) expr;
- } else {
- UnnestOperator unnestOp = (UnnestOperator) op;
- expr = (AbstractLogicalExpression) unnestOp.getExpressionRef().getValue();
- if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return null;
- }
- childFuncExpr = (AbstractFunctionCallExpression) expr;
- if (childFuncExpr.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) {
- return null;
- }
- expr = (AbstractLogicalExpression) childFuncExpr.getArguments().get(0).getValue();
- }
- if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return null;
- }
- AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
- FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
-
- boolean isByName = false;
- boolean isFieldAccess = false;
- String fieldName = null;
- List<String> nestedAccessFieldName = null;
- int fieldIndex = -1;
- if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME) {
- ILogicalExpression nameArg = funcExpr.getArguments().get(1).getValue();
- if (nameArg.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- return null;
- }
- ConstantExpression constExpr = (ConstantExpression) nameArg;
- fieldName = ((AString) ((AsterixConstantValue) constExpr.getValue()).getObject()).getStringValue();
- isFieldAccess = true;
- isByName = true;
- } else if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
- ILogicalExpression idxArg = funcExpr.getArguments().get(1).getValue();
- if (idxArg.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- return null;
- }
- ConstantExpression constExpr = (ConstantExpression) idxArg;
- fieldIndex = ((AInt32) ((AsterixConstantValue) constExpr.getValue()).getObject()).getIntegerValue();
- isFieldAccess = true;
- } else if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_NESTED) {
- ILogicalExpression nameArg = funcExpr.getArguments().get(1).getValue();
- if (nameArg.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- return null;
- }
- ConstantExpression constExpr = (ConstantExpression) nameArg;
- AOrderedList orderedNestedFieldName = (AOrderedList) ((AsterixConstantValue) constExpr.getValue())
- .getObject();
- nestedAccessFieldName = new ArrayList<String>();
- for (int i = 0; i < orderedNestedFieldName.size(); i++) {
- nestedAccessFieldName.add(((AString) orderedNestedFieldName.getItem(i)).getStringValue());
- }
- isFieldAccess = true;
- isByName = true;
- }
- if (isFieldAccess) {
- optFuncExpr.setLogicalExpr(funcVarIndex, parentFuncExpr);
- int[] assignAndExpressionIndexes = null;
-
- //go forward through nested assigns until you find the relevant one
- for (int i = opIndex + 1; i < subTree.assignsAndUnnests.size(); i++) {
- AbstractLogicalOperator subOp = subTree.assignsAndUnnests.get(i);
- List<LogicalVariable> varList;
-
- if (subOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- //Nested was an assign
- varList = ((AssignOperator) subOp).getVariables();
- } else if (subOp.getOperatorTag() == LogicalOperatorTag.UNNEST) {
- //Nested is not an assign
- varList = ((UnnestOperator) subOp).getVariables();
- } else {
- break;
- }
-
- //Go through variables in assign to check for match
- for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
- LogicalVariable var = varList.get(varIndex);
- ArrayList<LogicalVariable> parentVars = new ArrayList<LogicalVariable>();
- expr.getUsedVariables(parentVars);
-
- if (parentVars.contains(var)) {
- //Found the variable we are looking for.
- //return assign and index of expression
- int[] returnValues = { i, varIndex };
- assignAndExpressionIndexes = returnValues;
- }
- }
- }
- if (assignAndExpressionIndexes != null && assignAndExpressionIndexes[0] > -1) {
- //We found the nested assign
-
- //Recursive call on nested assign
- List<String> parentFieldNames = getFieldNameFromSubTree(optFuncExpr, subTree,
- assignAndExpressionIndexes[0], assignAndExpressionIndexes[1], recordType, funcVarIndex,
- parentFuncExpr);
-
- if (parentFieldNames == null) {
- //Nested assign was not a field access.
- //We will not use index
- return null;
- }
-
- if (!isByName) {
- try {
- fieldName = ((ARecordType) recordType.getSubFieldType(parentFieldNames)).getFieldNames()[fieldIndex];
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- }
- optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex));
- //add fieldName to the nested fieldName, return
- if (nestedAccessFieldName != null) {
- for (int i = 0; i < nestedAccessFieldName.size(); i++) {
- parentFieldNames.add(nestedAccessFieldName.get(i));
- }
- } else {
- parentFieldNames.add(fieldName);
- }
- return (parentFieldNames);
- }
-
- optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex));
- //no nested assign, we are at the lowest level.
- if (isByName) {
- if (nestedAccessFieldName != null) {
- return nestedAccessFieldName;
- }
- return new ArrayList<String>(Arrays.asList(fieldName));
- }
- return new ArrayList<String>(Arrays.asList(recordType.getFieldNames()[fieldIndex]));
-
- }
-
- if (funcIdent != AsterixBuiltinFunctions.WORD_TOKENS && funcIdent != AsterixBuiltinFunctions.GRAM_TOKENS
- && funcIdent != AsterixBuiltinFunctions.SUBSTRING
- && funcIdent != AsterixBuiltinFunctions.SUBSTRING_BEFORE
- && funcIdent != AsterixBuiltinFunctions.SUBSTRING_AFTER
- && funcIdent != AsterixBuiltinFunctions.CREATE_POLYGON
- && funcIdent != AsterixBuiltinFunctions.CREATE_MBR
- && funcIdent != AsterixBuiltinFunctions.CREATE_RECTANGLE
- && funcIdent != AsterixBuiltinFunctions.CREATE_CIRCLE
- && funcIdent != AsterixBuiltinFunctions.CREATE_LINE
- && funcIdent != AsterixBuiltinFunctions.CREATE_POINT) {
- return null;
- }
- // We use a part of the field in edit distance computation
- if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CHECK) {
- optFuncExpr.setPartialField(true);
- }
- // We expect the function's argument to be a variable, otherwise we
- // cannot apply an index.
- ILogicalExpression argExpr = funcExpr.getArguments().get(0).getValue();
- if (argExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
- return null;
- }
- LogicalVariable curVar = ((VariableReferenceExpression) argExpr).getVariableReference();
- // We look for the assign or unnest operator that produces curVar below
- // the current operator
- for (int assignOrUnnestIndex = opIndex + 1; assignOrUnnestIndex < subTree.assignsAndUnnests.size(); assignOrUnnestIndex++) {
- AbstractLogicalOperator curOp = subTree.assignsAndUnnests.get(assignOrUnnestIndex);
- if (curOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- AssignOperator assignOp = (AssignOperator) curOp;
- List<LogicalVariable> varList = assignOp.getVariables();
- for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
- LogicalVariable var = varList.get(varIndex);
- if (var.equals(curVar)) {
- optFuncExpr.setSourceVar(funcVarIndex, var);
- return getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, varIndex, recordType,
- funcVarIndex, childFuncExpr);
- }
- }
- } else {
- UnnestOperator unnestOp = (UnnestOperator) curOp;
- LogicalVariable var = unnestOp.getVariable();
- if (var.equals(curVar)) {
- getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, 0, recordType, funcVarIndex,
- childFuncExpr);
- }
- }
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
deleted file mode 100644
index cc24912..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules.am;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.asterix.metadata.entities.Dataset;
-import edu.uci.ics.asterix.metadata.entities.Index;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-
-/**
- * Context for analyzing the applicability of a single access method.
- */
-public class AccessMethodAnalysisContext {
-
- public List<IOptimizableFuncExpr> matchedFuncExprs = new ArrayList<IOptimizableFuncExpr>();
-
- // Contains candidate indexes and a list of (integer,integer) tuples that index into matchedFuncExprs and matched variable inside this expr.
- // We are mapping from candidate indexes to a list of function expressions
- // that match one of the index's expressions.
- public Map<Index, List<Pair<Integer, Integer>>> indexExprsAndVars = new TreeMap<Index, List<Pair<Integer, Integer>>>();
-
- // Maps from index to the dataset it is indexing.
- public Map<Index, Dataset> indexDatasetMap = new TreeMap<Index, Dataset>();
-
- // Maps from an index to the number of matched fields in the query plan (for performing prefix search)
- public Map<Index, Integer> indexNumMatchedKeys = new TreeMap<Index, Integer>();
-
- // variables for resetting null placeholder for left-outer-join
- private Mutable<ILogicalOperator> lojGroupbyOpRef = null;
- private ScalarFunctionCallExpression lojIsNullFuncInGroupBy = null;
-
- public void addIndexExpr(Dataset dataset, Index index, Integer exprIndex, Integer varIndex) {
- List<Pair<Integer, Integer>> exprs = indexExprsAndVars.get(index);
- if (exprs == null) {
- exprs = new ArrayList<Pair<Integer, Integer>>();
- indexExprsAndVars.put(index, exprs);
- }
- exprs.add(new Pair<Integer, Integer>(exprIndex, varIndex));
- indexDatasetMap.put(index, dataset);
- }
-
- public List<Pair<Integer, Integer>> getIndexExprs(Index index) {
- return indexExprsAndVars.get(index);
- }
-
- public void setLOJGroupbyOpRef(Mutable<ILogicalOperator> opRef) {
- lojGroupbyOpRef = opRef;
- }
-
- public Mutable<ILogicalOperator> getLOJGroupbyOpRef() {
- return lojGroupbyOpRef;
- }
-
- public void setLOJIsNullFuncInGroupBy(ScalarFunctionCallExpression isNullFunc) {
- lojIsNullFuncInGroupBy = isNullFunc;
- }
-
- public ScalarFunctionCallExpression getLOJIsNullFuncInGroupBy() {
- return lojIsNullFuncInGroupBy;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java
deleted file mode 100644
index 31943e4..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodJobGenParams.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules.am;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.om.base.AInt32;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-
-/**
- * Helper class for reading and writing job-gen parameters for access methods to
- * and from a list of function arguments, typically of an unnest-map.
- */
-public class AccessMethodJobGenParams {
- protected String indexName;
- protected IndexType indexType;
- protected String dataverseName;
- protected String datasetName;
- protected boolean retainInput;
- protected boolean retainNull;
- protected boolean requiresBroadcast;
- protected boolean isPrimaryIndex;
-
- private final int NUM_PARAMS = 7;
-
- public AccessMethodJobGenParams() {
- }
-
- public AccessMethodJobGenParams(String indexName, IndexType indexType, String dataverseName, String datasetName,
- boolean retainInput, boolean retainNull, boolean requiresBroadcast) {
- this.indexName = indexName;
- this.indexType = indexType;
- this.dataverseName = dataverseName;
- this.datasetName = datasetName;
- this.retainInput = retainInput;
- this.retainNull = retainNull;
- this.requiresBroadcast = requiresBroadcast;
- this.isPrimaryIndex = datasetName.equals(indexName);
- }
-
- public void writeToFuncArgs(List<Mutable<ILogicalExpression>> funcArgs) {
- funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createStringConstant(indexName)));
- funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createInt32Constant(indexType.ordinal())));
- funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createStringConstant(dataverseName)));
- funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createStringConstant(datasetName)));
- funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createBooleanConstant(retainInput)));
- funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createBooleanConstant(retainNull)));
- funcArgs.add(new MutableObject<ILogicalExpression>(AccessMethodUtils.createBooleanConstant(requiresBroadcast)));
- }
-
- public void readFromFuncArgs(List<Mutable<ILogicalExpression>> funcArgs) {
- indexName = AccessMethodUtils.getStringConstant(funcArgs.get(0));
- indexType = IndexType.values()[AccessMethodUtils.getInt32Constant(funcArgs.get(1))];
- dataverseName = AccessMethodUtils.getStringConstant(funcArgs.get(2));
- datasetName = AccessMethodUtils.getStringConstant(funcArgs.get(3));
- retainInput = AccessMethodUtils.getBooleanConstant(funcArgs.get(4));
- retainNull = AccessMethodUtils.getBooleanConstant(funcArgs.get(5));
- requiresBroadcast = AccessMethodUtils.getBooleanConstant(funcArgs.get(6));
- isPrimaryIndex = datasetName.equals(indexName);
- }
-
- public String getIndexName() {
- return indexName;
- }
-
- public IndexType getIndexType() {
- return indexType;
- }
-
- public String getDataverseName() {
- return dataverseName;
- }
-
- public String getDatasetName() {
- return datasetName;
- }
-
- public boolean getRetainInput() {
- return retainInput;
- }
-
- public boolean getRetainNull() {
- return retainNull;
- }
-
- public boolean getRequiresBroadcast() {
- return requiresBroadcast;
- }
-
- protected void writeVarList(List<LogicalVariable> varList, List<Mutable<ILogicalExpression>> funcArgs) {
- Mutable<ILogicalExpression> numKeysRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
- new AsterixConstantValue(new AInt32(varList.size()))));
- funcArgs.add(numKeysRef);
- for (LogicalVariable keyVar : varList) {
- Mutable<ILogicalExpression> keyVarRef = new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(keyVar));
- funcArgs.add(keyVarRef);
- }
- }
-
- protected int readVarList(List<Mutable<ILogicalExpression>> funcArgs, int index, List<LogicalVariable> varList) {
- int numLowKeys = AccessMethodUtils.getInt32Constant(funcArgs.get(index));
- if (numLowKeys > 0) {
- for (int i = 0; i < numLowKeys; i++) {
- LogicalVariable var = ((VariableReferenceExpression) funcArgs.get(index + 1 + i).getValue())
- .getVariableReference();
- varList.add(var);
- }
- }
- return index + numLowKeys + 1;
- }
-
- protected int getNumParams() {
- return NUM_PARAMS;
- }
-
- public boolean isPrimaryIndex() {
- return isPrimaryIndex;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
deleted file mode 100644
index 73377b9..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ /dev/null
@@ -1,609 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.optimizer.rules.am;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.algebra.operators.physical.ExternalDataLookupPOperator;
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
-import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.metadata.declared.AqlSourceId;
-import edu.uci.ics.asterix.metadata.entities.Dataset;
-import edu.uci.ics.asterix.metadata.entities.Index;
-import edu.uci.ics.asterix.metadata.external.IndexingConstants;
-import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
-import edu.uci.ics.asterix.om.base.ABoolean;
-import edu.uci.ics.asterix.om.base.AInt32;
-import edu.uci.ics.asterix.om.base.AString;
-import edu.uci.ics.asterix.om.base.IAObject;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.ATypeTag;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.types.hierachy.ATypeHierarchy;
-import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-
-/**
- * Static helper functions for rewriting plans using indexes.
- */
-public class AccessMethodUtils {
-
- public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, List<Object> target)
- throws IOException, AlgebricksException {
- ARecordType recordType = (ARecordType) itemType;
- List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
- for (List<String> partitioningKey : partitioningKeys) {
- target.add(recordType.getSubFieldType(partitioningKey));
- }
- target.add(itemType);
- }
-
- public static ConstantExpression createStringConstant(String str) {
- return new ConstantExpression(new AsterixConstantValue(new AString(str)));
- }
-
- public static ConstantExpression createInt32Constant(int i) {
- return new ConstantExpression(new AsterixConstantValue(new AInt32(i)));
- }
-
- public static ConstantExpression createBooleanConstant(boolean b) {
- if (b) {
- return new ConstantExpression(new AsterixConstantValue(ABoolean.TRUE));
- } else {
- return new ConstantExpression(new AsterixConstantValue(ABoolean.FALSE));
- }
- }
-
- public static String getStringConstant(Mutable<ILogicalExpression> expr) {
- IAObject obj = ((AsterixConstantValue) ((ConstantExpression) expr.getValue()).getValue()).getObject();
- return ((AString) obj).getStringValue();
- }
-
- public static int getInt32Constant(Mutable<ILogicalExpression> expr) {
- IAObject obj = ((AsterixConstantValue) ((ConstantExpression) expr.getValue()).getValue()).getObject();
- return ((AInt32) obj).getIntegerValue();
- }
-
- public static boolean getBooleanConstant(Mutable<ILogicalExpression> expr) {
- IAObject obj = ((AsterixConstantValue) ((ConstantExpression) expr.getValue()).getValue()).getObject();
- return ((ABoolean) obj).getBoolean();
- }
-
- public static boolean analyzeFuncExprArgsForOneConstAndVar(AbstractFunctionCallExpression funcExpr,
- AccessMethodAnalysisContext analysisCtx) {
- IAlgebricksConstantValue constFilterVal = null;
- LogicalVariable fieldVar = null;
- ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
- ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
- // One of the args must be a constant, and the other arg must be a variable.
- if (arg1.getExpressionTag() == LogicalExpressionTag.CONSTANT
- && arg2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- // The arguments of contains() function are asymmetrical, we can only use index if it is on the first argument
- if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.CONTAINS) {
- return false;
- }
- ConstantExpression constExpr = (ConstantExpression) arg1;
- constFilterVal = constExpr.getValue();
- VariableReferenceExpression varExpr = (VariableReferenceExpression) arg2;
- fieldVar = varExpr.getVariableReference();
- } else if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE
- && arg2.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
- ConstantExpression constExpr = (ConstantExpression) arg2;
- constFilterVal = constExpr.getValue();
- VariableReferenceExpression varExpr = (VariableReferenceExpression) arg1;
- fieldVar = varExpr.getVariableReference();
- } else {
- return false;
- }
- OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, fieldVar, constFilterVal);
- for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
- //avoid additional optFuncExpressions in case of a join
- if (optFuncExpr.getFuncExpr().equals(funcExpr))
- return true;
- }
- analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
- return true;
- }
-
- public static boolean analyzeFuncExprArgsForTwoVars(AbstractFunctionCallExpression funcExpr,
- AccessMethodAnalysisContext analysisCtx) {
- LogicalVariable fieldVar1 = null;
- LogicalVariable fieldVar2 = null;
- ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
- ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
- if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE
- && arg2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- fieldVar1 = ((VariableReferenceExpression) arg1).getVariableReference();
- fieldVar2 = ((VariableReferenceExpression) arg2).getVariableReference();
- } else {
- return false;
- }
- OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, new LogicalVariable[] { fieldVar1,
- fieldVar2 }, null);
- for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
- //avoid additional optFuncExpressions in case of a join
- if (optFuncExpr.getFuncExpr().equals(funcExpr))
- return true;
- }
- analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
- return true;
- }
-
- public static int getNumSecondaryKeys(Index index, ARecordType recordType) throws AlgebricksException {
- switch (index.getIndexType()) {
- case BTREE:
- case SINGLE_PARTITION_WORD_INVIX:
- case SINGLE_PARTITION_NGRAM_INVIX:
- case LENGTH_PARTITIONED_WORD_INVIX:
- case LENGTH_PARTITIONED_NGRAM_INVIX: {
- return index.getKeyFieldNames().size();
- }
- case RTREE: {
- Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
- index.getKeyFieldNames().get(0), recordType);
- IAType keyType = keyPairType.first;
- int numDimensions = NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag());
- return numDimensions * 2;
- }
- default: {
- throw new AlgebricksException("Unknown index kind: " + index.getIndexType());
- }
- }
- }
-
- /**
- * Appends the types of the fields produced by the given secondary index to dest.
- */
- public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType recordType, Index index,
- boolean primaryKeysOnly, List<Object> dest) throws AlgebricksException {
- if (!primaryKeysOnly) {
- switch (index.getIndexType()) {
- case BTREE:
- case SINGLE_PARTITION_WORD_INVIX:
- case SINGLE_PARTITION_NGRAM_INVIX: {
- for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
- Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes()
- .get(i), index.getKeyFieldNames().get(i), recordType);
- dest.add(keyPairType.first);
- }
- break;
- }
- case RTREE: {
- Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(
- index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), recordType);
- IAType keyType = keyPairType.first;
- IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
- int numKeys = getNumSecondaryKeys(index, recordType);
- for (int i = 0; i < numKeys; i++) {
- dest.add(nestedKeyType);
- }
- break;
- }
- case LENGTH_PARTITIONED_NGRAM_INVIX:
- case LENGTH_PARTITIONED_WORD_INVIX:
- default:
- break;
- }
- }
- // Primary keys.
- if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- //add primary keys
- try {
- appendExternalRecPrimaryKeys(dataset, dest);
- } catch (AsterixException e) {
- throw new AlgebricksException(e);
- }
- } else {
- List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
- for (List<String> partitioningKey : partitioningKeys) {
- try {
- dest.add(recordType.getSubFieldType(partitioningKey));
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- }
- }
- }
-
- public static void appendSecondaryIndexOutputVars(Dataset dataset, ARecordType recordType, Index index,
- boolean primaryKeysOnly, IOptimizationContext context, List<LogicalVariable> dest)
- throws AlgebricksException {
- int numPrimaryKeys = 0;
- if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
- } else {
- numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
- }
- int numSecondaryKeys = getNumSecondaryKeys(index, recordType);
- int numVars = (primaryKeysOnly) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys;
- for (int i = 0; i < numVars; i++) {
- dest.add(context.newVar());
- }
- }
-
- public static List<LogicalVariable> getPrimaryKeyVarsFromSecondaryUnnestMap(Dataset dataset,
- ILogicalOperator unnestMapOp) {
- int numPrimaryKeys;
- if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
- } else {
- numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
- }
- List<LogicalVariable> primaryKeyVars = new ArrayList<LogicalVariable>();
- List<LogicalVariable> sourceVars = ((UnnestMapOperator) unnestMapOp).getVariables();
- // Assumes the primary keys are located at the end.
- int start = sourceVars.size() - numPrimaryKeys;
- int stop = sourceVars.size();
- for (int i = start; i < stop; i++) {
- primaryKeyVars.add(sourceVars.get(i));
- }
- return primaryKeyVars;
- }
-
- public static List<LogicalVariable> getPrimaryKeyVarsFromPrimaryUnnestMap(Dataset dataset,
- ILogicalOperator unnestMapOp) {
- int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
- List<LogicalVariable> primaryKeyVars = new ArrayList<LogicalVariable>();
- List<LogicalVariable> sourceVars = ((UnnestMapOperator) unnestMapOp).getVariables();
- // Assumes the primary keys are located at the beginning.
- for (int i = 0; i < numPrimaryKeys; i++) {
- primaryKeyVars.add(sourceVars.get(i));
- }
- return primaryKeyVars;
- }
-
- /**
- * Returns the search key expression which feeds a secondary-index search. If we are optimizing a selection query then this method returns
- * the a ConstantExpression from the first constant value in the optimizable function expression.
- * If we are optimizing a join, then this method returns the VariableReferenceExpression that should feed the secondary index probe.
- *
- * @throws AlgebricksException
- */
- public static Pair<ILogicalExpression, Boolean> createSearchKeyExpr(IOptimizableFuncExpr optFuncExpr,
- OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree)
- throws AlgebricksException {
- if (probeSubTree == null) {
- // We are optimizing a selection query. Search key is a constant.
- // Type Checking and type promotion is done here
- IAType fieldType = optFuncExpr.getFieldType(0);
- IAObject constantObj = ((AsterixConstantValue) optFuncExpr.getConstantVal(0)).getObject();
- ATypeTag constantValueTag = constantObj.getType().getTypeTag();
- // type casting applied?
- boolean typeCastingApplied = false;
- // type casting happened from real (FLOAT, DOUBLE) value -> INT value?
- boolean realTypeConvertedToIntegerType = false;
- AsterixConstantValue replacedConstantValue = null;
-
- // if the constant type and target type does not match, we do a type conversion
- if (constantValueTag != fieldType.getTypeTag()) {
- replacedConstantValue = ATypeHierarchy.getAsterixConstantValueFromNumericTypeObject(constantObj,
- fieldType.getTypeTag());
- if (replacedConstantValue != null) {
- typeCastingApplied = true;
- }
-
- // To check whether the constant is REAL values, and target field is an INT type field.
- // In this case, we need to change the search parameter. Refer to the caller section for the detail.
- switch (constantValueTag) {
- case DOUBLE:
- case FLOAT:
- switch (fieldType.getTypeTag()) {
- case INT8:
- case INT16:
- case INT32:
- case INT64:
- realTypeConvertedToIntegerType = true;
- break;
- default:
- break;
- }
- default:
- break;
- }
- }
-
- if (typeCastingApplied) {
- return new Pair<ILogicalExpression, Boolean>(new ConstantExpression(replacedConstantValue),
- realTypeConvertedToIntegerType);
- } else {
- return new Pair<ILogicalExpression, Boolean>(new ConstantExpression(optFuncExpr.getConstantVal(0)),
- false);
- }
- } else {
- // We are optimizing a join query. Determine which variable feeds the secondary index.
- if (optFuncExpr.getOperatorSubTree(0) == null || optFuncExpr.getOperatorSubTree(0) == probeSubTree) {
- return new Pair<ILogicalExpression, Boolean>(new VariableReferenceExpression(
- optFuncExpr.getLogicalVar(0)), false);
- } else {
- return new Pair<ILogicalExpression, Boolean>(new VariableReferenceExpression(
- optFuncExpr.getLogicalVar(1)), false);
- }
- }
- }
-
- /**
- * Returns the first expr optimizable by this index.
- */
- public static IOptimizableFuncExpr chooseFirstOptFuncExpr(Index chosenIndex, AccessMethodAnalysisContext analysisCtx) {
- List<Pair<Integer, Integer>> indexExprs = analysisCtx.getIndexExprs(chosenIndex);
- int firstExprIndex = indexExprs.get(0).first;
- return analysisCtx.matchedFuncExprs.get(firstExprIndex);
- }
-
- public static int chooseFirstOptFuncVar(Index chosenIndex, AccessMethodAnalysisContext analysisCtx) {
- List<Pair<Integer, Integer>> indexExprs = analysisCtx.getIndexExprs(chosenIndex);
- return indexExprs.get(0).second;
- }
-
- public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType, Index index,
- ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams, IOptimizationContext context,
- boolean outputPrimaryKeysOnly, boolean retainInput) throws AlgebricksException {
- // The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments.
- ArrayList<Mutable<ILogicalExpression>> secondaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
- jobGenParams.writeToFuncArgs(secondaryIndexFuncArgs);
- // Variables and types coming out of the secondary-index search.
- List<LogicalVariable> secondaryIndexUnnestVars = new ArrayList<LogicalVariable>();
- List<Object> secondaryIndexOutputTypes = new ArrayList<Object>();
- // Append output variables/types generated by the secondary-index search (not forwarded from input).
- appendSecondaryIndexOutputVars(dataset, recordType, index, outputPrimaryKeysOnly, context,
- secondaryIndexUnnestVars);
- appendSecondaryIndexTypes(dataset, recordType, index, outputPrimaryKeysOnly, secondaryIndexOutputTypes);
- // An index search is expressed as an unnest over an index-search function.
- IFunctionInfo secondaryIndexSearch = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH);
- UnnestingFunctionCallExpression secondaryIndexSearchFunc = new UnnestingFunctionCallExpression(
- secondaryIndexSearch, secondaryIndexFuncArgs);
- secondaryIndexSearchFunc.setReturnsUniqueValues(true);
- // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine
- // which index to use, which variables contain the index-search keys, what is the original dataset, etc.
- UnnestMapOperator secondaryIndexUnnestOp = new UnnestMapOperator(secondaryIndexUnnestVars,
- new MutableObject<ILogicalExpression>(secondaryIndexSearchFunc), secondaryIndexOutputTypes, retainInput);
- secondaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
- context.computeAndSetTypeEnvironmentForOperator(secondaryIndexUnnestOp);
- secondaryIndexUnnestOp.setExecutionMode(ExecutionMode.PARTITIONED);
- return secondaryIndexUnnestOp;
- }
-
- public static UnnestMapOperator createPrimaryIndexUnnestMap(AbstractDataSourceOperator dataSourceOp,
- Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context,
- boolean sortPrimaryKeys, boolean retainInput, boolean retainNull, boolean requiresBroadcast)
- throws AlgebricksException {
- List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
- inputOp);
- // Optionally add a sort on the primary-index keys before searching the primary index.
- OrderOperator order = null;
- if (sortPrimaryKeys) {
- order = new OrderOperator();
- for (LogicalVariable pkVar : primaryKeyVars) {
- Mutable<ILogicalExpression> vRef = new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(pkVar));
- order.getOrderExpressions().add(
- new Pair<IOrder, Mutable<ILogicalExpression>>(OrderOperator.ASC_ORDER, vRef));
- }
- // The secondary-index search feeds into the sort.
- order.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
- order.setExecutionMode(ExecutionMode.LOCAL);
- context.computeAndSetTypeEnvironmentForOperator(order);
- }
- // The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments.
- List<Mutable<ILogicalExpression>> primaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
- BTreeJobGenParams jobGenParams = new BTreeJobGenParams(dataset.getDatasetName(), IndexType.BTREE,
- dataset.getDataverseName(), dataset.getDatasetName(), retainInput, retainNull, requiresBroadcast);
- // Set low/high inclusive to true for a point lookup.
- jobGenParams.setLowKeyInclusive(true);
- jobGenParams.setHighKeyInclusive(true);
- jobGenParams.setLowKeyVarList(primaryKeyVars, 0, primaryKeyVars.size());
- jobGenParams.setHighKeyVarList(primaryKeyVars, 0, primaryKeyVars.size());
- jobGenParams.setIsEqCondition(true);
- jobGenParams.writeToFuncArgs(primaryIndexFuncArgs);
- // Variables and types coming out of the primary-index search.
- List<LogicalVariable> primaryIndexUnnestVars = new ArrayList<LogicalVariable>();
- List<Object> primaryIndexOutputTypes = new ArrayList<Object>();
- // Append output variables/types generated by the primary-index search (not forwarded from input).
- primaryIndexUnnestVars.addAll(dataSourceOp.getVariables());
- try {
- appendPrimaryIndexTypes(dataset, recordType, primaryIndexOutputTypes);
- } catch (IOException e) {
- throw new AlgebricksException(e);
- }
- // An index search is expressed as an unnest over an index-search function.
- IFunctionInfo primaryIndexSearch = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH);
- AbstractFunctionCallExpression primaryIndexSearchFunc = new ScalarFunctionCallExpression(primaryIndexSearch,
- primaryIndexFuncArgs);
- // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine
- // which index to use, which variables contain the index-search keys, what is the original dataset, etc.
- UnnestMapOperator primaryIndexUnnestOp = new UnnestMapOperator(primaryIndexUnnestVars,
- new MutableObject<ILogicalExpression>(primaryIndexSearchFunc), primaryIndexOutputTypes, retainInput);
- // Fed by the order operator or the secondaryIndexUnnestOp.
- if (sortPrimaryKeys) {
- primaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
- } else {
- primaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
- }
- context.computeAndSetTypeEnvironmentForOperator(primaryIndexUnnestOp);
- primaryIndexUnnestOp.setExecutionMode(ExecutionMode.PARTITIONED);
- return primaryIndexUnnestOp;
- }
-
- public static ScalarFunctionCallExpression findLOJIsNullFuncInGroupBy(GroupByOperator lojGroupbyOp)
- throws AlgebricksException {
- //find IS_NULL function of which argument has the nullPlaceholder variable in the nested plan of groupby.
- ALogicalPlanImpl subPlan = (ALogicalPlanImpl) lojGroupbyOp.getNestedPlans().get(0);
- Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0);
- AbstractLogicalOperator subPlanRootOp = (AbstractLogicalOperator) subPlanRootOpRef.getValue();
- boolean foundSelectNonNull = false;
- ScalarFunctionCallExpression isNullFuncExpr = null;
- AbstractLogicalOperator inputOp = subPlanRootOp;
- while (inputOp != null) {
- if (inputOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
- SelectOperator selectOp = (SelectOperator) inputOp;
- if (selectOp.getCondition().getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- if (((AbstractFunctionCallExpression) selectOp.getCondition().getValue()).getFunctionIdentifier()
- .equals(AlgebricksBuiltinFunctions.NOT)) {
- ScalarFunctionCallExpression notFuncExpr = (ScalarFunctionCallExpression) selectOp
- .getCondition().getValue();
- if (notFuncExpr.getArguments().get(0).getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- if (((AbstractFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue())
- .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
- isNullFuncExpr = (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0)
- .getValue();
- if (isNullFuncExpr.getArguments().get(0).getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- foundSelectNonNull = true;
- break;
- }
- }
- }
- }
- }
- }
- inputOp = inputOp.getInputs().size() > 0 ? (AbstractLogicalOperator) inputOp.getInputs().get(0).getValue()
- : null;
- }
-
- if (!foundSelectNonNull) {
- throw new AlgebricksException(
- "Could not find the non-null select operator in GroupByOperator for LEFTOUTERJOIN plan optimization.");
- }
- return isNullFuncExpr;
- }
-
- public static void resetLOJNullPlaceholderVariableInGroupByOp(AccessMethodAnalysisContext analysisCtx,
- LogicalVariable newNullPlaceholderVaraible, IOptimizationContext context) throws AlgebricksException {
-
- //reset the null placeholder variable in groupby operator
- ScalarFunctionCallExpression isNullFuncExpr = analysisCtx.getLOJIsNullFuncInGroupBy();
- isNullFuncExpr.getArguments().clear();
- isNullFuncExpr.getArguments().add(
- new MutableObject<ILogicalExpression>(new VariableReferenceExpression(newNullPlaceholderVaraible)));
-
- //recompute type environment.
- OperatorPropertiesUtil.typeOpRec(analysisCtx.getLOJGroupbyOpRef(), context);
- }
-
- // New < For external datasets indexing>
- private static void appendExternalRecTypes(Dataset dataset, IAType itemType, List<Object> target) {
- target.add(itemType);
- }
-
- private static void appendExternalRecPrimaryKeys(Dataset dataset, List<Object> target) throws AsterixException {
- int numPrimaryKeys = IndexingConstants.getRIDSize(dataset);
- for (int i = 0; i < numPrimaryKeys; i++) {
- target.add(IndexingConstants.getFieldType(i));
- }
- }
-
- private static void writeVarList(List<LogicalVariable> varList, List<Mutable<ILogicalExpression>> funcArgs) {
- Mutable<ILogicalExpression> numKeysRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
- new AsterixConstantValue(new AInt32(varList.size()))));
- funcArgs.add(numKeysRef);
- for (LogicalVariable keyVar : varList) {
- Mutable<ILogicalExpression> keyVarRef = new MutableObject<ILogicalExpression>(
- new VariableReferenceExpression(keyVar));
- funcArgs.add(keyVarRef);
- }
- }
-
- private static void addStringArg(String argument, List<Mutable<ILogicalExpression>> funcArgs) {
- Mutable<ILogicalExpression> stringRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
- new AsterixConstantValue(new AString(argument))));
- funcArgs.add(stringRef);
- }
-
- public static ExternalDataLookupOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator dataSourceOp,
- Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context,
- Index secondaryIndex, boolean retainInput, boolean retainNull) throws AlgebricksException {
- List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
- inputOp);
-
- // add a sort on the RID fields before fetching external data.
- OrderOperator order = new OrderOperator();
- for (LogicalVariable pkVar : primaryKeyVars) {
- Mutable<ILogicalExpression> vRef = new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
- pkVar));
- order.getOrderExpressions().add(
- new Pair<IOrder, Mutable<ILogicalExpression>>(OrderOperator.ASC_ORDER, vRef));
- }
- // The secondary-index search feeds into the sort.
- order.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
- order.setExecutionMode(ExecutionMode.LOCAL);
- context.computeAndSetTypeEnvironmentForOperator(order);
- List<Mutable<ILogicalExpression>> externalRIDAccessFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
- //Add dataverse and dataset to the arguments
- AccessMethodUtils.addStringArg(dataset.getDataverseName(), externalRIDAccessFuncArgs);
- AccessMethodUtils.addStringArg(dataset.getDatasetName(), externalRIDAccessFuncArgs);
- AccessMethodUtils.writeVarList(primaryKeyVars, externalRIDAccessFuncArgs);
-
- // Variables and types coming out of the external access.
- List<LogicalVariable> externalAccessByRIDVars = new ArrayList<LogicalVariable>();
- List<Object> externalAccessOutputTypes = new ArrayList<Object>();
- // Append output variables/types generated by the data scan (not forwarded from input).
- externalAccessByRIDVars.addAll(dataSourceOp.getVariables());
- appendExternalRecTypes(dataset, recordType, externalAccessOutputTypes);
-
- IFunctionInfo externalAccessByRID = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.EXTERNAL_LOOKUP);
- AbstractFunctionCallExpression externalAccessFunc = new ScalarFunctionCallExpression(externalAccessByRID,
- externalRIDAccessFuncArgs);
-
- ExternalDataLookupOperator externalLookupOp = new ExternalDataLookupOperator(externalAccessByRIDVars,
- new MutableObject<ILogicalExpression>(externalAccessFunc), externalAccessOutputTypes, retainInput,
- dataSourceOp.getDataSource());
- // Fed by the order operator or the secondaryIndexUnnestOp.
- externalLookupOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
-
- context.computeAndSetTypeEnvironmentForOperator(externalLookupOp);
- externalLookupOp.setExecutionMode(ExecutionMode.PARTITIONED);
-
- //set the physical operator
- AqlSourceId dataSourceId = new AqlSourceId(dataset.getDataverseName(), dataset.getDatasetName());
- externalLookupOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType,
- secondaryIndex, primaryKeyVars, false, retainInput, retainNull));
- return externalLookupOp;
- }
-}