You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by wa...@apache.org on 2018/02/16 19:04:19 UTC
[13/16] asterixdb git commit: [ASTERIXDB-1972][COMP][RT][TX]
index-only plan
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index 1c7330a..6f99330 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -19,8 +19,10 @@
package org.apache.asterix.optimizer.rules.am;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -53,6 +55,7 @@ import org.apache.asterix.runtime.evaluators.functions.FullTextContainsDescripto
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
@@ -70,7 +73,6 @@ import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -107,33 +109,34 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
DISJUNCTIVE
}
- private static List<FunctionIdentifier> funcIdents = new ArrayList<>();
-
- static {
- funcIdents.add(BuiltinFunctions.STRING_CONTAINS);
- // For matching similarity-check functions. For example, similarity-jaccard-check returns a list of two items,
- // and the select condition will get the first list-item and check whether it evaluates to true.
- funcIdents.add(BuiltinFunctions.GET_ITEM);
- // Full-text search function
- funcIdents.add(BuiltinFunctions.FULLTEXT_CONTAINS);
- funcIdents.add(BuiltinFunctions.FULLTEXT_CONTAINS_WO_OPTION);
- }
+ // The second boolean value tells whether the given function generates false positive results.
+ // That is, this function can produce false positive results if it is set to true.
+ // In this case, an index-search alone cannot replace the given SELECT condition and
+ // that SELECT condition needs to be applied after the index-search to get the correct results.
+ // Currently, only full-text index search does not generate false positive results.
+ private static final List<Pair<FunctionIdentifier, Boolean>> FUNC_IDENTIFIERS = Collections.unmodifiableList(
+ Arrays.asList(new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.STRING_CONTAINS, true),
+ // For matching similarity-check functions. For example, similarity-jaccard-check returns
+ // a list of two items, and the select condition will get the first list-item and
+ // check whether it evaluates to true.
+ new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.GET_ITEM, true),
+ // Full-text search function
+ new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.FULLTEXT_CONTAINS, false),
+ new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.FULLTEXT_CONTAINS_WO_OPTION, false)));
// These function identifiers are matched in this AM's analyzeFuncExprArgs(),
// and are not visible to the outside driver.
- private static HashSet<FunctionIdentifier> secondLevelFuncIdents = new HashSet<>();
-
- static {
- secondLevelFuncIdents.add(BuiltinFunctions.SIMILARITY_JACCARD_CHECK);
- secondLevelFuncIdents.add(BuiltinFunctions.EDIT_DISTANCE_CHECK);
- secondLevelFuncIdents.add(BuiltinFunctions.EDIT_DISTANCE_CONTAINS);
- }
+ private static final List<Pair<FunctionIdentifier, Boolean>> SECOND_LEVEL_FUNC_IDENTIFIERS =
+ Collections.unmodifiableList(Arrays.asList(
+ new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.SIMILARITY_JACCARD_CHECK, true),
+ new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.EDIT_DISTANCE_CHECK, true),
+ new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.EDIT_DISTANCE_CONTAINS, true)));
public static InvertedIndexAccessMethod INSTANCE = new InvertedIndexAccessMethod();
@Override
- public List<FunctionIdentifier> getOptimizableFunctions() {
- return funcIdents;
+ public List<Pair<FunctionIdentifier, Boolean>> getOptimizableFunctions() {
+ return FUNC_IDENTIFIERS;
}
@Override
@@ -224,10 +227,21 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
}
}
}
- // Check that the matched function is optimizable by this access method.
- if (!secondLevelFuncIdents.contains(matchedFuncExpr.getFunctionIdentifier())) {
+ // Checks that the matched function is optimizable by this access method.
+ boolean found = false;
+ for (Iterator<Pair<FunctionIdentifier, Boolean>> iterator = SECOND_LEVEL_FUNC_IDENTIFIERS.iterator(); iterator
+ .hasNext();) {
+ FunctionIdentifier fID = iterator.next().first;
+
+ if (fID != null && matchedFuncExpr != null && fID.equals(matchedFuncExpr.getFunctionIdentifier())) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
return false;
}
+
boolean selectMatchFound = analyzeSelectSimilarityCheckFuncExprArgs(matchedFuncExpr, assignsAndUnnests,
matchedAssignOrUnnestIndex, analysisCtx);
boolean joinMatchFound = analyzeJoinSimilarityCheckFuncExprArgs(matchedFuncExpr, assignsAndUnnests,
@@ -376,10 +390,18 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
}
@Override
- public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
- OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast,
- IOptimizationContext context) throws AlgebricksException {
+ public ILogicalOperator createIndexSearchPlan(List<Mutable<ILogicalOperator>> afterTopOpRefs,
+ Mutable<ILogicalOperator> topOpRef, Mutable<ILogicalExpression> conditionRef,
+ List<Mutable<ILogicalOperator>> assignBeforeTopOpRefs, OptimizableOperatorSubTree indexSubTree,
+ OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context,
+ LogicalVariable newNullPlaceHolderForLOJ) throws AlgebricksException {
+ // TODO: we currently do not support the index-only plan for the inverted index searches since
+ // there can be many <SK, PK> pairs for the same PK and we may see two different records with the same PK
+ // (e.g., the record is deleted and inserted with the same PK). The reason is that there are
+ // no locking processes during a secondary index DML operation. When a secondary index search can see
+ // the only one version of the record during the lifetime of a query, index-only plan can be applied.
+ boolean generateInstantTrylockResultFromIndexSearch = false;
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.getDataset();
@@ -412,6 +434,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
inputOp.getInputs().add(new MutableObject<>(
OperatorManipulationUtil.deepCopy(dataSourceScan.getInputs().get(0).getValue())));
+ context.computeAndSetTypeEnvironmentForOperator(inputOp);
inputOp.setExecutionMode(dataSourceScan.getExecutionMode());
} else {
// We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs.
@@ -420,13 +443,17 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
inputOp = (AbstractLogicalOperator) probeSubTree.getRoot();
}
jobGenParams.setKeyVarList(keyVarList);
+ // By default, we don't generate SK output for an inverted index
+ // since it doesn't contain a field value, only part of it.
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
- metaRecordType, chosenIndex, inputOp, jobGenParams, context, true, retainInput, retainNull);
+ metaRecordType, chosenIndex, inputOp, jobGenParams, context, retainInput, retainNull,
+ generateInstantTrylockResultFromIndexSearch);
- // Generate the rest of the upstream plan which feeds the search results into the primary index.
- AbstractUnnestMapOperator primaryIndexUnnestOp =
- AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset, recordType, metaRecordType,
- secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
+ // Generates the rest of the upstream plan which feeds the search results into the primary index.
+ ILogicalOperator primaryIndexUnnestOp = AccessMethodUtils.createRestOfIndexSearchPlan(afterTopOpRefs, topOpRef,
+ conditionRef, assignBeforeTopOpRefs, dataSourceScan, dataset, recordType, metaRecordType,
+ secondaryIndexUnnestOp, context, true, retainInput, retainNull, false, chosenIndex, analysisCtx,
+ indexSubTree, newNullPlaceHolderForLOJ);
return primaryIndexUnnestOp;
}
@@ -451,23 +478,24 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
public boolean applySelectPlanTransformation(List<Mutable<ILogicalOperator>> afterSelectRefs,
Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException {
- ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx,
- AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(),
- afterSelectRefs),
- false, subTree.getDataSourceRef().getValue().getInputs().get(0).getValue()
- .getExecutionMode() == ExecutionMode.UNPARTITIONED,
- context);
+ SelectOperator selectOp = (SelectOperator) selectRef.getValue();
+ ILogicalOperator indexPlanRootOp =
+ createIndexSearchPlan(afterSelectRefs, selectRef, selectOp.getCondition(),
+ subTree.getAssignsAndUnnestsRefs(),
+ subTree, null, chosenIndex, analysisCtx, false, false, subTree.getDataSourceRef().getValue()
+ .getInputs().get(0).getValue().getExecutionMode() == ExecutionMode.UNPARTITIONED,
+ context, null);
+
// Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
subTree.getDataSourceRef().setValue(indexPlanRootOp);
return true;
}
@Override
- public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef,
- OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin,
- boolean hasGroupBy) throws AlgebricksException {
- // Figure out if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
+ public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
+ Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree,
+ OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException {
Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex);
OptimizableOperatorSubTree indexSubTree;
OptimizableOperatorSubTree probeSubTree;
@@ -485,7 +513,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
// The arguments of edit-distance-contains() function are asymmetrical, we can only use index
- // if the dataset of index subtree and the dataset of first argument's subtree is the same
+ // if the dataset of index subtree and the dataset of first argument's subtree is the same.
if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == BuiltinFunctions.EDIT_DISTANCE_CONTAINS
&& optFuncExpr.getOperatorSubTree(0).getDataset() != null && !optFuncExpr.getOperatorSubTree(0)
.getDataset().getDatasetName().equals(indexSubTree.getDataset().getDatasetName())) {
@@ -500,7 +528,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
//reset the null place holder variable
- AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+ AccessMethodUtils.resetLOJMissingPlaceholderVarInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
}
AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) joinRef.getValue();
@@ -535,8 +563,9 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
probeSubTree.setRoot(newProbeRootRef.getValue());
}
// Create regular indexed-nested loop join path.
- ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, indexSubTree, probeSubTree, chosenIndex,
- analysisCtx, true, isLeftOuterJoin, true, context);
+ ILogicalOperator indexPlanRootOp = createIndexSearchPlan(afterJoinRefs, joinRef,
+ new MutableObject<ILogicalExpression>(joinCond), indexSubTree.getAssignsAndUnnestsRefs(), indexSubTree,
+ probeSubTree, chosenIndex, analysisCtx, true, isLeftOuterJoin, true, context, newNullPlaceHolderVar);
indexSubTree.getDataSourceRef().setValue(indexPlanRootOp);
// Change join into a select with the same condition.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
index 2534680..7c2edb9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
@@ -19,7 +19,9 @@
package org.apache.asterix.optimizer.rules.am;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -61,6 +63,7 @@ public class OptimizableOperatorSubTree {
EXTERNAL_SCAN,
PRIMARY_INDEX_LOOKUP,
COLLECTION_SCAN,
+ INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP,
NO_DATASOURCE
}
@@ -75,6 +78,9 @@ public class OptimizableOperatorSubTree {
private Dataset dataset = null;
private ARecordType recordType = null;
private ARecordType metaRecordType = null;
+ // Contains the field names for all assign operations in this sub-tree.
+ // This will be used for the index-only plan check.
+ private Map<LogicalVariable, List<String>> varsToFieldNameMap = new HashMap<>();
// Additional datasources can exist if IntroduceJoinAccessMethodRule has been applied.
// (E.g. There are index-nested-loop-joins in the plan.)
@@ -83,6 +89,9 @@ public class OptimizableOperatorSubTree {
private List<Dataset> ixJoinOuterAdditionalDatasets = null;
private List<ARecordType> ixJoinOuterAdditionalRecordTypes = null;
+ /**
+ * Identifies the root of the subtree and initializes the data-source, assign, and unnest information.
+ */
public boolean initFromSubTree(Mutable<ILogicalOperator> subTreeOpRef) throws AlgebricksException {
reset();
rootRef = subTreeOpRef;
@@ -160,15 +169,11 @@ public class OptimizableOperatorSubTree {
AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
jobGenParams.readFromFuncArgs(f.getArguments());
if (jobGenParams.isPrimaryIndex()) {
- if (getDataSourceRef() == null) {
- setDataSourceRef(subTreeOpRef);
- setDataSourceType(DataSourceType.PRIMARY_INDEX_LOOKUP);
- } else {
- // One datasource already exists. This is an additional datasource.
- initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
- getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.PRIMARY_INDEX_LOOKUP);
- getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
- }
+ intializeDataSourceRefAndType(DataSourceType.PRIMARY_INDEX_LOOKUP, subTreeOpRef);
+ dataSourceFound = true;
+ } else if (unnestMapOp.getGenerateCallBackProceedResultVar()) {
+ intializeDataSourceRefAndType(DataSourceType.INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP,
+ subTreeOpRef);
dataSourceFound = true;
}
} else if (f.getFunctionIdentifier().equals(BuiltinFunctions.EXTERNAL_LOOKUP)) {
@@ -213,6 +218,18 @@ public class OptimizableOperatorSubTree {
return false;
}
+ private void intializeDataSourceRefAndType(DataSourceType dsType, Mutable<ILogicalOperator> opRef) {
+ if (getDataSourceRef() == null) {
+ setDataSourceRef(opRef);
+ setDataSourceType(dsType);
+ } else {
+ // One datasource already exists. This is an additional datasource.
+ initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
+ getIxJoinOuterAdditionalDataSourceTypes().add(dsType);
+ getIxJoinOuterAdditionalDataSourceRefs().add(opRef);
+ }
+ }
+
/**
* Find the dataset corresponding to the datasource scan in the metadata.
* Also sets recordType to be the type of that dataset.
@@ -254,6 +271,7 @@ public class OptimizableOperatorSubTree {
datasetName = datasetInfo.second;
break;
case PRIMARY_INDEX_LOOKUP:
+ case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP:
AbstractUnnestOperator unnestMapOp = (AbstractUnnestOperator) sourceOpRefs.get(i).getValue();
ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
@@ -369,7 +387,7 @@ public class OptimizableOperatorSubTree {
}
/**
- * Get primary key variables from the given data-source.
+ * Gets the primary key variables from the given data-source.
*/
public void getPrimaryKeyVars(Mutable<ILogicalOperator> dataSourceRefToFetch, List<LogicalVariable> target)
throws AlgebricksException {
@@ -389,12 +407,26 @@ public class OptimizableOperatorSubTree {
primaryKeys = AccessMethodUtils.getPrimaryKeyVarsFromPrimaryUnnestMap(dataset, unnestMapOp);
target.addAll(primaryKeys);
break;
+ case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP:
+ AbstractUnnestMapOperator idxOnlyPlanUnnestMapOp =
+ (AbstractUnnestMapOperator) dataSourceRefToFetchKey.getValue();
+ List<LogicalVariable> idxOnlyPlanKeyVars = idxOnlyPlanUnnestMapOp.getVariables();
+ int indexOnlyPlanNumPrimaryKeys = dataset.getPrimaryKeys().size();
+ // The order of variables: SK, PK, the result of instantTryLock on PK.
+ // The last variable keeps the result of instantTryLock on PK.
+ // Thus, we deduct 1 to only count key variables.
+ int start = idxOnlyPlanKeyVars.size() - 1 - indexOnlyPlanNumPrimaryKeys;
+ int end = start + indexOnlyPlanNumPrimaryKeys;
+
+ for (int i = start; i < end; i++) {
+ target.add(idxOnlyPlanKeyVars.get(i));
+ }
+ break;
case EXTERNAL_SCAN:
break;
case NO_DATASOURCE:
default:
throw CompilationException.create(ErrorCode.SUBTREE_HAS_NO_DATA_SOURCE);
-
}
}
@@ -405,6 +437,11 @@ public class OptimizableOperatorSubTree {
case PRIMARY_INDEX_LOOKUP:
AbstractScanOperator scanOp = (AbstractScanOperator) getDataSourceRef().getValue();
return scanOp.getVariables();
+ case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP:
+ // This data-source doesn't have record variables.
+ List<LogicalVariable> pkVars = new ArrayList<>();
+ getPrimaryKeyVars(dataSourceRef, pkVars);
+ return pkVars;
case COLLECTION_SCAN:
return new ArrayList<>();
case NO_DATASOURCE:
@@ -422,6 +459,10 @@ public class OptimizableOperatorSubTree {
AbstractScanOperator scanOp =
(AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs().get(idx).getValue();
return scanOp.getVariables();
+ case INDEXONLY_PLAN_SECONDARY_INDEX_LOOKUP:
+ List<LogicalVariable> PKVars = new ArrayList<>();
+ getPrimaryKeyVars(ixJoinOuterAdditionalDataSourceRefs.get(idx), PKVars);
+ return PKVars;
case COLLECTION_SCAN:
return new ArrayList<>();
case NO_DATASOURCE:
@@ -539,4 +580,8 @@ public class OptimizableOperatorSubTree {
this.ixJoinOuterAdditionalRecordTypes = ixJoinOuterAdditionalRecordTypes;
}
+ public Map<LogicalVariable, List<String>> getVarsToFieldNameMap() {
+ return varsToFieldNameMap;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index fd46194..f431603 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -19,6 +19,8 @@
package org.apache.asterix.optimizer.rules.am;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
@@ -31,15 +33,19 @@ import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Quadruple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -49,27 +55,31 @@ import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
/**
* Class for helping rewrite rules to choose and apply RTree indexes.
*/
public class RTreeAccessMethod implements IAccessMethod {
- private static List<FunctionIdentifier> funcIdents = new ArrayList<>();
-
- static {
- funcIdents.add(BuiltinFunctions.SPATIAL_INTERSECT);
- }
+ // The second boolean value tells whether the given function generates false positive results.
+ // That is, this function can produce false positive results if it is set to true.
+ // In this case, an index-search alone cannot replace the given SELECT condition and
+ // that SELECT condition needs to be applied after the index-search to get the final results.
+ // In R-Tree case, depending on the parameters of the SPATIAL_INTERSECT function, it may/may not produce
+ // false positive results. Thus, we need to have one more step to check whether the SPATIAL_INTERSECT generates
+ // false positive results or not.
+ private static final List<Pair<FunctionIdentifier, Boolean>> FUNC_IDENTIFIERS = Collections.unmodifiableList(
+ Arrays.asList(new Pair<FunctionIdentifier, Boolean>(BuiltinFunctions.SPATIAL_INTERSECT, true)));
public static final RTreeAccessMethod INSTANCE = new RTreeAccessMethod();
@Override
- public List<FunctionIdentifier> getOptimizableFunctions() {
- return funcIdents;
+ public List<Pair<FunctionIdentifier, Boolean>> getOptimizableFunctions() {
+ return FUNC_IDENTIFIERS;
}
@Override
@@ -98,87 +108,110 @@ public class RTreeAccessMethod implements IAccessMethod {
public boolean applySelectPlanTransformation(List<Mutable<ILogicalOperator>> afterSelectRefs,
Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException {
+ SelectOperator selectOp = (SelectOperator) selectRef.getValue();
+ Mutable<ILogicalExpression> conditionRef = selectOp.getCondition();
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) conditionRef.getValue();
+ ARecordType recordType = subTree.getRecordType();
+
// TODO: We can probably do something smarter here based on selectivity or MBR area.
- ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(subTree, null, chosenIndex, analysisCtx,
- AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(),
- afterSelectRefs),
- false, false, context);
- if (primaryIndexUnnestOp == null) {
+ IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
+
+ int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx),
+ optFuncExpr.getFieldName(optFieldIdx), recordType);
+ if (keyPairType == null) {
return false;
}
- // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
- subTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
- return true;
- }
- @Override
- public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
- OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast,
- IOptimizationContext context) throws AlgebricksException {
- return createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex, analysisCtx, retainInput,
- retainNull, requiresBroadcast, context);
- }
+ // To check whether the given plan is an index-only plan:
+ // index-only plan possible?
+ boolean isIndexOnlyPlan = false;
- @Override
- public boolean applyJoinPlanTransformation(Mutable<ILogicalOperator> joinRef,
- OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin,
- boolean hasGroupBy) throws AlgebricksException {
- // Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left
- // side).
- Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex);
- OptimizableOperatorSubTree indexSubTree;
- OptimizableOperatorSubTree probeSubTree;
+ // secondary key field usage after the select operator
+ boolean secondaryKeyFieldUsedAfterSelectOp = false;
- // We assume that the left subtree is the outer branch and the right subtree is the inner branch.
- // This assumption holds true since we only use an index from the right subtree.
- // The following is just a sanity check.
- if (rightSubTree.hasDataSourceScan()
- && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
- indexSubTree = rightSubTree;
- probeSubTree = leftSubTree;
- } else {
+ // Whether a verification is required after the secondary index search
+ // In other words, can the chosen method generate any false positive results?
+ boolean requireVerificationAfterSIdxSearch = false;
+ Pair<Boolean, Boolean> functionFalsePositiveCheck =
+ AccessMethodUtils.canFunctionGenerateFalsePositiveResultsUsingIndex(funcExpr, FUNC_IDENTIFIERS);
+
+ if (!functionFalsePositiveCheck.first) {
return false;
}
- LogicalVariable newNullPlaceHolderVar = null;
- if (isLeftOuterJoin) {
- // get a new null place holder variable that is the first field variable of the primary key
- // from the indexSubTree's datasourceScanOp
- newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
+ // Does the given index can cover all search predicates?
+ boolean doesSIdxSearchCoverAllPredicates = false;
+
+ // Preliminary check for the index-only plan for R-Tree:
+ // If the given index is not built on a POINT or a RECTANGLE field,
+ // the query result can include false positives. And the result from secondary index search is an MBR,
+ // thus we can't construct original secondary field value to remove any false positive results.
+ if (keyPairType.first.getTypeTag() == BuiltinType.APOINT.getTypeTag()
+ || keyPairType.first.getTypeTag() == BuiltinType.ARECTANGLE.getTypeTag()) {
+ isIndexOnlyPlan = true;
+ // The following variable can be changed if a query shape is not a POINT or rectangle.
+ requireVerificationAfterSIdxSearch = false;
+ } else {
+ isIndexOnlyPlan = false;
+ requireVerificationAfterSIdxSearch = true;
}
- // TODO: We can probably do something smarter here based on selectivity or MBR area.
- ILogicalOperator primaryIndexUnnestOp = createSecondaryToPrimaryPlan(indexSubTree, probeSubTree, chosenIndex,
- analysisCtx, true, isLeftOuterJoin, true, context);
+ Quadruple<Boolean, Boolean, Boolean, Boolean> indexOnlyPlanInfo =
+ new Quadruple<>(isIndexOnlyPlan, secondaryKeyFieldUsedAfterSelectOp, requireVerificationAfterSIdxSearch,
+ doesSIdxSearchCoverAllPredicates);
+
+ Dataset dataset = subTree.getDataset();
+
+ // Is this plan an index-only plan?
+ if (isIndexOnlyPlan) {
+ if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+ AccessMethodUtils.indexOnlyPlanCheck(afterSelectRefs, selectRef, subTree, null, chosenIndex,
+ analysisCtx, context, indexOnlyPlanInfo);
+ isIndexOnlyPlan = indexOnlyPlanInfo.getFirst();
+ } else {
+ // An index on an external dataset can't be optimized for the index-only plan.
+ isIndexOnlyPlan = false;
+ indexOnlyPlanInfo.setFirst(isIndexOnlyPlan);
+ }
+ }
+
+ analysisCtx.setIndexOnlyPlanInfo(indexOnlyPlanInfo);
+
+ ILogicalOperator primaryIndexUnnestOp = createIndexSearchPlan(afterSelectRefs, selectRef,
+ selectOp.getCondition(), subTree.getAssignsAndUnnestsRefs(), subTree, null, chosenIndex, analysisCtx,
+ AccessMethodUtils.retainInputs(subTree.getDataSourceVariables(), subTree.getDataSourceRef().getValue(),
+ afterSelectRefs),
+ false, false, context, null);
+
if (primaryIndexUnnestOp == null) {
return false;
}
- if (isLeftOuterJoin && hasGroupBy) {
- // reset the null place holder variable
- AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
+ // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
+ if (!isIndexOnlyPlan || dataset.getDatasetType() == DatasetType.EXTERNAL) {
+ subTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
+ } else {
+ // If this is an index-only plan, the topmost operator returned is UNIONALL operator.
+ if (primaryIndexUnnestOp.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
+ selectRef.setValue(primaryIndexUnnestOp);
+ } else {
+ subTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
+ }
}
-
- indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
- // Change join into a select with the same condition.
- AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
- SelectOperator topSelect = new SelectOperator(joinOp.getCondition(), isLeftOuterJoin, newNullPlaceHolderVar);
- topSelect.getInputs().add(indexSubTree.getRootRef());
- topSelect.setExecutionMode(ExecutionMode.LOCAL);
- context.computeAndSetTypeEnvironmentForOperator(topSelect);
- // Replace the original join with the new subtree rooted at the select op.
- joinRef.setValue(topSelect);
return true;
}
- private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
+ @Override
+ public ILogicalOperator createIndexSearchPlan(List<Mutable<ILogicalOperator>> afterTopRefs,
+ Mutable<ILogicalOperator> topRef, Mutable<ILogicalExpression> conditionRef,
+ List<Mutable<ILogicalOperator>> assignBeforeTopRefs, OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
- boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
- throws AlgebricksException {
-
+ boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context,
+ LogicalVariable newNullPlaceHolderForLOJ) throws AlgebricksException {
+ // TODO: We can probably do something smarter here based on selectivity or MBR area.
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
+
Dataset dataset = indexSubTree.getDataset();
ARecordType recordType = indexSubTree.getRecordType();
ARecordType metaRecordType = indexSubTree.getMetaRecordType();
@@ -195,43 +228,48 @@ public class RTreeAccessMethod implements IAccessMethod {
IAType spatialType = keyPairType.first;
int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numSecondaryKeys = numDimensions * 2;
- // we made sure indexSubTree has datasource scan
+
+ Quadruple<Boolean, Boolean, Boolean, Boolean> indexOnlyPlanInfo = analysisCtx.getIndexOnlyPlanInfo();
+ boolean isIndexOnlyPlan = indexOnlyPlanInfo.getFirst();
+ // We apply index-only plan for an internal dataset.
+ boolean generateInstantTrylockResultFromIndexSearch =
+ dataset.getDatasetType() == DatasetType.INTERNAL && isIndexOnlyPlan ? true : false;
+
+ // We made sure that the indexSubTree has a datasource scan.
AbstractDataSourceOperator dataSourceOp =
(AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
RTreeJobGenParams jobGenParams = new RTreeJobGenParams(chosenIndex.getIndexName(), IndexType.RTREE,
dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
// A spatial object is serialized in the constant of the func expr we are optimizing.
// The R-Tree expects as input an MBR represented with 1 field per dimension.
- // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the
- // R-Tree expects them).
- // List of variables for the assign.
+ // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple
+ // (as the R-Tree expects them). List of variables for the assign.
ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
// List of expressions for the assign.
ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
- Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(chosenIndex,
- optFuncExpr, optFieldType, indexSubTree, probeSubTree);
- ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
+ ILogicalExpression returnedSearchKeyExpr =
+ AccessMethodUtils.createSearchKeyExpr(chosenIndex, optFuncExpr, optFieldType, probeSubTree).first;
for (int i = 0; i < numSecondaryKeys; i++) {
// The create MBR function "extracts" one field of an MBR around the given spatial object.
AbstractFunctionCallExpression createMBR =
new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR));
// Spatial object is the constant from the func expr we are optimizing.
- createMBR.getArguments().add(new MutableObject<>(searchKeyExpr));
- // The number of dimensions.
+ createMBR.getArguments().add(new MutableObject<>(returnedSearchKeyExpr));
+ // The number of dimensions
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
new ConstantExpression(new AsterixConstantValue(new AInt32(numDimensions)))));
- // Which part of the MBR to extract.
+ // Which part of the MBR to extract?
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(
new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
- // Add a variable and its expr to the lists which will be passed into an assign op.
+ // Adds a variable and its expr to the lists which will be passed into an assign op.
LogicalVariable keyVar = context.newVar();
keyVarList.add(keyVar);
keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
}
jobGenParams.setKeyVarList(keyVarList);
- // Assign operator that "extracts" the MBR fields from the func-expr constant into a tuple.
+ // Assigns an operator that "extracts" the MBR fields from the func-expr constant into a tuple.
AssignOperator assignSearchKeys = new AssignOperator(keyVarList, keyExprList);
if (probeSubTree == null) {
// We are optimizing a selection query.
@@ -242,17 +280,77 @@ public class RTreeAccessMethod implements IAccessMethod {
} else {
// We are optimizing a join, place the assign op top of the probe subtree.
assignSearchKeys.getInputs().add(probeSubTree.getRootRef());
+ assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
+ OperatorPropertiesUtil.typeOpRec(probeSubTree.getRootRef(), context);
}
+ context.computeAndSetTypeEnvironmentForOperator(assignSearchKeys);
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
- metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput, retainNull);
+ metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, retainInput, retainNull,
+ generateInstantTrylockResultFromIndexSearch);
- // Generate the rest of the upstream plan which feeds the search results into the primary index.
+ // Generates the rest of the upstream plan which feeds the search results into the primary index.
return dataset.getDatasetType() == DatasetType.EXTERNAL
- ? AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType,
- secondaryIndexUnnestOp, context, retainInput, retainNull)
- : AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType,
- secondaryIndexUnnestOp, context, true, retainInput, false, false);
+ ? AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, metaRecordType,
+ secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull)
+ : AccessMethodUtils.createRestOfIndexSearchPlan(afterTopRefs, topRef, conditionRef, assignBeforeTopRefs,
+ dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true,
+ retainInput, retainNull, false, chosenIndex, analysisCtx, indexSubTree,
+ newNullPlaceHolderForLOJ);
+ }
+
+ @Override
+ public boolean applyJoinPlanTransformation(List<Mutable<ILogicalOperator>> afterJoinRefs,
+ Mutable<ILogicalOperator> joinRef, OptimizableOperatorSubTree leftSubTree,
+ OptimizableOperatorSubTree rightSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
+ IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException {
+ AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
+ Mutable<ILogicalExpression> conditionRef = joinOp.getCondition();
+
+ AbstractFunctionCallExpression funcExpr = null;
+ if (conditionRef.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ funcExpr = (AbstractFunctionCallExpression) conditionRef.getValue();
+ }
+
+ Dataset dataset = analysisCtx.getIndexDatasetMap().get(chosenIndex);
+
+ // Determine if the index is applicable on the right (inner) side.
+ OptimizableOperatorSubTree indexSubTree = null;
+ OptimizableOperatorSubTree probeSubTree = null;
+ // We assume that the left subtree is the outer branch and the right subtree is the inner branch.
+ // This assumption holds true since we only use an index from the right subtree.
+ // The following is just a sanity check.
+ if (rightSubTree.hasDataSourceScan()
+ && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
+ indexSubTree = rightSubTree;
+ probeSubTree = leftSubTree;
+ } else {
+ return false;
+ }
+
+ LogicalVariable newNullPlaceHolderVar = null;
+ if (isLeftOuterJoin) {
+ // Gets a new null place holder variable that is the first field variable of the primary key
+ // from the indexSubTree's datasourceScanOp.
+ newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
+ }
+
+ boolean canContinue = AccessMethodUtils.setIndexOnlyPlanInfo(afterJoinRefs, joinRef, probeSubTree, indexSubTree,
+ chosenIndex, analysisCtx, context, funcExpr, FUNC_IDENTIFIERS);
+ if (!canContinue) {
+ return false;
+ }
+
+ ILogicalOperator indexSearchOp = createIndexSearchPlan(afterJoinRefs, joinRef, conditionRef,
+ indexSubTree.getAssignsAndUnnestsRefs(), indexSubTree, probeSubTree, chosenIndex, analysisCtx, true,
+ isLeftOuterJoin, true, context, newNullPlaceHolderVar);
+
+ if (indexSearchOp == null) {
+ return false;
+ }
+
+ return AccessMethodUtils.finalizeJoinPlanTransformation(afterJoinRefs, joinRef, indexSubTree, analysisCtx,
+ context, isLeftOuterJoin, hasGroupBy, indexSearchOp, newNullPlaceHolderVar, conditionRef, dataset);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/data/nontagged/customerData2.json
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/data/nontagged/customerData2.json b/asterixdb/asterix-app/data/nontagged/customerData2.json
new file mode 100644
index 0000000..7624a71
--- /dev/null
+++ b/asterixdb/asterix-app/data/nontagged/customerData2.json
@@ -0,0 +1,6 @@
+{ "cid": 775, "name": "Jodi Rotruck", "cashBack": 775, "age": null, "address": { "number": 8389, "street": "Hill St.", "city": "Mountain View" }, "lastorder": { "oid": 66, "total": 38.618626f } }
+{ "cid": 5, "name": "Jodi Alex", "cashBack": 5, "age": 19, "address": null, "lastorder": { "oid": 48, "total": 318.618626f } }
+{ "cid": 1, "name": "Mike Carey", "cashBack": 1, "address": { "number": 389, "street": "Hill St.", "city": "Mountain View" }, "lastorder": { "oid": 18, "total": 338.618626f } }
+{ "cid": 0, "name": "Mike ley", "cashBack": 0, "address": null, "lastorder": { "oid": 0258, "total": 368.618626f } }
+{ "cid": 4, "name": "Mary Carey", "cashBack": 4, "age": 12, "address": { "number": 8, "street": "Hill St.", "city": "Mountain View" }, "lastorder": { "oid": 4545, "total": 87.618626f } }
+
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 189a7e1..c15704a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -64,6 +64,7 @@ import org.apache.asterix.lang.common.statement.StartFeedStatement;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.optimizer.base.FuzzyUtils;
+import org.apache.asterix.optimizer.rules.am.AbstractIntroduceAccessMethodRule;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.asterix.translator.IStatementExecutor.Stats;
@@ -127,7 +128,7 @@ public class APIFramework {
FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME,
FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION,
FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS, "inline_with",
- "hash_merge", "output-record-type");
+ "hash_merge", "output-record-type", AbstractIntroduceAccessMethodRule.NO_INDEX_ONLY_PLAN_OPTION);
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql
new file mode 100644
index 0000000..f2c4028
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01-index-only.aql
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree
+ * : and another for secondary btree index in index subtree.
+ * : In fact, this is an index-only plan from the inner branch's perspective since only PK and SK
+ * : variables are used and returned.
+ * Issue : 730, 741
+ * Expected Res : Success
+ * Date : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int32,
+ statuses-count: int32,
+ name: string,
+ followers-count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int32,
+ countB: int32
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"count1":$t1.countA,
+"t2info": for $t2 in dataset('TweetMessages')
+ where $t1.countA /* +indexnl */= $t2.countB
+ order by $t2.tweetid
+ return {"tweetid2": $t2.tweetid,
+ "count2":$t2.countB}
+};
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql
index 38032f5..71e94e2 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/leftouterjoin-probe-pidx-with-join-btree-sidx_01.aql
@@ -17,7 +17,8 @@
* under the License.
*/
/*
- * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Description : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and
+ * : another for secondary btree index in index subtree.
* Issue : 730, 741
* Expected Res : Success
* Date : 8th May 2014
@@ -55,7 +56,7 @@ create index msgCountAIx on TweetMessages(countA) type btree;
create index msgCountBIx on TweetMessages(countB) type btree;
create index msgTextIx on TweetMessages(message-text) type keyword;
-write output to asterix_nc1:"rttest/btree-index-join_leftouterjoin-probe-pidx-with-join-btree-sidx_01.adm";
+set noindexonly 'true';
for $t1 in dataset('TweetMessages')
where $t1.tweetid < int64("10")
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql
new file mode 100644
index 0000000..08e786f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/primary-to-secondary-indexonly-plan-equi-join_01.aql
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Equi joins two datasets, Customers and Orders, based on the customer id.
+ * Given the 'indexnl' hint we expect the join to be transformed
+ * into an indexed nested-loop join using Orders' secondary index.
+ * In fact, this is an index-only plan.
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type AddressType as open {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ cashBack: int64,
+ age: int64?,
+ address: AddressType?,
+ lastorder: {
+ oid: int64,
+ total: float
+ }
+}
+
+create type OrderType as open {
+ oid: int64,
+ cid: int64,
+ orderstatus: string,
+ orderpriority: string,
+ clerk: string,
+ total: float,
+ items: [int64]
+}
+
+create dataset Customers(CustomerType) primary key cid;
+create dataset Orders(OrderType) primary key oid;
+
+create index CustomerID_idx on Orders(cid);
+
+count(
+for $c in dataset('Customers')
+for $o in dataset('Orders')
+where $c.cid /*+ indexnl */ = $o.cid
+return {"oid": $o.oid, "cid":$c.cid}
+);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp
new file mode 100644
index 0000000..15107b8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_04.sqlpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Joins three datasets.
+ * : Since the given dataset in the inner branch has a secondary index on the field
+ * : that is being joined, we expect this join to be transformed into an indexed nested-loop join.
+ * Issue : ASTERIXDB-1984
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type TestType as
+{
+ id : integer,
+ val : integer
+};
+
+create dataset testdst(TestType) primary key id;
+create dataset testdst2(TestType) primary key id;
+create dataset testdst3(TestType) primary key id;
+
+create index sec3_Idx on testdst3(val) type btree;
+
+SELECT * FROM
+testdst a JOIN testdst2 b ON a.val = b.val JOIN testdst3 c ON b.val /* +indexnl */ = c.val;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp
new file mode 100644
index 0000000..31fc953
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_05.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Joins a constant array with a dataset.
+ * : Since the given dataset in the inner branch has a secondary index on the field that is being joined,
+ * : we expect this join to be transformed into an indexed nested-loop join.
+ * Issue : ASTERIXDB-1984
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type TestType as
+{
+ id : integer,
+ val : integer
+};
+
+create dataset testdst(TestType) primary key id;
+
+create index sec_Idx on testdst (val) type btree;
+
+SELECT * FROM
+[1, 2, 3] AS bar JOIN testdst ON bar /* +indexnl */ = testdst.val;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp
new file mode 100644
index 0000000..558e172
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-equi-join_06.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Joins two datasets.
+ * : Since the given dataset in the inner branch has a secondary index on the field that is being joined,
+ * : we expect this join to be transformed into an indexed nested-loop join.
+ * Issue : ASTERIXDB-1984
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type TestType as
+{
+ id : integer,
+ val : integer
+};
+
+create dataset testdst(TestType) primary key id;
+create dataset testdst2(TestType) primary key id;
+
+create index sec_Idx on testdst2(val) type btree;
+
+SELECT * FROM
+(SELECT val, COUNT(*) FROM testdst GROUP BY val) AS bar JOIN testdst2 ON bar.val /* +indexnl */ = testdst2.val;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql
new file mode 100644
index 0000000..d001bc4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-primary-equi-join_01.aql
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ /*
+ * Description : Equi joins two datasets, Customers and Orders, based on the customer id.
+ * Given the 'indexnl' hint we expect the join to be transformed
+ * into an indexed nested-loop join using Customers' primary index.
+ * An index-only plan exists in the outer branch.
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as open {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ cashBack: int64,
+ age: int64?,
+ address: AddressType?,
+ lastorder: {
+ oid: int64,
+ total: float
+ }
+}
+
+create type OrderType as open {
+ oid: int64,
+ cid: int64,
+ orderstatus: string,
+ orderpriority: string,
+ clerk: string,
+ total: float,
+ items: [int64]
+}
+
+create dataset Customers(CustomerType) primary key cid;
+create dataset Orders(OrderType) primary key oid;
+
+create index CustomerID_idx on Orders(cid);
+
+count(
+for $o in dataset('Orders')
+for $c in dataset('Customers')
+where $o.cid < 800 and $o.cid /*+ indexnl */ = $c.cid
+return {"oid": $o.oid, "cid":$c.cid}
+);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql
new file mode 100644
index 0000000..44efb8a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Equi joins two datasets, Customers and Orders, based on the customer id.
+ * Given the 'indexnl' hint we expect the join to be transformed
+ * into an indexed nested-loop join using Orders' secondary index.
+ * Each branch (outer and inner) will be transformed as an index-only plan.
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as open {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ cashBack: int64,
+ age: int64?,
+ address: AddressType?,
+ lastorder: {
+ oid: int64,
+ total: float
+ }
+}
+
+create type OrderType as open {
+ oid: int64,
+ cid: int64,
+ orderstatus: string,
+ orderpriority: string,
+ clerk: string,
+ total: float,
+ items: [int64]
+}
+
+create dataset Customers(CustomerType) primary key cid;
+create dataset Orders(OrderType) primary key oid;
+
+create index CustomerID_idx on Orders(cid);
+create index Cashback_idx on Customers(cashBack);
+
+count(
+for $o in dataset('Orders')
+for $c in dataset('Customers')
+where $o.cid < 100000 and $o.cid /*+ indexnl */ = $c.cashBack
+return {"oid": $o.oid, "cid":$c.cid}
+);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql
new file mode 100644
index 0000000..950e451
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-non-indexonly-plan-to-secondary-indexonly-plan-equi-join_01.aql
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Equi joins two datasets, Customers and Orders, based on the customer id.
+ * Given the 'indexnl' hint we expect the join to be transformed
+ * into an indexed nested-loop join using Orders' secondary index.
+ * Inner branch will be transformed as an index-only plan.
+ * Outer branch cannot be transformed as an index-only plan as an index can't cover
+ * all search predicates even excluding the join condition.
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as open {
+ number: int64,
+ street: string,
+ city: string
+}
+
+create type CustomerType as closed {
+ cid: int64,
+ name: string,
+ cashBack: int64,
+ age: int64?,
+ address: AddressType?,
+ lastorder: {
+ oid: int64,
+ total: float
+ }
+}
+
+create type OrderType as open {
+ oid: int64,
+ cid: int64,
+ orderstatus: string,
+ orderpriority: string,
+ clerk: string,
+ total: float,
+ items: [int64]
+}
+
+create dataset Customers(CustomerType) primary key cid;
+create dataset Orders(OrderType) primary key oid;
+
+create index CustomerID_idx on Orders(cid);
+create index Cashback_idx on Customers(cashBack);
+
+count(
+for $o in dataset('Orders')
+for $c in dataset('Customers')
+where $o.cid < 100000 and $o.total >= 0 and $o.cid /*+ indexnl */ = $c.cashBack
+return {"oid": $o.oid, "cid":$c.cid}
+);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql
new file mode 100644
index 0000000..e71101a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join-index-only.aql
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Self-equi joins on a dataset using two fields - countA and countB.
+ * TweetMessages has a secondary btree index on countB, and given the 'indexnl' hint
+ * we expect the join to be transformed into an indexed nested-loop join.
+ * In fact, this is an index-only plan from the inner branch's perspective since only PK and SK
+ * variables are used and returned.
+ * Success : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends-count: int64,
+ statuses-count: int64,
+ name: string,
+ followers-count: int64
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+ countA: int64,
+ countB: int64
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid;
+
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index msgTextIx on TweetMessages(message-text) type keyword;
+
+for $t1 in dataset('TweetMessages')
+for $t2 in dataset('TweetMessages')
+let $c := $t1.countA + 20
+where $c /* +indexnl */= $t2.countB
+order by $t2.tweetid
+return {"tweetid2": $t2.tweetid, "count2":$t2.countB};
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql
index abfd197..74f2d87 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index-join/secondary-self-equi-join.aql
@@ -55,7 +55,7 @@ create index msgCountAIx on TweetMessages(countA) type btree;
create index msgCountBIx on TweetMessages(countB) type btree;
create index msgTextIx on TweetMessages(message-text) type keyword;
-write output to asterix_nc1:"rttest/btree-index-join_self-secondary-equi-join.adm";
+set noindexonly 'true';
for $t1 in dataset('TweetMessages')
for $t2 in dataset('TweetMessages')
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql
new file mode 100644
index 0000000..2599bd1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-composite-index-indexonly-plan-01.aql
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Secondary BTree Index index-only selection plan verification test
+ * : This test is intended to verify that the secondary BTree index is
+ * : used in the optimized query plan.
+ * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index.
+ * : The plan should have two paths after the secondary index-lookup.
+ * : The left path:
+ * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ...
+ * : The right path:
+ * ... -> unnest-map (sidx) -> split -> union -> ...
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+}
+
+create dataset MyData(MyRecord)
+ primary key id;
+
+create index btree_index_docid_val1 on MyData(docid,val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+for $o in dataset('MyData')
+where $o.docid < 3
+return {"pk":$o.id, "sk":$o.val1}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp
new file mode 100644
index 0000000..0be684e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01-disable-indexonly-plan.sqlpp
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Secondary BTree Index index-only selection plan verification test
+ * : The test is intended to verify that the secondary BTree index is used in the optimized query plan.
+ * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index.
+ * : The plan should have two paths after the secondary index-lookup.
+ * : The left path:
+ * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ...
+ * : The right path:
+ * ... -> unnest-map (sidx) -> split -> -> union -> ...
+ * : However, we set the "noindexonly" option to true. So, the index-only plan should not be triggered.
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+};
+
+create dataset MyData(MyRecord) primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+set noindexonly 'true';
+
+select element {"pk":o.id, "sk":o.docid}
+from MyData o
+where o.docid < 3
+order by o.id;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c3c23574/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp
new file mode 100644
index 0000000..af1a099
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/btree-index/btree-secondary-index-indexonly-plan-01.sqlpp
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description : Secondary BTree Index index-only selection plan verification test
+ * : The test is intended to verify that the secondary BTree index is used in the optimized query plan.
+ * : In this plan, we fetch PK and SK based on a select condition that utilizes a secondary index.
+ * : The plan should have two paths after the secondary index-lookup.
+ * : The left path:
+ * ... -> unnest-map (sidx) -> split -> unnest-map (pidx) -> select -> union -> ...
+ * : The right path:
+ * ... -> unnest-map (sidx) -> split -> -> union -> ...
+ * Expected Result : Success
+ *
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create type MyRecord as closed {
+ id: int64,
+ docid: int64,
+ val1: int64,
+ title: string,
+ point: point,
+ kwds: string,
+ line1: line,
+ line2: line,
+ poly1: polygon,
+ poly2: polygon,
+ rec: rectangle,
+ circle: circle
+};
+
+create dataset MyData(MyRecord) primary key id;
+
+create index btree_index_docid on MyData(docid) type btree;
+create index btree_index_val1 on MyData(val1) type btree;
+create index rtree_index_point on MyData(point) type rtree;
+create index rtree_index_rec on MyData(rec) type rtree;
+create index ngram_index_title on MyData(title) type ngram(3);
+create index keyword_index_title on MyData(title) type keyword;
+
+select element {"pk":o.id, "sk":o.docid}
+from MyData o
+where o.docid < 3
+order by o.id;