You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2016/02/16 18:19:48 UTC
incubator-asterixdb git commit: ASTERIXDB-1272 Remove
ExternalLookupOperator
Repository: incubator-asterixdb
Updated Branches:
refs/heads/master bcfd410d6 -> d33d3d1a6
ASTERIXDB-1272 Remove ExternalLookupOperator
ExternalLookupOperator has been removed in this change
and UnnestMapOperator was used instead. different visitors
and methods in the UnnestMapOperator were changed to
support both external and internal data.
Change-Id: Ic9a9400586b75fd8cb3c0b8f4aeb45aee76cdb92
Reviewed-on: https://asterix-gerrit.ics.uci.edu/592
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <bu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/d33d3d1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/d33d3d1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/d33d3d1a
Branch: refs/heads/master
Commit: d33d3d1a677392fbf12478c4e1d01d42d8b0a78b
Parents: bcfd410
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Mon Feb 15 08:48:00 2016 +0300
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Tue Feb 16 09:14:45 2016 -0800
----------------------------------------------------------------------
.../physical/ExternalDataLookupPOperator.java | 16 +++----
.../asterix/optimizer/base/AnalysisUtil.java | 9 ++++
.../IntroduceInstantLockSearchCallbackRule.java | 9 ++--
.../SweepIllegalNonfunctionalFunctions.java | 7 ---
.../optimizer/rules/am/AccessMethodUtils.java | 49 ++++++++++----------
.../optimizer/rules/am/BTreeAccessMethod.java | 7 ++-
.../am/IntroduceLSMComponentFilterRule.java | 20 +++++---
.../rules/am/OptimizableOperatorSubTree.java | 6 +--
.../optimizer/rules/am/RTreeAccessMethod.java | 6 +--
.../subplan/InlineAllNtsInSubplanVisitor.java | 14 ------
...neLeftNtsInSubplanJoinFlatteningVisitor.java | 17 -------
.../SubplanSpecialFlatteningCheckVisitor.java | 6 ---
.../om/functions/AsterixBuiltinFunctions.java | 12 ++---
13 files changed, 74 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index 9b85674..ade7efd 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -45,8 +45,8 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.operators.physical.AbstractScanPOperator;
import org.apache.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
@@ -124,8 +124,8 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
throws AlgebricksException {
- ExternalDataLookupOperator edabro = (ExternalDataLookupOperator) op;
- ILogicalExpression expr = edabro.getExpressionRef().getValue();
+ UnnestMapOperator unnestMap = (UnnestMapOperator) op;
+ ILogicalExpression expr = unnestMap.getExpressionRef().getValue();
if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
throw new IllegalStateException();
}
@@ -138,19 +138,19 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
List<LogicalVariable> outputVars = new ArrayList<LogicalVariable>();
if (retainInput) {
- VariableUtilities.getLiveVariables(edabro, outputVars);
+ VariableUtilities.getLiveVariables(unnestMap, outputVars);
} else {
- VariableUtilities.getProducedVariables(edabro, outputVars);
+ VariableUtilities.getProducedVariables(unnestMap, outputVars);
}
AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = AqlMetadataProvider
.buildExternalDataLookupRuntime(builder.getJobSpec(), dataset, secondaryIndex, ridIndexes, retainInput,
typeEnv, outputVars, opSchema, context, metadataProvider, retainNull);
- builder.contributeHyracksOperator(edabro, externalLoopup.first);
+ builder.contributeHyracksOperator(unnestMap, externalLoopup.first);
builder.contributeAlgebricksPartitionConstraint(externalLoopup.first, externalLoopup.second);
- ILogicalOperator srcExchange = edabro.getInputs().get(0).getValue();
- builder.contributeGraphEdge(srcExchange, 0, edabro, 0);
+ ILogicalOperator srcExchange = unnestMap.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(srcExchange, 0, unnestMap, 0);
}
protected int[] getKeyIndexes(List<LogicalVariable> keyVarList, IOperatorSchema[] inputSchemas) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
index 0a66013..beefc46 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.asterix.metadata.declared.AqlSourceId;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.optimizer.rules.am.AccessMethodUtils;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -35,6 +36,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExp
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
public class AnalysisUtil {
/*
@@ -130,6 +132,13 @@ public class AnalysisUtil {
return new Pair<String, String>(srcId.getDataverseName(), srcId.getDatasourceName());
}
+ public static Pair<String, String> getExternalDatasetInfo(UnnestMapOperator op) throws AlgebricksException {
+ AbstractFunctionCallExpression unnestExpr = (AbstractFunctionCallExpression) op.getExpressionRef().getValue();
+ String dataverseName = AccessMethodUtils.getStringConstant(unnestExpr.getArguments().get(0));
+ String datasetName = AccessMethodUtils.getStringConstant(unnestExpr.getArguments().get(1));
+ return new Pair<String, String>(dataverseName, datasetName);
+ }
+
private static List<FunctionIdentifier> fieldAccessFunctions = new ArrayList<FunctionIdentifier>();
static {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
index 54db5d3..90fb137 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
@@ -23,14 +23,13 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
-import org.apache.commons.lang3.mutable.Mutable;
-
import org.apache.asterix.algebra.operators.CommitOperator;
import org.apache.asterix.algebra.operators.physical.BTreeSearchPOperator;
import org.apache.asterix.metadata.declared.AqlDataSource;
import org.apache.asterix.metadata.declared.AqlMetadataImplConfig;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
import org.apache.asterix.optimizer.rules.am.AccessMethodJobGenParams;
+import org.apache.commons.lang3.mutable.Mutable;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -51,7 +50,8 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class IntroduceInstantLockSearchCallbackRule implements IAlgebraicRewriteRule {
@Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
return false;
}
@@ -67,6 +67,9 @@ public class IntroduceInstantLockSearchCallbackRule implements IAlgebraicRewrite
if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
FunctionIdentifier fid = f.getFunctionIdentifier();
+ if (fid.equals(AsterixBuiltinFunctions.EXTERNAL_LOOKUP)) {
+ return;
+ }
if (!fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
throw new IllegalStateException();
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 238eaa3..097d192 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -37,7 +37,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistributeRe
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -301,12 +300,6 @@ public class SweepIllegalNonfunctionalFunctions extends AbstractExtractExprRule
public Void visitTokenizeOperator(TokenizeOperator op, Void tag) throws AlgebricksException {
return null;
}
-
- @Override
- public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
- throws AlgebricksException {
- return null;
- }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 9165506..f39d190 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -69,7 +69,6 @@ import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
@@ -600,7 +599,7 @@ public class AccessMethodUtils {
funcArgs.add(stringRef);
}
- public static ExternalDataLookupOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator dataSourceOp,
+ public static UnnestMapOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator dataSourceOp,
Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context,
Index secondaryIndex, boolean retainInput, boolean retainNull) throws AlgebricksException {
List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
@@ -618,37 +617,37 @@ public class AccessMethodUtils {
order.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
order.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(order);
- List<Mutable<ILogicalExpression>> externalRIDAccessFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
- //Add dataverse and dataset to the arguments
- AccessMethodUtils.addStringArg(dataset.getDataverseName(), externalRIDAccessFuncArgs);
- AccessMethodUtils.addStringArg(dataset.getDatasetName(), externalRIDAccessFuncArgs);
- AccessMethodUtils.writeVarList(primaryKeyVars, externalRIDAccessFuncArgs);
+ List<Mutable<ILogicalExpression>> externalLookupArgs = new ArrayList<Mutable<ILogicalExpression>>();
+ //Add dataverse to the arguments
+ AccessMethodUtils.addStringArg(dataset.getDataverseName(), externalLookupArgs);
+ //Add dataset to the arguments
+ AccessMethodUtils.addStringArg(dataset.getDatasetName(), externalLookupArgs);
+ //Add PK vars to the arguments
+ AccessMethodUtils.writeVarList(primaryKeyVars, externalLookupArgs);
// Variables and types coming out of the external access.
- List<LogicalVariable> externalAccessByRIDVars = new ArrayList<LogicalVariable>();
- List<Object> externalAccessOutputTypes = new ArrayList<Object>();
+ List<LogicalVariable> externalUnnestVars = new ArrayList<LogicalVariable>();
+ List<Object> outputTypes = new ArrayList<Object>();
// Append output variables/types generated by the data scan (not forwarded from input).
- externalAccessByRIDVars.addAll(dataSourceOp.getVariables());
- appendExternalRecTypes(dataset, recordType, externalAccessOutputTypes);
-
- IFunctionInfo externalAccessByRID = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.EXTERNAL_LOOKUP);
- AbstractFunctionCallExpression externalAccessFunc = new ScalarFunctionCallExpression(externalAccessByRID,
- externalRIDAccessFuncArgs);
-
- ExternalDataLookupOperator externalLookupOp = new ExternalDataLookupOperator(externalAccessByRIDVars,
- new MutableObject<ILogicalExpression>(externalAccessFunc), externalAccessOutputTypes, retainInput,
- dataSourceOp.getDataSource());
+ externalUnnestVars.addAll(dataSourceOp.getVariables());
+ appendExternalRecTypes(dataset, recordType, outputTypes);
+
+ IFunctionInfo externalLookup = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.EXTERNAL_LOOKUP);
+ AbstractFunctionCallExpression externalLookupFunc = new ScalarFunctionCallExpression(externalLookup,
+ externalLookupArgs);
+ UnnestMapOperator unnestOp = new UnnestMapOperator(externalUnnestVars,
+ new MutableObject<ILogicalExpression>(externalLookupFunc), outputTypes, retainInput);
// Fed by the order operator or the secondaryIndexUnnestOp.
- externalLookupOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
+ unnestOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
- context.computeAndSetTypeEnvironmentForOperator(externalLookupOp);
- externalLookupOp.setExecutionMode(ExecutionMode.PARTITIONED);
+ context.computeAndSetTypeEnvironmentForOperator(unnestOp);
+ unnestOp.setExecutionMode(ExecutionMode.PARTITIONED);
//set the physical operator
AqlSourceId dataSourceId = new AqlSourceId(dataset.getDataverseName(), dataset.getDatasetName());
- externalLookupOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType,
- secondaryIndex, primaryKeyVars, false, retainInput, retainNull));
- return externalLookupOp;
+ unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType, secondaryIndex,
+ primaryKeyVars, false, retainInput, retainNull));
+ return unnestOp;
}
//If the expression is constant at runtime, runturn the type
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 370b90d..3fdbc5c 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -60,7 +60,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogi
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
@@ -489,9 +488,9 @@ public class BTreeAccessMethod implements IAccessMethod {
boolean isPrimaryIndex = chosenIndex.isPrimaryIndex();
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
// External dataset
- ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(
- dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
- retainNull);
+ UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp,
+ dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull);
+ indexSubTree.dataSourceRef.setValue(externalDataAccessOp);
return externalDataAccessOp;
} else if (!isPrimaryIndex) {
primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
index 55ba426..94dd84e 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
@@ -230,13 +230,21 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
FunctionIdentifier fid = f.getFunctionIdentifier();
- if (!fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
- throw new IllegalStateException();
+ String dataverseName = null;
+ String datasetName = null;
+ if (AsterixBuiltinFunctions.EXTERNAL_LOOKUP.equals(fid)) {
+ dataverseName = AccessMethodUtils.getStringConstant(f.getArguments().get(0));
+ datasetName = AccessMethodUtils.getStringConstant(f.getArguments().get(1));
+ } else if (fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
+ AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
+ jobGenParams.readFromFuncArgs(f.getArguments());
+ dataverseName = jobGenParams.dataverseName;
+ datasetName = jobGenParams.datasetName;
+ } else {
+ throw new AlgebricksException("Unexpected function for Unnest Map: " + fid);
}
- AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
- jobGenParams.readFromFuncArgs(f.getArguments());
- return ((AqlMetadataProvider) context.getMetadataProvider()).findDataset(jobGenParams.dataverseName,
- jobGenParams.datasetName);
+ return ((AqlMetadataProvider) context.getMetadataProvider()).findDataset(dataverseName,
+ datasetName);
}
}
if (descendantOp.getInputs().isEmpty()) {
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
index 52582ba..39a9ab8 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
@@ -42,7 +42,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogi
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
/**
@@ -230,9 +229,8 @@ public class OptimizableOperatorSubTree {
dataverseName = jobGenParams.getDataverseName();
break;
case EXTERNAL_SCAN:
- ExternalDataLookupOperator externalScan = (ExternalDataLookupOperator) sourceOpRefs.get(i)
- .getValue();
- datasetInfo = AnalysisUtil.getDatasetInfo(externalScan);
+ UnnestMapOperator externalScan = (UnnestMapOperator) sourceOpRefs.get(i).getValue();
+ datasetInfo = AnalysisUtil.getExternalDatasetInfo(externalScan);
dataverseName = datasetInfo.first;
datasetName = datasetInfo.second;
break;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index c5bd8af..3c925e2 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -51,7 +51,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractData
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
@@ -244,9 +243,8 @@ public class RTreeAccessMethod implements IAccessMethod {
// Generate the rest of the upstream plan which feeds the search results into the primary index.
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(
- dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
- retainNull);
+ UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp,
+ dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull);
return externalDataAccessOp;
} else {
UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 0715d46..b3e07e5 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -54,7 +54,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -589,19 +588,6 @@ class InlineAllNtsInSubplanVisitor implements IQueryOperatorVisitor<ILogicalOper
}
@Override
- public ILogicalOperator visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
- throws AlgebricksException {
- visitSingleInputOperator(op);
- Set<LogicalVariable> liveVars = new HashSet<>();
- VariableUtilities.getLiveVariables(op, liveVars);
- if (!liveVars.containsAll(correlatedKeyVars)) {
- op.setPropagateInput(true);
- }
- context.computeAndSetTypeEnvironmentForOperator(op);
- return op;
- }
-
- @Override
public ILogicalOperator visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
return visitSingleInputOperator(op);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index 5aaa59f..33ed9f9 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -42,7 +42,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -365,22 +364,6 @@ class InlineLeftNtsInSubplanJoinFlatteningVisitor implements IQueryOperatorVisit
}
@Override
- public ILogicalOperator visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
- throws AlgebricksException {
- visitSingleInputOperator(op);
- if (!rewritten) {
- return op;
- }
- Set<LogicalVariable> liveVars = new HashSet<>();
- VariableUtilities.getLiveVariables(op, liveVars);
- if (!liveVars.containsAll(liveVarsFromSubplanInput)) {
- op.setPropagateInput(true);
- }
- context.computeAndSetTypeEnvironmentForOperator(op);
- return op;
- }
-
- @Override
public ILogicalOperator visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
return visitSingleInputOperator(op);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
index c691365..d2fe52f 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
@@ -28,7 +28,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOper
import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -220,11 +219,6 @@ class SubplanSpecialFlatteningCheckVisitor implements IQueryOperatorVisitor<Bool
}
@Override
- public Boolean visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
- return visitInputs(op);
- }
-
- @Override
public Boolean visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
return visitInputs(op);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d33d3d1a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
index f1b0e67..a1c4e33 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -18,6 +18,12 @@
*/
package org.apache.asterix.om.functions;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
@@ -113,12 +119,6 @@ import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFun
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
public class AsterixBuiltinFunctions {
public enum SpatialFilterKind {