You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/08/25 18:44:31 UTC
[43/51] [partial] incubator-asterixdb git commit: Change folder
structure for Java repackage
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
deleted file mode 100644
index 65c9da8..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.algebra.operators.physical.BTreeSearchPOperator;
-import edu.uci.ics.asterix.algebra.operators.physical.InvertedIndexPOperator;
-import edu.uci.ics.asterix.algebra.operators.physical.RTreeSearchPOperator;
-import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
-import edu.uci.ics.asterix.metadata.declared.AqlSourceId;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.optimizer.rules.am.AccessMethodJobGenParams;
-import edu.uci.ics.asterix.optimizer.rules.am.BTreeJobGenParams;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ExternalGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreclusteredGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.algebricks.rewriter.util.JoinUtils;
-
-public class SetAsterixPhysicalOperatorsRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- return false;
- }
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (context.checkIfInDontApplySet(this, op)) {
- return false;
- }
-
- computeDefaultPhysicalOp(op, context);
- context.addToDontApplySet(this, op);
- return true;
- }
-
- private static void setPhysicalOperators(ILogicalPlan plan, IOptimizationContext context)
- throws AlgebricksException {
- for (Mutable<ILogicalOperator> root : plan.getRoots()) {
- computeDefaultPhysicalOp((AbstractLogicalOperator) root.getValue(), context);
- }
- }
-
- private static void computeDefaultPhysicalOp(AbstractLogicalOperator op, IOptimizationContext context)
- throws AlgebricksException {
- PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
- if (op.getOperatorTag().equals(LogicalOperatorTag.GROUP)) {
- GroupByOperator gby = (GroupByOperator) op;
- if (gby.getNestedPlans().size() == 1) {
- ILogicalPlan p0 = gby.getNestedPlans().get(0);
- if (p0.getRoots().size() == 1) {
- Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
- if (((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().equals(
- LogicalOperatorTag.AGGREGATE)) {
- AggregateOperator aggOp = (AggregateOperator) r0.getValue();
- boolean serializable = true;
- for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
- AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) exprRef.getValue();
- if (!AsterixBuiltinFunctions.isAggregateFunctionSerializable(expr.getFunctionIdentifier())) {
- serializable = false;
- break;
- }
- }
-
- if ((gby.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY) == Boolean.TRUE || gby
- .getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY) == Boolean.TRUE)) {
- boolean setToExternalGby = false;
- if (serializable) {
- // if serializable, use external group-by
- // now check whether the serialized version aggregation function has corresponding intermediate agg
- boolean hasIntermediateAgg = true;
- IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
- .getMergeAggregationExpressionFactory();
- List<LogicalVariable> originalVariables = aggOp.getVariables();
- List<Mutable<ILogicalExpression>> aggExprs = aggOp.getExpressions();
- int aggNum = aggExprs.size();
- for (int i = 0; i < aggNum; i++) {
- AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) aggExprs
- .get(i).getValue();
- AggregateFunctionCallExpression serialAggExpr = AsterixBuiltinFunctions
- .makeSerializableAggregateFunctionExpression(expr.getFunctionIdentifier(),
- expr.getArguments());
- if (mergeAggregationExpressionFactory.createMergeAggregation(
- originalVariables.get(i), serialAggExpr, context) == null) {
- hasIntermediateAgg = false;
- break;
- }
- }
-
- if (hasIntermediateAgg) {
- for (int i = 0; i < aggNum; i++) {
- AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) aggExprs
- .get(i).getValue();
- AggregateFunctionCallExpression serialAggExpr = AsterixBuiltinFunctions
- .makeSerializableAggregateFunctionExpression(
- expr.getFunctionIdentifier(), expr.getArguments());
- aggOp.getExpressions().get(i).setValue(serialAggExpr);
- }
- ExternalGroupByPOperator externalGby = new ExternalGroupByPOperator(
- gby.getGroupByList(),
- physicalOptimizationConfig.getMaxFramesExternalGroupBy(),
- physicalOptimizationConfig.getExternalGroupByTableSize());
- generateMergeAggregationExpressions(gby, context);
- op.setPhysicalOperator(externalGby);
- setToExternalGby = true;
- }
- }
-
- if (!setToExternalGby) {
- // if not serializable or no intermediate agg, use pre-clustered group-by
- List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
- List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
- for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
- ILogicalExpression expr = p.second.getValue();
- if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
- columnList.add(varRef.getVariableReference());
- }
- }
- op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList));
- }
- }
- } else if (((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().equals(
- LogicalOperatorTag.RUNNINGAGGREGATE)) {
- List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
- List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
- for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
- ILogicalExpression expr = p.second.getValue();
- if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
- columnList.add(varRef.getVariableReference());
- }
- }
- op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList));
- } else {
- throw new AlgebricksException("Unsupported nested operator within a group-by: "
- + ((AbstractLogicalOperator) (r0.getValue())).getOperatorTag().name());
- }
- }
- }
- }
- if (op.getPhysicalOperator() == null) {
- switch (op.getOperatorTag()) {
- case INNERJOIN: {
- JoinUtils.setJoinAlgorithmAndExchangeAlgo((InnerJoinOperator) op, context);
- break;
- }
- case LEFTOUTERJOIN: {
- JoinUtils.setJoinAlgorithmAndExchangeAlgo((LeftOuterJoinOperator) op, context);
- break;
- }
- case UNNEST_MAP: {
- UnnestMapOperator unnestMap = (UnnestMapOperator) op;
- ILogicalExpression unnestExpr = unnestMap.getExpressionRef().getValue();
- if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
- FunctionIdentifier fid = f.getFunctionIdentifier();
- if (!fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
- throw new IllegalStateException();
- }
- AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
- jobGenParams.readFromFuncArgs(f.getArguments());
- AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider();
- AqlSourceId dataSourceId = new AqlSourceId(jobGenParams.getDataverseName(),
- jobGenParams.getDatasetName());
- IDataSourceIndex<String, AqlSourceId> dsi = mp.findDataSourceIndex(jobGenParams.getIndexName(),
- dataSourceId);
- if (dsi == null) {
- throw new AlgebricksException("Could not find index " + jobGenParams.getIndexName()
- + " for dataset " + dataSourceId);
- }
- IndexType indexType = jobGenParams.getIndexType();
- boolean requiresBroadcast = jobGenParams.getRequiresBroadcast();
- switch (indexType) {
- case BTREE: {
- BTreeJobGenParams btreeJobGenParams = new BTreeJobGenParams();
- btreeJobGenParams.readFromFuncArgs(f.getArguments());
- op.setPhysicalOperator(new BTreeSearchPOperator(dsi, requiresBroadcast,
- btreeJobGenParams.isPrimaryIndex(), btreeJobGenParams.isEqCondition(),
- btreeJobGenParams.getLowKeyVarList(), btreeJobGenParams.getHighKeyVarList()));
- break;
- }
- case RTREE: {
- op.setPhysicalOperator(new RTreeSearchPOperator(dsi, requiresBroadcast));
- break;
- }
- case SINGLE_PARTITION_WORD_INVIX:
- case SINGLE_PARTITION_NGRAM_INVIX: {
- op.setPhysicalOperator(new InvertedIndexPOperator(dsi, requiresBroadcast, false));
- break;
- }
- case LENGTH_PARTITIONED_WORD_INVIX:
- case LENGTH_PARTITIONED_NGRAM_INVIX: {
- op.setPhysicalOperator(new InvertedIndexPOperator(dsi, requiresBroadcast, true));
- break;
- }
- default: {
- throw new NotImplementedException(indexType + " indexes are not implemented.");
- }
- }
- }
- break;
- }
- }
- }
- if (op.hasNestedPlans()) {
- AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
- for (ILogicalPlan p : nested.getNestedPlans()) {
- setPhysicalOperators(p, context);
- }
- }
- for (Mutable<ILogicalOperator> opRef : op.getInputs()) {
- computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getValue(), context);
- }
- }
-
- private static void generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context)
- throws AlgebricksException {
- if (gby.getNestedPlans().size() != 1) {
- throw new AlgebricksException(
- "External group-by currently works only for one nested plan with one root containing"
- + "an aggregate and a nested-tuple-source.");
- }
- ILogicalPlan p0 = gby.getNestedPlans().get(0);
- if (p0.getRoots().size() != 1) {
- throw new AlgebricksException(
- "External group-by currently works only for one nested plan with one root containing"
- + "an aggregate and a nested-tuple-source.");
- }
- IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
- .getMergeAggregationExpressionFactory();
- Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
- AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
- if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
- throw new AlgebricksException("The merge aggregation expression generation should not process a "
- + r0Logical.getOperatorTag() + " operator.");
- }
- AggregateOperator aggOp = (AggregateOperator) r0.getValue();
- List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
- List<LogicalVariable> aggProducedVars = aggOp.getVariables();
- int n = aggOp.getExpressions().size();
- List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
- for (int i = 0; i < n; i++) {
- ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(
- aggProducedVars.get(i), aggFuncRefs.get(i).getValue(), context);
- if (mergeExpr == null) {
- throw new AlgebricksException("The aggregation function " + aggFuncRefs.get(i).getValue()
- + " does not have a registered intermediate aggregation function.");
- }
- mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
- }
- aggOp.setMergeExpressions(mergeExpressionRefs);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java
deleted file mode 100644
index 4b9a947..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
-import edu.uci.ics.asterix.om.types.AOrderedListType;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.AUnionType;
-import edu.uci.ics.asterix.om.types.AUnorderedListType;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.AbstractConstVarFunVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * open-record-constructor() becomes closed-record-constructor() if all the
- * branches below lead to dataset scans for closed record types
- */
-
-public class SetClosedRecordConstructorsRule implements IAlgebraicRewriteRule {
-
- private SettingClosedRecordVisitor recordVisitor;
-
- public SetClosedRecordConstructorsRule() {
- this.recordVisitor = new SettingClosedRecordVisitor();
- }
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext ctx) throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (ctx.checkIfInDontApplySet(this, op)) {
- return false;
- }
- ctx.addToDontApplySet(this, op);
- this.recordVisitor.setOptimizationContext(ctx, op.computeInputTypeEnvironment(ctx));
- boolean res = op.acceptExpressionTransform(recordVisitor);
- if (res) {
- ctx.computeAndSetTypeEnvironmentForOperator(op);
- }
- return res;
- }
-
- private static class SettingClosedRecordVisitor extends AbstractConstVarFunVisitor<ClosedDataInfo, Void> implements
- ILogicalExpressionReferenceTransform {
-
- private IOptimizationContext context;
- private IVariableTypeEnvironment env;
-
- public void setOptimizationContext(IOptimizationContext context, IVariableTypeEnvironment env) {
- this.context = context;
- this.env = env;
- }
-
- @Override
- public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
- AbstractLogicalExpression expr = (AbstractLogicalExpression) exprRef.getValue();
- ClosedDataInfo cdi = expr.accept(this, null);
- if (cdi.expressionChanged) {
- exprRef.setValue(cdi.expression);
- }
- return cdi.expressionChanged;
- }
-
- @Override
- public ClosedDataInfo visitConstantExpression(ConstantExpression expr, Void arg) throws AlgebricksException {
- return new ClosedDataInfo(false, hasClosedType(expr), expr);
- }
-
- @Override
- public ClosedDataInfo visitFunctionCallExpression(AbstractFunctionCallExpression expr, Void arg)
- throws AlgebricksException {
- boolean allClosed = true;
- boolean changed = false;
- if (expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)) {
- ARecordType reqType = (ARecordType) TypeComputerUtilities.getRequiredType(expr);
- if (reqType == null || !reqType.isOpen()) {
- int n = expr.getArguments().size();
- if (n % 2 > 0) {
- throw new AlgebricksException(
- "Record constructor expected to have an even number of arguments: " + expr);
- }
- for (int i = 0; i < n / 2; i++) {
- ILogicalExpression a0 = expr.getArguments().get(2 * i).getValue();
- if (a0.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- allClosed = false;
- }
- Mutable<ILogicalExpression> aRef1 = expr.getArguments().get(2 * i + 1);
- ILogicalExpression a1 = aRef1.getValue();
- ClosedDataInfo cdi = a1.accept(this, arg);
- if (!cdi.dataIsClosed) {
- allClosed = false;
- }
- if (cdi.expressionChanged) {
- aRef1.setValue(cdi.expression);
- changed = true;
- }
- }
- if (allClosed) {
- expr.setFunctionInfo(FunctionUtils
- .getFunctionInfo(AsterixBuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR));
- GlobalConfig.ASTERIX_LOGGER.finest("Switching to CLOSED record constructor in " + expr + ".\n");
- changed = true;
- }
- }
- } else {
- boolean rewrite = true;
- if (expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.ORDERED_LIST_CONSTRUCTOR)
- || (expr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.UNORDERED_LIST_CONSTRUCTOR))) {
- IAType reqType = TypeComputerUtilities.getRequiredType(expr);
- if (reqType == null) {
- rewrite = false;
- }
- }
- if (rewrite) {
- for (Mutable<ILogicalExpression> e : expr.getArguments()) {
- ILogicalExpression ale = e.getValue();
- ClosedDataInfo cdi = ale.accept(this, arg);
- if (!cdi.dataIsClosed) {
- allClosed = false;
- }
- if (cdi.expressionChanged) {
- e.setValue(cdi.expression);
- changed = true;
- }
- }
- }
- }
- return new ClosedDataInfo(changed, hasClosedType(expr), expr);
- }
-
- @Override
- public ClosedDataInfo visitVariableReferenceExpression(VariableReferenceExpression expr, Void arg)
- throws AlgebricksException {
- Object varType = env.getVarType(expr.getVariableReference());
- if (varType == null) {
- throw new AlgebricksException("Could not infer type for variable '" + expr.getVariableReference()
- + "'.");
- }
- boolean dataIsClosed = isClosedRec((IAType) varType);
- return new ClosedDataInfo(false, dataIsClosed, expr);
- }
-
- private boolean hasClosedType(ILogicalExpression expr) throws AlgebricksException {
- IAType t = (IAType) context.getExpressionTypeComputer().getType(expr, context.getMetadataProvider(), env);
- return isClosedRec(t);
- }
-
- private static boolean isClosedRec(IAType t) throws AlgebricksException {
- switch (t.getTypeTag()) {
- case ANY: {
- return false;
- }
- case CIRCLE:
- case INT8:
- case INT16:
- case INT32:
- case INT64:
- case BINARY:
- case BITARRAY:
- case FLOAT:
- case DOUBLE:
- case STRING:
- case LINE:
- case NULL:
- case BOOLEAN:
- case DATETIME:
- case DATE:
- case TIME:
- case UUID:
- case DURATION:
- case YEARMONTHDURATION:
- case DAYTIMEDURATION:
- case INTERVAL:
- case POINT:
- case POINT3D:
- case POLYGON:
- case RECTANGLE:
- case SHORTWITHOUTTYPEINFO: {
- return true;
- }
- case RECORD: {
- return !((ARecordType) t).isOpen();
- }
- case UNION: {
- AUnionType ut = (AUnionType) t;
- for (IAType t2 : ut.getUnionList()) {
- if (!isClosedRec(t2)) {
- return false;
- }
- }
- return true;
- }
- case ORDEREDLIST: {
- AOrderedListType ol = (AOrderedListType) t;
- return isClosedRec(ol.getItemType());
- }
- case UNORDEREDLIST: {
- AUnorderedListType ul = (AUnorderedListType) t;
- return isClosedRec(ul.getItemType());
- }
- default: {
- throw new IllegalStateException("Closed type analysis not implemented for type " + t);
- }
- }
- }
- }
-
- private static class ClosedDataInfo {
- boolean expressionChanged;
- boolean dataIsClosed;
- ILogicalExpression expression;
-
- public ClosedDataInfo(boolean expressionChanged, boolean dataIsClosed, ILogicalExpression expression) {
- this.expressionChanged = expressionChanged;
- this.dataIsClosed = dataIsClosed;
- this.expression = expression;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SimilarityCheckRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SimilarityCheckRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SimilarityCheckRule.java
deleted file mode 100644
index 2d03d69..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SimilarityCheckRule.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.om.base.ADouble;
-import edu.uci.ics.asterix.om.base.AFloat;
-import edu.uci.ics.asterix.om.base.AInt32;
-import edu.uci.ics.asterix.om.base.IAObject;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ATypeTag;
-import edu.uci.ics.asterix.om.types.hierachy.ATypeHierarchy;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Looks for a select operator, containing a condition:
- * similarity-function GE/GT/LE/LE constant/variable
- * Rewrites the select condition (and possibly the assign expr) with the equivalent similarity-check function.
- */
-public class SimilarityCheckRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- // Look for select.
- if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
- return false;
- }
- SelectOperator select = (SelectOperator) op;
- Mutable<ILogicalExpression> condExpr = select.getCondition();
-
- // Gather assigns below this select.
- List<AssignOperator> assigns = new ArrayList<AssignOperator>();
- AbstractLogicalOperator childOp = (AbstractLogicalOperator) select.getInputs().get(0).getValue();
- // Skip selects.
- while (childOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
- childOp = (AbstractLogicalOperator) childOp.getInputs().get(0).getValue();
- }
- while (childOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- assigns.add((AssignOperator) childOp);
- childOp = (AbstractLogicalOperator) childOp.getInputs().get(0).getValue();
- }
- return replaceSelectConditionExprs(condExpr, assigns, context);
- }
-
- private boolean replaceSelectConditionExprs(Mutable<ILogicalExpression> expRef, List<AssignOperator> assigns,
- IOptimizationContext context) throws AlgebricksException {
- ILogicalExpression expr = expRef.getValue();
- if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return false;
- }
- AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
- FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
- // Recursively traverse conjuncts.
- // TODO: Ignore disjuncts for now, because some replacements may be invalid.
- // For example, if the result of the similarity function is used somewhere upstream,
- // then we may still need the true similarity value even if the GE/GT/LE/LE comparison returns false.
- if (funcIdent == AlgebricksBuiltinFunctions.AND) {
- boolean found = true;
- for (int i = 0; i < funcExpr.getArguments().size(); ++i) {
- found = found && replaceSelectConditionExprs(funcExpr.getArguments().get(i), assigns, context);
- }
- return found;
- }
-
- // Look for GE/GT/LE/LT.
- if (funcIdent != AlgebricksBuiltinFunctions.GE && funcIdent != AlgebricksBuiltinFunctions.GT
- && funcIdent != AlgebricksBuiltinFunctions.LE && funcIdent != AlgebricksBuiltinFunctions.LT) {
- return false;
- }
-
- // One arg should be a function call or a variable, the other a constant.
- AsterixConstantValue constVal = null;
- ILogicalExpression nonConstExpr = null;
- ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
- ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
- // Normalized GE/GT/LE/LT as if constant was on the right hand side.
- FunctionIdentifier normFuncIdent = null;
- // One of the args must be a constant.
- if (arg1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
- ConstantExpression constExpr = (ConstantExpression) arg1;
- constVal = (AsterixConstantValue) constExpr.getValue();
- nonConstExpr = arg2;
- // Get func ident as if swapping lhs and rhs.
- normFuncIdent = getLhsAndRhsSwappedFuncIdent(funcIdent);
- } else if (arg2.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
- ConstantExpression constExpr = (ConstantExpression) arg2;
- constVal = (AsterixConstantValue) constExpr.getValue();
- nonConstExpr = arg1;
- // Constant is already on rhs, so nothing to be done for normalizedFuncIdent.
- normFuncIdent = funcIdent;
- } else {
- return false;
- }
-
- // The other arg is a function call. We can directly replace the select condition with an equivalent similarity check expression.
- if (nonConstExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- return replaceWithFunctionCallArg(expRef, normFuncIdent, constVal,
- (AbstractFunctionCallExpression) nonConstExpr);
- }
- // The other arg ist a variable. We may have to introduce an assign operator that assigns the result of a similarity-check function to a variable.
- if (nonConstExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- return replaceWithVariableArg(expRef, normFuncIdent, constVal, (VariableReferenceExpression) nonConstExpr,
- assigns, context);
- }
- return false;
- }
-
- private boolean replaceWithVariableArg(Mutable<ILogicalExpression> expRef, FunctionIdentifier normFuncIdent,
- AsterixConstantValue constVal, VariableReferenceExpression varRefExpr, List<AssignOperator> assigns,
- IOptimizationContext context) throws AlgebricksException {
-
- // Find variable in assigns to determine its originating function.
- LogicalVariable var = varRefExpr.getVariableReference();
- Mutable<ILogicalExpression> simFuncExprRef = null;
- ScalarFunctionCallExpression simCheckFuncExpr = null;
- AssignOperator matchingAssign = null;
- for (int i = 0; i < assigns.size(); i++) {
- AssignOperator assign = assigns.get(i);
- for (int j = 0; j < assign.getVariables().size(); j++) {
- // Check if variables match.
- if (var != assign.getVariables().get(j)) {
- continue;
- }
- // Check if corresponding expr is a function call.
- if (assign.getExpressions().get(j).getValue().getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- continue;
- }
- simFuncExprRef = assign.getExpressions().get(j);
- // Analyze function expression and get equivalent similarity check function.
- simCheckFuncExpr = getSimilarityCheckExpr(normFuncIdent, constVal,
- (AbstractFunctionCallExpression) simFuncExprRef.getValue());
- matchingAssign = assign;
- break;
- }
- if (simCheckFuncExpr != null) {
- break;
- }
- }
-
- // Only non-null if we found that varRefExpr refers to an optimizable similarity function call.
- if (simCheckFuncExpr != null) {
- // Create a new assign under matchingAssign which assigns the result of our similarity-check function to a variable.
- LogicalVariable newVar = context.newVar();
- AssignOperator newAssign = new AssignOperator(newVar, new MutableObject<ILogicalExpression>(
- simCheckFuncExpr));
- // Hook up inputs.
- newAssign.getInputs()
- .add(new MutableObject<ILogicalOperator>(matchingAssign.getInputs().get(0).getValue()));
- matchingAssign.getInputs().get(0).setValue(newAssign);
-
- // Replace select condition with a get-item on newVar.
- List<Mutable<ILogicalExpression>> selectGetItemArgs = new ArrayList<Mutable<ILogicalExpression>>();
- // First arg is a variable reference expr on newVar.
- selectGetItemArgs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(newVar)));
- // Second arg is the item index to be accessed, here 0.
- selectGetItemArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(
- new AsterixConstantValue(new AInt32(0)))));
- ILogicalExpression selectGetItemExpr = new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.GET_ITEM), selectGetItemArgs);
- // Replace the old similarity function call with the new getItemExpr.
- expRef.setValue(selectGetItemExpr);
-
- // Replace expr corresponding to original variable in the original assign with a get-item on newVar.
- List<Mutable<ILogicalExpression>> assignGetItemArgs = new ArrayList<Mutable<ILogicalExpression>>();
- // First arg is a variable reference expr on newVar.
- assignGetItemArgs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(newVar)));
- // Second arg is the item index to be accessed, here 1.
- assignGetItemArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(
- new AsterixConstantValue(new AInt32(1)))));
- ILogicalExpression assignGetItemExpr = new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.GET_ITEM), assignGetItemArgs);
- // Replace the original assign expr with the get-item expr.
- simFuncExprRef.setValue(assignGetItemExpr);
-
- context.computeAndSetTypeEnvironmentForOperator(newAssign);
- context.computeAndSetTypeEnvironmentForOperator(matchingAssign);
-
- return true;
- }
-
- return false;
- }
-
- private boolean replaceWithFunctionCallArg(Mutable<ILogicalExpression> expRef, FunctionIdentifier normFuncIdent,
- AsterixConstantValue constVal, AbstractFunctionCallExpression funcExpr) throws AlgebricksException {
- // Analyze func expr to see if it is an optimizable similarity function.
- ScalarFunctionCallExpression simCheckFuncExpr = getSimilarityCheckExpr(normFuncIdent, constVal, funcExpr);
-
- // Replace the expr in the select condition.
- if (simCheckFuncExpr != null) {
- // Get item 0 from var.
- List<Mutable<ILogicalExpression>> getItemArgs = new ArrayList<Mutable<ILogicalExpression>>();
- // First arg is the similarity-check function call.
- getItemArgs.add(new MutableObject<ILogicalExpression>(simCheckFuncExpr));
- // Second arg is the item index to be accessed.
- getItemArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
- new AInt32(0)))));
- ILogicalExpression getItemExpr = new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.GET_ITEM), getItemArgs);
- // Replace the old similarity function call with the new getItemExpr.
- expRef.setValue(getItemExpr);
- return true;
- }
-
- return false;
- }
-
- private ScalarFunctionCallExpression getSimilarityCheckExpr(FunctionIdentifier normFuncIdent,
- AsterixConstantValue constVal, AbstractFunctionCallExpression funcExpr) throws AlgebricksException {
- // Remember args from original similarity function to add them to the similarity-check function later.
- ArrayList<Mutable<ILogicalExpression>> similarityArgs = null;
- ScalarFunctionCallExpression simCheckFuncExpr = null;
- // Look for jaccard function call, and GE or GT.
- if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.SIMILARITY_JACCARD) {
- IAObject jaccThresh;
- if (normFuncIdent == AlgebricksBuiltinFunctions.GE) {
- if (constVal.getObject() instanceof AFloat) {
- jaccThresh = constVal.getObject();
- } else {
- jaccThresh = new AFloat((float) ((ADouble) constVal.getObject()).getDoubleValue());
- }
- } else if (normFuncIdent == AlgebricksBuiltinFunctions.GT) {
- float threshVal = 0.0f;
- if (constVal.getObject() instanceof AFloat) {
- threshVal = ((AFloat) constVal.getObject()).getFloatValue();
- } else {
- threshVal = (float) ((ADouble) constVal.getObject()).getDoubleValue();
- }
- float f = threshVal + Float.MIN_VALUE;
- if (f > 1.0f)
- f = 1.0f;
- jaccThresh = new AFloat(f);
- } else {
- return null;
- }
- similarityArgs = new ArrayList<Mutable<ILogicalExpression>>();
- similarityArgs.addAll(funcExpr.getArguments());
- similarityArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
- jaccThresh))));
- simCheckFuncExpr = new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.SIMILARITY_JACCARD_CHECK), similarityArgs);
- }
-
- // Look for edit-distance function call, and LE or LT.
- if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE) {
- AInt32 aInt = new AInt32(0);
- try {
- aInt = (AInt32) ATypeHierarchy.convertNumericTypeObject(constVal.getObject(), ATypeTag.INT32);
- } catch (AsterixException e) {
- throw new AlgebricksException(e);
- }
-
- AInt32 edThresh;
- if (normFuncIdent == AlgebricksBuiltinFunctions.LE) {
- edThresh = aInt;
- } else if (normFuncIdent == AlgebricksBuiltinFunctions.LT) {
- int ed = aInt.getIntegerValue() - 1;
- if (ed < 0)
- ed = 0;
- edThresh = new AInt32(ed);
- } else {
- return null;
- }
- similarityArgs = new ArrayList<Mutable<ILogicalExpression>>();
- similarityArgs.addAll(funcExpr.getArguments());
- similarityArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
- edThresh))));
- simCheckFuncExpr = new ScalarFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.EDIT_DISTANCE_CHECK), similarityArgs);
- }
- // Preserve all annotations.
- if (simCheckFuncExpr != null) {
- simCheckFuncExpr.getAnnotations().putAll(funcExpr.getAnnotations());
- }
- return simCheckFuncExpr;
- }
-
- private FunctionIdentifier getLhsAndRhsSwappedFuncIdent(FunctionIdentifier oldFuncIdent) {
- if (oldFuncIdent == AlgebricksBuiltinFunctions.GE) {
- return AlgebricksBuiltinFunctions.LE;
- }
- if (oldFuncIdent == AlgebricksBuiltinFunctions.GT) {
- return AlgebricksBuiltinFunctions.LT;
- }
- if (oldFuncIdent == AlgebricksBuiltinFunctions.LE) {
- return AlgebricksBuiltinFunctions.GE;
- }
- if (oldFuncIdent == AlgebricksBuiltinFunctions.LT) {
- return AlgebricksBuiltinFunctions.GT;
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
deleted file mode 100644
index e2f2f4f..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.AbstractExtractExprRule;
-
-public class SweepIllegalNonfunctionalFunctions extends AbstractExtractExprRule implements IAlgebraicRewriteRule {
-
- private final IllegalNonfunctionalFunctionSweeperOperatorVisitor visitor;
-
- public SweepIllegalNonfunctionalFunctions() {
- visitor = new IllegalNonfunctionalFunctionSweeperOperatorVisitor();
- }
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- ILogicalOperator op = opRef.getValue();
- if (context.checkIfInDontApplySet(this, op)) {
- return false;
- }
-
- op.accept(visitor, null);
- context.computeAndSetTypeEnvironmentForOperator(op);
- context.addToDontApplySet(this, op);
- return false;
- }
-
- private class IllegalNonfunctionalFunctionSweeperOperatorVisitor implements ILogicalOperatorVisitor<Void, Void> {
-
- private void sweepExpression(ILogicalExpression expr, ILogicalOperator op) throws AlgebricksException {
- if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- if (!expr.isFunctional()) {
- AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
- throw new AlgebricksException("Found non-functional function " + fce.getFunctionIdentifier()
- + " in op " + op);
- }
- }
- }
-
- @Override
- public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
- for (Mutable<ILogicalExpression> me : op.getExpressions()) {
- sweepExpression(me.getValue(), op);
- }
- List<Mutable<ILogicalExpression>> mergeExprs = op.getMergeExpressions();
- if (mergeExprs != null) {
- for (Mutable<ILogicalExpression> me : mergeExprs) {
- sweepExpression(me.getValue(), op);
- }
- }
- return null;
- }
-
- @Override
- public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
- for (Mutable<ILogicalExpression> me : op.getExpressions()) {
- sweepExpression(me.getValue(), op);
- }
- return null;
- }
-
- @Override
- public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
- for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
- sweepExpression(p.second.getValue(), op);
- }
- for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
- sweepExpression(p.second.getValue(), op);
- }
- return null;
- }
-
- @Override
- public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
- sweepExpression(op.getCondition().getValue(), op);
- return null;
- }
-
- @Override
- public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
- sweepExpression(op.getCondition().getValue(), op);
- return null;
- }
-
- @Override
- public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
- for (Pair<IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
- sweepExpression(p.second.getValue(), op);
- }
- return null;
- }
-
- @Override
- public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
- for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
- sweepExpression(expr.getValue(), op);
- }
- return null;
- }
-
- @Override
- public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
- for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
- sweepExpression(expr.getValue(), op);
- }
- return null;
- }
-
- @Override
- public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void tag) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void tag) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitTokenizeOperator(TokenizeOperator op, Void tag) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
- return null;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
deleted file mode 100644
index 6defc07..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
-import edu.uci.ics.asterix.common.feeds.FeedActivity.FeedActivityDetails;
-import edu.uci.ics.asterix.common.feeds.api.IFeedLifecycleListener.ConnectionLocation;
-import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
-import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
-import edu.uci.ics.asterix.metadata.declared.AqlSourceId;
-import edu.uci.ics.asterix.metadata.declared.FeedDataSource;
-import edu.uci.ics.asterix.metadata.entities.Dataset;
-import edu.uci.ics.asterix.metadata.entities.Dataverse;
-import edu.uci.ics.asterix.metadata.entities.Feed;
-import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
-import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
-import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
-import edu.uci.ics.asterix.om.base.AString;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.ATypeTag;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.optimizer.rules.util.EquivalenceClassUtils;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
- return false;
- }
- UnnestOperator unnest = (UnnestOperator) op;
- ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue();
-
- if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
- FunctionIdentifier fid = f.getFunctionIdentifier();
-
- if (fid.equals(AsterixBuiltinFunctions.DATASET)) {
- if (unnest.getPositionalVariable() != null) {
- // TODO remove this after enabling the support of positional variables in data scan
- throw new AlgebricksException("No positional variables are allowed over datasets.");
- }
- ILogicalExpression expr = f.getArguments().get(0).getValue();
- if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- return false;
- }
- ConstantExpression ce = (ConstantExpression) expr;
- IAlgebricksConstantValue acv = ce.getValue();
- if (!(acv instanceof AsterixConstantValue)) {
- return false;
- }
- AsterixConstantValue acv2 = (AsterixConstantValue) acv;
- if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
- return false;
- }
- String datasetArg = ((AString) acv2.getObject()).getStringValue();
-
- AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
- Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg);
- String dataverseName = datasetReference.first;
- String datasetName = datasetReference.second;
- Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
- if (dataset == null) {
- throw new AlgebricksException("Could not find dataset " + datasetName + " in dataverse "
- + dataverseName);
- }
-
- AqlSourceId asid = new AqlSourceId(dataverseName, datasetName);
-
- ArrayList<LogicalVariable> v = new ArrayList<LogicalVariable>();
-
- if (dataset.getDatasetType() == DatasetType.INTERNAL) {
- int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
- for (int i = 0; i < numPrimaryKeys; i++) {
- v.add(context.newVar());
- }
- }
- v.add(unnest.getVariable());
- AqlDataSource dataSource = metadataProvider.findDataSource(asid);
- DataSourceScanOperator scan = new DataSourceScanOperator(v, dataSource);
- List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
- scanInpList.addAll(unnest.getInputs());
- opRef.setValue(scan);
- addPrimaryKey(v, context);
- context.computeAndSetTypeEnvironmentForOperator(scan);
-
- // Adds equivalence classes --- one equivalent class between a primary key
- // variable and a record field-access expression.
- IAType[] schemaTypes = dataSource.getSchemaTypes();
- ARecordType recordType = (ARecordType) schemaTypes[schemaTypes.length - 1];
- EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, v, recordType, dataset, context);
- return true;
- }
-
- if (fid.equals(AsterixBuiltinFunctions.FEED_COLLECT)) {
- if (unnest.getPositionalVariable() != null) {
- throw new AlgebricksException("No positional variables are allowed over datasets.");
- }
-
- String dataverse = getStringArgument(f, 0);
- String sourceFeedName = getStringArgument(f, 1);
- String getTargetFeed = getStringArgument(f, 2);
- String subscriptionLocation = getStringArgument(f, 3);
- String targetDataset = getStringArgument(f, 4);
- String outputType = getStringArgument(f, 5);
-
- AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
-
- AqlSourceId asid = new AqlSourceId(dataverse, getTargetFeed);
- String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
- FeedPolicy policy = metadataProvider.findFeedPolicy(dataverse, policyName);
- if (policy == null) {
- policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
- if (policy == null) {
- throw new AlgebricksException("Unknown feed policy:" + policyName);
- }
- }
-
- ArrayList<LogicalVariable> v = new ArrayList<LogicalVariable>();
- v.add(unnest.getVariable());
-
- String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS);
- DataSourceScanOperator scan = new DataSourceScanOperator(v, createFeedDataSource(asid, targetDataset,
- sourceFeedName, subscriptionLocation, metadataProvider, policy, outputType, csLocations));
-
- List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
- scanInpList.addAll(unnest.getInputs());
- opRef.setValue(scan);
- addPrimaryKey(v, context);
- context.computeAndSetTypeEnvironmentForOperator(scan);
-
- return true;
- }
-
- }
-
- return false;
- }
-
- public void addPrimaryKey(List<LogicalVariable> scanVariables, IOptimizationContext context) {
- int n = scanVariables.size();
- List<LogicalVariable> head = new ArrayList<LogicalVariable>(scanVariables.subList(0, n - 1));
- List<LogicalVariable> tail = new ArrayList<LogicalVariable>(1);
- tail.add(scanVariables.get(n - 1));
- FunctionalDependency pk = new FunctionalDependency(head, tail);
- context.addPrimaryKey(pk);
- }
-
- private AqlDataSource createFeedDataSource(AqlSourceId aqlId, String targetDataset, String sourceFeedName,
- String subscriptionLocation, AqlMetadataProvider metadataProvider, FeedPolicy feedPolicy,
- String outputType, String locations) throws AlgebricksException {
- if (!aqlId.getDataverseName().equals(
- metadataProvider.getDefaultDataverse() == null ? null : metadataProvider.getDefaultDataverse()
- .getDataverseName())) {
- return null;
- }
- IAType feedOutputType = metadataProvider.findType(aqlId.getDataverseName(), outputType);
- Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName);
-
- FeedDataSource feedDataSource = new FeedDataSource(aqlId, targetDataset, feedOutputType,
- AqlDataSource.AqlDataSourceType.FEED, sourceFeed.getFeedId(), sourceFeed.getFeedType(),
- ConnectionLocation.valueOf(subscriptionLocation), locations.split(","));
- feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy);
- return feedDataSource;
- }
-
- private Pair<String, String> parseDatasetReference(AqlMetadataProvider metadataProvider, String datasetArg)
- throws AlgebricksException {
- String[] datasetNameComponents = datasetArg.split("\\.");
- String dataverseName;
- String datasetName;
- if (datasetNameComponents.length == 1) {
- Dataverse defaultDataverse = metadataProvider.getDefaultDataverse();
- if (defaultDataverse == null) {
- throw new AlgebricksException("Unresolved dataset " + datasetArg + " Dataverse not specified.");
- }
- dataverseName = defaultDataverse.getDataverseName();
- datasetName = datasetNameComponents[0];
- } else {
- dataverseName = datasetNameComponents[0];
- datasetName = datasetNameComponents[1];
- }
- return new Pair<String, String>(dataverseName, datasetName);
- }
-
- private String getStringArgument(AbstractFunctionCallExpression f, int index) {
-
- ILogicalExpression expr = f.getArguments().get(index).getValue();
- if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- return null;
- }
- ConstantExpression ce = (ConstantExpression) expr;
- IAlgebricksConstantValue acv = ce.getValue();
- if (!(acv instanceof AsterixConstantValue)) {
- return null;
- }
- AsterixConstantValue acv2 = (AsterixConstantValue) acv;
- if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
- return null;
- }
- String argument = ((AString) acv2.getObject()).getStringValue();
- return argument;
- }
-
-}