You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by bu...@apache.org on 2015/11/17 01:57:00 UTC
[28/34] incubator-asterixdb git commit: SQL++ support in AsterixDB:
1. implemented SQL++ expression to logical plan translator;
2. refactored REST API to be agnostic of query languages;
3. disabled fuzzy join queries for SQL++ runtime tests; 4. fixed sev
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
new file mode 100644
index 0000000..a708cf6
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -0,0 +1,492 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.expression.FieldBinding;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.clause.HavingClause;
+import org.apache.asterix.lang.sqlpp.clause.JoinClause;
+import org.apache.asterix.lang.sqlpp.clause.NestClause;
+import org.apache.asterix.lang.sqlpp.clause.Projection;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.clause.SelectElement;
+import org.apache.asterix.lang.sqlpp.clause.SelectRegular;
+import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
+import org.apache.asterix.lang.sqlpp.clause.UnnestClause;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.optype.JoinType;
+import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OuterUnnestOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+
+/**
+ * Each visit returns a pair of an operator and a variable. The variable
+ * corresponds to the new column, if any, added to the tuple flow. E.g., for
+ * Unnest, the column is the variable bound to the elements in the list, for
+ * Subplan it is null. The first argument of a visit method is the expression
+ * which is translated. The second argument of a visit method is the tuple
+ * source for the current subtree.
+ */
+class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTranslator implements ILangExpressionToPlanTranslator,
+ ISqlppVisitor<Pair<ILogicalOperator, LogicalVariable>, Mutable<ILogicalOperator>> {
+
+ public SqlppExpressionToPlanTranslator(AqlMetadataProvider metadataProvider, int currentVarCounter)
+ throws AlgebricksException {
+ super(metadataProvider, currentVarCounter);
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(Query q, Mutable<ILogicalOperator> tupSource)
+ throws AsterixException {
+ Expression queryBody = q.getBody();
+ if (queryBody.getKind() == Kind.SELECT_EXPRESSION) {
+ SelectExpression selectExpr = (SelectExpression) queryBody;
+ if (q.isTopLevel()) {
+ selectExpr.setSubquery(false);
+ }
+ return queryBody.accept(this, tupSource);
+ } else {
+ LogicalVariable var = context.newVar();
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = aqlExprToAlgExpression(queryBody, tupSource);
+ AssignOperator assignOp = new AssignOperator(var, new MutableObject<ILogicalExpression>(eo.first));
+ assignOp.getInputs().add(eo.second);
+ ProjectOperator projectOp = new ProjectOperator(var);
+ projectOp.getInputs().add(new MutableObject<ILogicalOperator>(assignOp));
+ return new Pair<ILogicalOperator, LogicalVariable>(projectOp, var);
+ }
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(SelectExpression selectExpression,
+ Mutable<ILogicalOperator> tupSource) throws AsterixException {
+ if (selectExpression.isSubquery()) {
+ context.enterSubplan();
+ }
+ Mutable<ILogicalOperator> currentOpRef = tupSource;
+ if (selectExpression.hasLetClauses()) {
+ for (LetClause letClause : selectExpression.getLetList()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(letClause.accept(this, currentOpRef).first);
+ }
+ }
+ Pair<ILogicalOperator, LogicalVariable> select = selectExpression.getSelectSetOperation().accept(this,
+ currentOpRef);
+ currentOpRef = new MutableObject<ILogicalOperator>(select.first);
+ if (selectExpression.hasOrderby()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(
+ selectExpression.getOrderbyClause().accept(this, currentOpRef).first);
+ }
+ if (selectExpression.hasLimit()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(
+ selectExpression.getLimitClause().accept(this, currentOpRef).first);
+ }
+ Pair<ILogicalOperator, LogicalVariable> result = produceSelectPlan(selectExpression.isSubquery(), currentOpRef,
+ select.second);
+ if (selectExpression.isSubquery()) {
+ context.existSubplan();
+ }
+ return result;
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(SelectSetOperation selectSetOperation,
+ Mutable<ILogicalOperator> tupSource) throws AsterixException {
+ Mutable<ILogicalOperator> currentOpRef = tupSource;
+ Pair<ILogicalOperator, LogicalVariable> currentResult = selectSetOperation.getLeftInput().accept(this,
+ currentOpRef);
+ if (selectSetOperation.hasRightInputs()) {
+ throw new NotImplementedException();
+ }
+ return currentResult;
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(SelectBlock selectBlock, Mutable<ILogicalOperator> tupSource)
+ throws AsterixException {
+ Mutable<ILogicalOperator> currentOpRef = tupSource;
+ if (selectBlock.hasFromClause()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(
+ selectBlock.getFromClause().accept(this, currentOpRef).first);
+ }
+ if (selectBlock.hasLetClauses()) {
+ for (LetClause letClause : selectBlock.getLetList()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(letClause.accept(this, currentOpRef).first);
+ }
+ }
+ if (selectBlock.hasWhereClause()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(
+ selectBlock.getWhereClause().accept(this, currentOpRef).first);
+ }
+ if (selectBlock.hasGroupbyClause()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(
+ selectBlock.getGroupbyClause().accept(this, currentOpRef).first);
+ }
+ if (selectBlock.hasLetClausesAfterGroupby()) {
+ for (LetClause letClause : selectBlock.getLetListAfterGroupby()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(letClause.accept(this, currentOpRef).first);
+ }
+ }
+ if (selectBlock.hasHavingClause()) {
+ currentOpRef = new MutableObject<ILogicalOperator>(
+ selectBlock.getHavingClause().accept(this, currentOpRef).first);
+ }
+ return selectBlock.getSelectClause().accept(this, currentOpRef);
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(FromClause fromClause, Mutable<ILogicalOperator> arg)
+ throws AsterixException {
+ Mutable<ILogicalOperator> inputSrc = arg;
+ Pair<ILogicalOperator, LogicalVariable> topUnnest = null;
+ for (FromTerm fromTerm : fromClause.getFromTerms()) {
+ topUnnest = fromTerm.accept(this, inputSrc);
+ inputSrc = new MutableObject<ILogicalOperator>(topUnnest.first);
+ }
+ return topUnnest;
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(FromTerm fromTerm, Mutable<ILogicalOperator> tupSource)
+ throws AsterixException {
+ LogicalVariable fromVar = context.newVar(fromTerm.getLeftVariable());
+ Expression fromExpr = fromTerm.getLeftExpression();
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = aqlExprToAlgExpression(fromExpr, tupSource);
+ ILogicalOperator unnestOp;
+ if (fromTerm.hasPositionalVariable()) {
+ LogicalVariable pVar = context.newVar(fromTerm.getPositionalVariable());
+ // We set the positional variable type as INT64 type.
+ unnestOp = new UnnestOperator(fromVar,
+ new MutableObject<ILogicalExpression>(makeUnnestExpression(eo.first)), pVar, BuiltinType.AINT64,
+ new AqlPositionWriter());
+ } else {
+ unnestOp = new UnnestOperator(fromVar,
+ new MutableObject<ILogicalExpression>(makeUnnestExpression(eo.first)));
+ }
+ unnestOp.getInputs().add(eo.second);
+
+ // Processes joins, unnests, and nests.
+ Mutable<ILogicalOperator> topOpRef = new MutableObject<ILogicalOperator>(unnestOp);
+ if (fromTerm.hasCorrelateClauses()) {
+ for (AbstractBinaryCorrelateClause correlateClause : fromTerm.getCorrelateClauses()) {
+ topOpRef = new MutableObject<ILogicalOperator>(correlateClause.accept(this, topOpRef).first);
+ }
+ }
+ return new Pair<ILogicalOperator, LogicalVariable>(topOpRef.getValue(), fromVar);
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(JoinClause joinClause, Mutable<ILogicalOperator> inputRef)
+ throws AsterixException {
+ if (joinClause.getJoinType() == JoinType.INNER) {
+ Pair<ILogicalOperator, LogicalVariable> rightBranch = generateUnnestForBinaryCorrelateRightBranch(
+ joinClause, new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
+ // A join operator with condition TRUE.
+ AbstractBinaryJoinOperator joinOperator = new InnerJoinOperator(
+ new MutableObject<ILogicalExpression>(ConstantExpression.TRUE), inputRef,
+ new MutableObject<ILogicalOperator>(rightBranch.first));
+ Mutable<ILogicalOperator> joinOpRef = new MutableObject<ILogicalOperator>(joinOperator);
+
+ // Add an additional filter operator.
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> conditionExprOpPair = aqlExprToAlgExpression(
+ joinClause.getConditionExpression(), joinOpRef);
+ SelectOperator filter = new SelectOperator(new MutableObject<ILogicalExpression>(conditionExprOpPair.first),
+ false, null);
+ filter.getInputs().add(conditionExprOpPair.second);
+ return new Pair<ILogicalOperator, LogicalVariable>(filter, rightBranch.second);
+ } else {
+ // Creates a subplan operator.
+ SubplanOperator subplanOp = new SubplanOperator();
+ Mutable<ILogicalOperator> ntsRef = new MutableObject<ILogicalOperator>(
+ new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(subplanOp)));
+ subplanOp.getInputs().add(inputRef);
+
+ // Enters the translation for a subplan.
+ context.enterSubplan();
+
+ // Adds an unnest operator to unnest to right expression.
+ Pair<ILogicalOperator, LogicalVariable> rightBranch = generateUnnestForBinaryCorrelateRightBranch(
+ joinClause, ntsRef);
+ UnnestOperator rightUnnestOp = (UnnestOperator) rightBranch.first;
+
+ // Adds an additional filter operator for the join condition.
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> conditionExprOpPair = aqlExprToAlgExpression(
+ joinClause.getConditionExpression(), new MutableObject<ILogicalOperator>(rightUnnestOp));
+ SelectOperator filter = new SelectOperator(new MutableObject<ILogicalExpression>(conditionExprOpPair.first),
+ false, null);
+ filter.getInputs().add(conditionExprOpPair.second);
+
+ ILogicalOperator currentTopOp = filter;
+ LogicalVariable varToListify;
+ boolean hasRightPosVar = rightUnnestOp.getPositionalVariable() != null;
+ if (hasRightPosVar) {
+ // Creates record to get correlation between the two aggregate variables.
+ @SuppressWarnings("unchecked")
+ ScalarFunctionCallExpression recordCreationFunc = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR),
+ // Field name for the listified right unnest var.
+ new MutableObject<ILogicalExpression>(
+ new ConstantExpression(new AsterixConstantValue(new AString("unnestvar")))),
+ // The listified right unnest var
+ new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(rightUnnestOp.getVariable())),
+ // Field name for the listified right unnest positional var.
+ new MutableObject<ILogicalExpression>(
+ new ConstantExpression(new AsterixConstantValue(new AString("posvar")))),
+ // The listified right unnest positional var.
+ new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(rightUnnestOp.getPositionalVariable())));
+
+ // Assigns the record constructor function to a record variable.
+ LogicalVariable recordVar = context.newVar();
+ AssignOperator assignOp = new AssignOperator(recordVar,
+ new MutableObject<ILogicalExpression>(recordCreationFunc));
+ assignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
+
+ // Sets currentTopOp and varToListify for later usages.
+ currentTopOp = assignOp;
+ varToListify = recordVar;
+ } else {
+ varToListify = rightUnnestOp.getVariable();
+ }
+
+ // Adds an aggregate operator to listfy unnest variables.
+ AggregateFunctionCallExpression fListify = AsterixBuiltinFunctions
+ .makeAggregateFunctionExpression(AsterixBuiltinFunctions.LISTIFY, mkSingletonArrayList(
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(varToListify))));
+
+ LogicalVariable aggVar = context.newSubplanOutputVar();
+ AggregateOperator aggOp = new AggregateOperator(mkSingletonArrayList(aggVar),
+ mkSingletonArrayList(new MutableObject<ILogicalExpression>(fListify)));
+ aggOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
+
+ // Exits the translation of a subplan.
+ context.existSubplan();
+
+ // Sets the nested subplan of the subplan operator.
+ ILogicalPlan subplan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(aggOp));
+ subplanOp.getNestedPlans().add(subplan);
+
+ // Outer unnest the aggregated var from the subplan.
+ LogicalVariable outerUnnestVar = context.newVar();
+ OuterUnnestOperator outerUnnestOp = new OuterUnnestOperator(outerUnnestVar,
+ new MutableObject<ILogicalExpression>(
+ makeUnnestExpression(new VariableReferenceExpression(aggVar))));
+ outerUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(subplanOp));
+ currentTopOp = outerUnnestOp;
+
+ if (hasRightPosVar) {
+ @SuppressWarnings("unchecked")
+ ScalarFunctionCallExpression fieldAccessForRightUnnestVar = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX),
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(outerUnnestVar)),
+ new MutableObject<ILogicalExpression>(
+ new ConstantExpression(new AsterixConstantValue(new AInt32(0)))));
+
+ @SuppressWarnings("unchecked")
+ ScalarFunctionCallExpression fieldAccessForRightPosVar = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX),
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(outerUnnestVar)),
+ new MutableObject<ILogicalExpression>(
+ new ConstantExpression(new AsterixConstantValue(new AInt32(1)))));
+
+ // Creates variables for assign.
+ LogicalVariable rightUnnestVar = context.newVar();
+ LogicalVariable rightPosVar = context.newVar();
+
+ // Relate the assigned variables to the variable expression in AST.
+ context.setVar(joinClause.getRightVariable(), rightUnnestVar);
+ context.setVar(joinClause.getPositionalVariable(), rightPosVar);
+
+ // Varaibles to assign.
+ List<LogicalVariable> assignVars = new ArrayList<LogicalVariable>();
+ assignVars.add(rightUnnestVar);
+ assignVars.add(rightPosVar);
+
+ // Expressions for assign.
+ List<Mutable<ILogicalExpression>> assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
+ assignExprs.add(new MutableObject<ILogicalExpression>(fieldAccessForRightUnnestVar));
+ assignExprs.add(new MutableObject<ILogicalExpression>(fieldAccessForRightPosVar));
+
+ // Creates the assign operator.
+ AssignOperator assignOp = new AssignOperator(assignVars, assignExprs);
+ assignOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTopOp));
+ currentTopOp = assignOp;
+ } else {
+ context.setVar(joinClause.getRightVariable(), outerUnnestVar);
+ }
+ return new Pair<ILogicalOperator, LogicalVariable>(currentTopOp, null);
+ }
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(NestClause nestClause, Mutable<ILogicalOperator> arg)
+ throws AsterixException {
+ throw new NotImplementedException("Nest clause has not been implemented");
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(UnnestClause unnestClause,
+ Mutable<ILogicalOperator> inputOpRef) throws AsterixException {
+ return generateUnnestForBinaryCorrelateRightBranch(unnestClause, inputOpRef);
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(HavingClause havingClause, Mutable<ILogicalOperator> tupSource)
+ throws AsterixException {
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> p = aqlExprToAlgExpression(
+ havingClause.getFilterExpression(), tupSource);
+ SelectOperator s = new SelectOperator(new MutableObject<ILogicalExpression>(p.first), false, null);
+ s.getInputs().add(p.second);
+ return new Pair<ILogicalOperator, LogicalVariable>(s, null);
+ }
+
+ private Pair<ILogicalOperator, LogicalVariable> generateUnnestForBinaryCorrelateRightBranch(
+ AbstractBinaryCorrelateClause binaryCorrelate, Mutable<ILogicalOperator> inputOpRef)
+ throws AsterixException {
+ LogicalVariable rightVar = context.newVar(binaryCorrelate.getRightVariable());
+ Expression rightExpr = binaryCorrelate.getRightExpression();
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = aqlExprToAlgExpression(rightExpr, inputOpRef);
+ ILogicalOperator unnestOp;
+ if (binaryCorrelate.hasPositionalVariable()) {
+ LogicalVariable pVar = context.newVar(binaryCorrelate.getPositionalVariable());
+ // We set the positional variable type as INT64 type.
+ unnestOp = new UnnestOperator(rightVar,
+ new MutableObject<ILogicalExpression>(makeUnnestExpression(eo.first)), pVar, BuiltinType.AINT64,
+ new AqlPositionWriter());
+ } else {
+ unnestOp = new UnnestOperator(rightVar,
+ new MutableObject<ILogicalExpression>(makeUnnestExpression(eo.first)));
+ }
+ unnestOp.getInputs().add(eo.second);
+ return new Pair<ILogicalOperator, LogicalVariable>(unnestOp, rightVar);
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(SelectClause selectClause, Mutable<ILogicalOperator> tupSrc)
+ throws AsterixException {
+ Expression returnExpr;
+ if (selectClause.selectElement()) {
+ returnExpr = selectClause.getSelectElement().getExpression();
+ } else {
+ List<Projection> projections = selectClause.getSelectRegular().getProjections();
+ List<FieldBinding> fieldBindings = new ArrayList<FieldBinding>();
+ for (Projection projection : projections) {
+ fieldBindings.add(new FieldBinding(new LiteralExpr(new StringLiteral(projection.getName())),
+ projection.getExpression()));
+ }
+ returnExpr = new RecordConstructor(fieldBindings);
+ }
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = aqlExprToAlgExpression(returnExpr, tupSrc);
+ LogicalVariable returnVar;
+ ILogicalOperator returnOperator;
+ if (returnExpr.getKind() == Kind.VARIABLE_EXPRESSION) {
+ VariableExpr varExpr = (VariableExpr) returnExpr;
+ returnOperator = eo.second.getValue();
+ returnVar = context.getVar(varExpr.getVar().getId());
+ } else {
+ returnVar = context.newVar();
+ returnOperator = new AssignOperator(returnVar, new MutableObject<ILogicalExpression>(eo.first));
+ returnOperator.getInputs().add(eo.second);
+ }
+ if (selectClause.distinct()) {
+ DistinctOperator distinctOperator = new DistinctOperator(mkSingletonArrayList(
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(returnVar))));
+ distinctOperator.getInputs().add(new MutableObject<ILogicalOperator>(returnOperator));
+ return new Pair<ILogicalOperator, LogicalVariable>(distinctOperator, returnVar);
+ } else {
+ return new Pair<ILogicalOperator, LogicalVariable>(returnOperator, returnVar);
+ }
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(SelectElement selectElement, Mutable<ILogicalOperator> arg)
+ throws AsterixException {
+ throw new IllegalStateException("Translator should never enter this method!");
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(SelectRegular selectRegular, Mutable<ILogicalOperator> arg)
+ throws AsterixException {
+ throw new IllegalStateException("Translator should never enter this method!");
+ }
+
+ @Override
+ public Pair<ILogicalOperator, LogicalVariable> visit(Projection projection, Mutable<ILogicalOperator> arg)
+ throws AsterixException {
+ throw new IllegalStateException("Translator should never enter this method!");
+ }
+
+ private Pair<ILogicalOperator, LogicalVariable> produceSelectPlan(boolean isSubquery,
+ Mutable<ILogicalOperator> returnOpRef, LogicalVariable resVar) {
+ if (isSubquery) {
+ return aggListifyForSubquery(resVar, returnOpRef, false);
+ } else {
+ ProjectOperator pr = new ProjectOperator(resVar);
+ pr.getInputs().add(returnOpRef);
+ return new Pair<ILogicalOperator, LogicalVariable>(pr, resVar);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslatorFactory.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslatorFactory.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslatorFactory.java
new file mode 100644
index 0000000..611cc8a
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslatorFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
+import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public class SqlppExpressionToPlanTranslatorFactory implements ILangExpressionToPlanTranslatorFactory {
+
+ @Override
+ public ILangExpressionToPlanTranslator createExpressionToPlanTranslator(AqlMetadataProvider metadataProvider,
+ int currentVarCounter) throws AlgebricksException {
+ return new SqlppExpressionToPlanTranslator(metadataProvider, currentVarCounter);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java
index 70236b1..84a01b5 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java
@@ -19,6 +19,8 @@
package org.apache.asterix.translator;
import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.hyracks.algebricks.core.algebra.base.Counter;
@@ -27,7 +29,10 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
public final class TranslationContext {
private Counter varCounter;
- private HashMap<Integer, LogicalVariable> varEnv = new HashMap<Integer, LogicalVariable>();
+
+ /** The stack is the used to manage the scope of variables for group-by rebindings. */
+ private Stack<Map<Integer, LogicalVariable>> stack = new Stack<Map<Integer, LogicalVariable>>();
+ private Map<Integer, LogicalVariable> currentVarMap = new HashMap<Integer, LogicalVariable>();
private boolean topFlwor = true;
public TranslationContext(Counter varCounter) {
@@ -47,11 +52,11 @@ public final class TranslationContext {
}
public LogicalVariable getVar(Integer varId) {
- return varEnv.get(varId);
+ return currentVarMap.get(varId);
}
public LogicalVariable getVar(VariableExpr v) {
- return varEnv.get(v.getVar().getId());
+ return currentVarMap.get(v.getVar().getId());
}
public LogicalVariable newVar(VariableExpr v) {
@@ -60,18 +65,52 @@ public final class TranslationContext {
varCounter.set(i);
}
LogicalVariable var = new LogicalVariable(i);
- varEnv.put(i, var);
+ currentVarMap.put(i, var);
return var;
}
public void setVar(VariableExpr v, LogicalVariable var) {
- varEnv.put(v.getVar().getId(), var);
+ currentVarMap.put(v.getVar().getId(), var);
}
public LogicalVariable newVar() {
varCounter.inc();
LogicalVariable var = new LogicalVariable(varCounter.get());
- varEnv.put(varCounter.get(), var);
+ currentVarMap.put(varCounter.get(), var);
return var;
}
+
+ /**
+ * Within a subplan, an unbounded variable can be rebound in
+ * the group-by operator. But the rebinding only exists
+ * in the subplan.
+ * This method marks that the translation enters a subplan.
+ */
+ public void enterSubplan() {
+ Map<Integer, LogicalVariable> varMap = new HashMap<Integer, LogicalVariable>();
+ varMap.putAll(currentVarMap);
+ stack.push(currentVarMap);
+ currentVarMap = varMap;
+ }
+
+ /***
+ * This method marks that the translation exits a subplan.
+ */
+ public void existSubplan() {
+ if (!stack.isEmpty()) {
+ currentVarMap = stack.pop();
+ }
+ }
+
+ /**
+ * @return the variables produced by the top operator in a subplan.
+ */
+ public LogicalVariable newSubplanOutputVar() {
+ LogicalVariable newVar = newVar();
+ if (!stack.isEmpty()) {
+ Map<Integer, LogicalVariable> varMap = stack.peek();
+ varMap.put(varCounter.get(), newVar);
+ }
+ return newVar;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 9cdaf31..c0613ed 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -23,32 +23,33 @@ import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
+import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
import org.apache.asterix.api.common.Job.SubmissionMode;
import org.apache.asterix.common.config.AsterixCompilerProperties;
import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.common.config.OptimizationConfUtil;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.dataflow.data.common.AqlExpressionTypeComputer;
import org.apache.asterix.dataflow.data.common.AqlMergeAggregationExpressionFactory;
import org.apache.asterix.dataflow.data.common.AqlNullableTypeComputer;
import org.apache.asterix.dataflow.data.common.AqlPartialAggregationTypeComputer;
import org.apache.asterix.formats.base.IDataFormat;
-import org.apache.asterix.jobgen.AqlLogicalExpressionJobGen;
-import org.apache.asterix.lang.aql.rewrites.AqlRewriter;
-import org.apache.asterix.lang.aql.visitor.AQLPrintVisitor;
+import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen;
+import org.apache.asterix.lang.common.base.IAstPrintVisitorFactory;
+import org.apache.asterix.lang.common.base.IQueryRewriter;
+import org.apache.asterix.lang.common.base.IRewriterFactory;
import org.apache.asterix.lang.common.base.Statement.Kind;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.optimizer.base.RuleCollections;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.transaction.management.service.transaction.JobIdFactory;
-import org.apache.asterix.translator.AqlExpressionToPlanTranslator;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -85,6 +86,16 @@ import org.json.JSONException;
public class APIFramework {
public static final String HTML_STATEMENT_SEPARATOR = "<!-- BEGIN -->";
+ private final IRewriterFactory rewriterFactory;
+ private final IAstPrintVisitorFactory astPrintVisitorFactory;
+ private final ILangExpressionToPlanTranslatorFactory translatorFactory;
+
+ public APIFramework(ILangCompilationProvider compilationProvider) {
+ this.rewriterFactory = compilationProvider.getRewriterFactory();
+ this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory();
+ this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory();
+ }
+
private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultLogicalRewrites() {
List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>();
SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
@@ -158,8 +169,8 @@ public class APIFramework {
}
- public static Pair<Query, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions,
- AqlMetadataProvider metadataProvider, Query q, SessionConfig conf) throws AsterixException {
+ public Pair<Query, Integer> reWriteQuery(List<FunctionDecl> declaredFunctions, AqlMetadataProvider metadataProvider,
+ Query q, SessionConfig conf) throws AsterixException {
if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_EXPR_TREE)) {
conf.out().println();
@@ -172,23 +183,22 @@ public class APIFramework {
}
if (q != null) {
- q.accept(new AQLPrintVisitor(conf.out()), 0);
+ q.accept(astPrintVisitorFactory.createLangVisitor(conf.out()), 0);
}
if (conf.is(SessionConfig.FORMAT_HTML)) {
conf.out().println("</pre>");
}
}
- AqlRewriter rw = new AqlRewriter(declaredFunctions, q, metadataProvider);
- rw.rewrite();
- Query rwQ = rw.getExpr();
- return new Pair(rwQ, rw.getVarCounter());
+ IQueryRewriter rw = rewriterFactory.createQueryRewriter();
+ rw.rewrite(declaredFunctions, q, metadataProvider, new LangRewritingContext(q.getVarCounter()));
+ return new Pair<Query, Integer>(q, q.getVarCounter());
}
- public static JobSpecification compileQuery(List<FunctionDecl> declaredFunctions,
+ public JobSpecification compileQuery(List<FunctionDecl> declaredFunctions,
AqlMetadataProvider queryMetadataProvider, Query rwQ, int varCounter, String outputDatasetName,
- SessionConfig conf, ICompiledDmlStatement statement) throws AsterixException, AlgebricksException,
- JSONException, RemoteException, ACIDException {
+ SessionConfig conf, ICompiledDmlStatement statement)
+ throws AsterixException, AlgebricksException, JSONException, RemoteException, ACIDException {
if (!conf.is(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS) && conf.is(SessionConfig.OOB_REWRITTEN_EXPR_TREE)) {
conf.out().println();
@@ -201,7 +211,7 @@ public class APIFramework {
}
if (rwQ != null) {
- rwQ.accept(new AQLPrintVisitor(conf.out()), 0);
+ rwQ.accept(astPrintVisitorFactory.createLangVisitor(conf.out()), 0);
}
if (conf.is(SessionConfig.FORMAT_HTML)) {
@@ -211,15 +221,15 @@ public class APIFramework {
org.apache.asterix.common.transactions.JobId asterixJobId = JobIdFactory.generateJobId();
queryMetadataProvider.setJobId(asterixJobId);
- AqlExpressionToPlanTranslator t = new AqlExpressionToPlanTranslator(queryMetadataProvider, varCounter,
- outputDatasetName, statement);
+ ILangExpressionToPlanTranslator t = translatorFactory.createExpressionToPlanTranslator(queryMetadataProvider,
+ varCounter);
ILogicalPlan plan;
// statement = null when it's a query
if (statement == null || statement.getKind() != Kind.LOAD) {
- plan = t.translate(rwQ);
+ plan = t.translate(rwQ, outputDatasetName, statement);
} else {
- plan = t.translateLoad();
+ plan = t.translateLoad(statement);
}
LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
@@ -316,8 +326,8 @@ public class APIFramework {
builder.setBinaryIntegerInspectorFactory(format.getBinaryIntegerInspectorFactory());
builder.setClusterLocations(clusterLocs);
builder.setComparatorFactoryProvider(format.getBinaryComparatorFactoryProvider());
- builder.setExpressionRuntimeProvider(new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(
- AqlLogicalExpressionJobGen.INSTANCE));
+ builder.setExpressionRuntimeProvider(
+ new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(QueryLogicalExpressionJobGen.INSTANCE));
builder.setHashFunctionFactoryProvider(format.getBinaryHashFunctionFactoryProvider());
builder.setHashFunctionFamilyProvider(format.getBinaryHashFunctionFamilyProvider());
builder.setNullWriterFactory(format.getNullWriterFactory());
@@ -368,7 +378,7 @@ public class APIFramework {
return spec;
}
- public static void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out)
+ public void executeJobArray(IHyracksClientConnection hcc, JobSpecification[] specs, PrintWriter out)
throws Exception {
for (int i = 0; i < specs.length; i++) {
specs[i].setMaxReattempts(0);
@@ -382,7 +392,7 @@ public class APIFramework {
}
- public static void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out) throws Exception {
+ public void executeJobArray(IHyracksClientConnection hcc, Job[] jobs, PrintWriter out) throws Exception {
for (int i = 0; i < jobs.length; i++) {
jobs[i].getJobSpec().setMaxReattempts(0);
long startTime = System.currentTimeMillis();
@@ -400,19 +410,6 @@ public class APIFramework {
double duration = (endTime - startTime) / 1000.00;
out.println("<pre>Duration: " + duration + " sec</pre>");
}
-
- }
-
- private static IDataFormat getDataFormat(MetadataTransactionContext mdTxnCtx, String dataverseName)
- throws AsterixException {
- Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
- IDataFormat format;
- try {
- format = (IDataFormat) Class.forName(dataverse.getDataFormat()).newInstance();
- } catch (Exception e) {
- throw new AsterixException(e);
- }
- return format;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java b/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
index f25c5dd..6aad64b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/common/FeedWorkCollection.java
@@ -25,11 +25,13 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.AqlTranslator;
+import org.apache.asterix.aql.translator.QueryTranslator;
import org.apache.asterix.common.feeds.FeedConnectionRequest;
import org.apache.asterix.common.feeds.FeedConnectionRequest.ConnectionStatus;
import org.apache.asterix.common.feeds.api.IFeedWork;
import org.apache.asterix.common.feeds.api.IFeedWorkEventListener;
+import org.apache.asterix.compiler.provider.AqlCompilationProvider;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.feeds.FeedCollectInfo;
import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
import org.apache.asterix.lang.common.base.Statement;
@@ -44,6 +46,7 @@ import org.apache.hyracks.api.job.JobId;
public class FeedWorkCollection {
private static Logger LOGGER = Logger.getLogger(FeedWorkCollection.class.getName());
+ private static final ILangCompilationProvider compilationProvider = new AqlCompilationProvider();
/**
* The task of subscribing to a feed to obtain data.
@@ -79,14 +82,15 @@ public class FeedWorkCollection {
try {
PrintWriter writer = new PrintWriter(System.out, true);
SessionConfig pc = new SessionConfig(writer, OutputFormat.ADM);
- DataverseDecl dataverseDecl = new DataverseDecl(new Identifier(request.getReceivingFeedId()
- .getDataverse()));
+ DataverseDecl dataverseDecl = new DataverseDecl(
+ new Identifier(request.getReceivingFeedId().getDataverse()));
SubscribeFeedStatement subscribeStmt = new SubscribeFeedStatement(locations, request);
List<Statement> statements = new ArrayList<Statement>();
statements.add(dataverseDecl);
statements.add(subscribeStmt);
- AqlTranslator translator = new AqlTranslator(statements, pc);
- translator.compileAndExecute(AsterixAppContextInfo.getInstance().getHcc(), null, AqlTranslator.ResultDelivery.SYNC);
+ QueryTranslator translator = new QueryTranslator(statements, pc, compilationProvider);
+ translator.compileAndExecute(AsterixAppContextInfo.getInstance().getHcc(), null,
+ QueryTranslator.ResultDelivery.SYNC);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Submitted connection requests for execution: " + request);
}
@@ -187,8 +191,8 @@ public class FeedWorkCollection {
}
} catch (Exception e) {
if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Unable to resume feed " + finfo.feedConnectionId + " "
- + e.getMessage());
+ LOGGER.warning(
+ "Unable to resume feed " + finfo.feedConnectionId + " " + e.getMessage());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
index 2c0f897..2f998bb 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
@@ -37,11 +37,15 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.asterix.api.common.APIFramework;
import org.apache.asterix.api.common.SessionConfig;
import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.AqlTranslator;
+import org.apache.asterix.aql.translator.QueryTranslator;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.lang.aql.parser.AQLParser;
+import org.apache.asterix.compiler.provider.AqlCompilationProvider;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.lang.aql.parser.TokenMgrError;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.result.ResultReader;
@@ -57,8 +61,36 @@ public class APIServlet extends HttpServlet {
private static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
+ private final ILangCompilationProvider aqlCompilationProvider;
+ private final IParserFactory aqlParserFactory;
+ private final ILangCompilationProvider sqlppCompilationProvider;
+ private final IParserFactory sqlppParserFactory;
+
+ public APIServlet() {
+ this.aqlCompilationProvider = new AqlCompilationProvider();
+ this.aqlParserFactory = aqlCompilationProvider.getParserFactory();
+
+ this.sqlppCompilationProvider = new SqlppCompilationProvider();
+ this.sqlppParserFactory = sqlppCompilationProvider.getParserFactory();
+ }
+
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException {
+ // Query language
+ ILangCompilationProvider compilationProvider;
+ IParserFactory parserFactory;
+ String lang = request.getParameter("query-language");
+ if (lang.equals("AQL")) {
+ // Uses AQL compiler.
+ compilationProvider = aqlCompilationProvider;
+ parserFactory = aqlParserFactory;
+ } else {
+ // Uses SQL++ compiler.
+ compilationProvider = sqlppCompilationProvider;
+ parserFactory = sqlppParserFactory;
+ }
+
+ // Output format.
OutputFormat format;
boolean csv_and_header = false;
String output = request.getParameter("output-format");
@@ -101,7 +133,7 @@ public class APIServlet extends HttpServlet {
context.setAttribute(HYRACKS_DATASET_ATTR, hds);
}
}
- AQLParser parser = new AQLParser(query);
+ IParser parser = parserFactory.createParser(query);
List<Statement> aqlStatements = parser.parse();
SessionConfig sessionConfig = new SessionConfig(out, format, true, isSet(executeQuery), true);
sessionConfig.set(SessionConfig.FORMAT_HTML, true);
@@ -110,10 +142,10 @@ public class APIServlet extends HttpServlet {
sessionConfig.setOOBData(isSet(printExprParam), isSet(printRewrittenExprParam),
isSet(printLogicalPlanParam), isSet(printOptimizedLogicalPlanParam), isSet(printJob));
MetadataManager.INSTANCE.init();
- AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, sessionConfig);
+ QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
double duration = 0;
long startTime = System.currentTimeMillis();
- aqlTranslator.compileAndExecute(hcc, hds, AqlTranslator.ResultDelivery.SYNC);
+ translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC);
long endTime = System.currentTimeMillis();
duration = (endTime - startTime) / 1000.00;
out.println(APIFramework.HTML_STATEMENT_SEPARATOR);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
index 9a9b345..f9dc45c 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
@@ -23,6 +23,7 @@ import java.util.List;
import javax.servlet.http.HttpServletRequest;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement.Kind;
public class AQLAPIServlet extends RESTAPIServlet {
@@ -33,6 +34,10 @@ public class AQLAPIServlet extends RESTAPIServlet {
private static final List<Kind> allowedStatements = new ArrayList<>();
+ public AQLAPIServlet(ILangCompilationProvider compilationProvider) {
+ super(compilationProvider);
+ }
+
static {
for (Kind k : Kind.values()) {
allowedStatements.add(k);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
index 65760a0..0254d6d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
@@ -23,12 +23,17 @@ import java.util.List;
import javax.servlet.http.HttpServletRequest;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.base.Statement.Kind;
public class DDLAPIServlet extends RESTAPIServlet {
private static final long serialVersionUID = 1L;
+ public DDLAPIServlet(ILangCompilationProvider compilationProvider) {
+ super(compilationProvider);
+ }
+
protected String getQueryParameter(HttpServletRequest request) {
return request.getParameter("ddl");
}
@@ -37,7 +42,8 @@ public class DDLAPIServlet extends RESTAPIServlet {
Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DATAVERSE_DROP, Kind.DATASET_DECL, Kind.NODEGROUP_DECL,
Kind.NODEGROUP_DROP, Kind.TYPE_DECL, Kind.TYPE_DROP, Kind.CREATE_INDEX, Kind.INDEX_DECL,
Kind.CREATE_DATAVERSE, Kind.DATASET_DROP, Kind.INDEX_DROP, Kind.CREATE_FUNCTION, Kind.FUNCTION_DROP,
- Kind.CREATE_PRIMARY_FEED, Kind.CREATE_SECONDARY_FEED, Kind.DROP_FEED, Kind.CREATE_FEED_POLICY, Kind.DROP_FEED_POLICY };
+ Kind.CREATE_PRIMARY_FEED, Kind.CREATE_SECONDARY_FEED, Kind.DROP_FEED, Kind.CREATE_FEED_POLICY,
+ Kind.DROP_FEED_POLICY };
return Arrays.asList(statementsArray);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
index 9f7657d..9ada5ff 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
@@ -23,18 +23,24 @@ import java.util.List;
import javax.servlet.http.HttpServletRequest;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.base.Statement.Kind;
public class QueryAPIServlet extends RESTAPIServlet {
private static final long serialVersionUID = 1L;
+ public QueryAPIServlet(ILangCompilationProvider compilationProvider) {
+ super(compilationProvider);
+ }
+
protected String getQueryParameter(HttpServletRequest request) {
return request.getParameter("query");
}
protected List<Statement.Kind> getAllowedStatements() {
- Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.FUNCTION_DECL, Kind.QUERY, Kind.SET, Kind.WRITE, Kind.RUN };
+ Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.FUNCTION_DECL, Kind.QUERY, Kind.SET, Kind.WRITE,
+ Kind.RUN };
return Arrays.asList(statementsArray);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
index fdaee18..0fd782d 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
@@ -32,12 +32,13 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.asterix.api.common.SessionConfig;
import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.AqlTranslator;
+import org.apache.asterix.aql.translator.QueryTranslator;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.lang.aql.parser.AQLParser;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.aql.parser.TokenMgrError;
-import org.apache.asterix.lang.aql.util.AQLFormatPrintUtil;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.base.Statement.Kind;
import org.apache.asterix.metadata.MetadataManager;
@@ -56,6 +57,14 @@ abstract class RESTAPIServlet extends HttpServlet {
public static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
+ private final ILangCompilationProvider compilationProvider;
+ private final IParserFactory parserFactory;
+
+ public RESTAPIServlet(ILangCompilationProvider compilationProvider) {
+ this.compilationProvider = compilationProvider;
+ this.parserFactory = compilationProvider.getParserFactory();
+ }
+
/**
* Initialize the Content-Type of the response, and construct a
* SessionConfig with the appropriate output writer and output-format
@@ -90,8 +99,9 @@ abstract class RESTAPIServlet extends HttpServlet {
}
// If it's JSON, check for the "lossless" flag
- if (format == OutputFormat.CLEAN_JSON &&
- ("true".equals(request.getParameter("lossless")) || accept.contains("lossless=true")) ) {
+
+ if (format == OutputFormat.CLEAN_JSON
+ && ("true".equals(request.getParameter("lossless")) || accept.contains("lossless=true"))) {
format = OutputFormat.LOSSLESS_JSON;
}
@@ -131,8 +141,7 @@ abstract class RESTAPIServlet extends HttpServlet {
break;
case CSV: {
// Check for header parameter or in Accept:.
- if ("present".equals(request.getParameter("header")) ||
- accept.contains("header=present")) {
+ if ("present".equals(request.getParameter("header")) || accept.contains("header=present")) {
response.setContentType("text/csv; header=present");
sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true);
} else {
@@ -145,8 +154,8 @@ abstract class RESTAPIServlet extends HttpServlet {
}
@Override
- protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException,
- IOException {
+ protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
StringWriter sw = new StringWriter();
IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name());
String query = sw.toString();
@@ -162,7 +171,7 @@ abstract class RESTAPIServlet extends HttpServlet {
public void handleRequest(HttpServletRequest request, HttpServletResponse response, String query)
throws IOException {
SessionConfig sessionConfig = initResponse(request, response);
- AqlTranslator.ResultDelivery resultDelivery = whichResultDelivery(request);
+ QueryTranslator.ResultDelivery resultDelivery = whichResultDelivery(request);
ServletContext context = getServletContext();
IHyracksClientConnection hcc;
@@ -178,12 +187,12 @@ abstract class RESTAPIServlet extends HttpServlet {
}
}
- AQLParser parser = new AQLParser(query);
+ IParser parser = parserFactory.createParser(query);
List<Statement> aqlStatements = parser.parse();
if (!containsForbiddenStatements(aqlStatements)) {
MetadataManager.INSTANCE.init();
- AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, sessionConfig);
- aqlTranslator.compileAndExecute(hcc, hds, resultDelivery);
+ QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+ translator.compileAndExecute(hcc, hds, resultDelivery);
}
} catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe);
@@ -207,16 +216,16 @@ abstract class RESTAPIServlet extends HttpServlet {
return false;
}
- protected AqlTranslator.ResultDelivery whichResultDelivery(HttpServletRequest request) {
+ protected QueryTranslator.ResultDelivery whichResultDelivery(HttpServletRequest request) {
String mode = request.getParameter("mode");
if (mode != null) {
if (mode.equals("asynchronous")) {
- return AqlTranslator.ResultDelivery.ASYNC;
+ return QueryTranslator.ResultDelivery.ASYNC;
} else if (mode.equals("asynchronous-deferred")) {
- return AqlTranslator.ResultDelivery.ASYNC_DEFERRED;
+ return QueryTranslator.ResultDelivery.ASYNC_DEFERRED;
}
}
- return AqlTranslator.ResultDelivery.SYNC;
+ return QueryTranslator.ResultDelivery.SYNC;
}
protected abstract String getQueryParameter(HttpServletRequest request);
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
index d08134f..d75625f 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
@@ -23,19 +23,25 @@ import java.util.List;
import javax.servlet.http.HttpServletRequest;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.base.Statement.Kind;
public class UpdateAPIServlet extends RESTAPIServlet {
private static final long serialVersionUID = 1L;
+ public UpdateAPIServlet(ILangCompilationProvider compilationProvider) {
+ super(compilationProvider);
+ }
+
protected String getQueryParameter(HttpServletRequest request) {
return request.getParameter("statements");
}
protected List<Statement.Kind> getAllowedStatements() {
Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPDATE, Kind.DML_CMD_LIST,
- Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT, Kind.EXTERNAL_DATASET_REFRESH };
+ Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT,
+ Kind.EXTERNAL_DATASET_REFRESH };
return Arrays.asList(statementsArray);
}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/caea8f0e/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java b/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
index a9b012d..964893b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
+++ b/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
@@ -26,8 +26,10 @@ import org.apache.asterix.api.common.APIFramework;
import org.apache.asterix.api.common.Job;
import org.apache.asterix.api.common.SessionConfig;
import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.AqlTranslator;
-import org.apache.asterix.lang.aql.parser.AQLParser;
+import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.hyracks.api.client.IHyracksClientConnection;
@@ -41,14 +43,23 @@ public class AsterixJavaClient {
private Job[] dmlJobs;
private JobSpecification queryJobSpec;
- public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText, PrintWriter writer) {
+ private final ILangCompilationProvider compilationProvider;
+ private final IParserFactory parserFactory;
+ private final APIFramework apiFramework;
+
+ public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText, PrintWriter writer,
+ ILangCompilationProvider compilationProvider) {
this.hcc = hcc;
this.queryText = queryText;
this.writer = writer;
+ this.compilationProvider = compilationProvider;
+ parserFactory = compilationProvider.getParserFactory();
+ this.apiFramework = new APIFramework(compilationProvider);
}
- public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText) {
- this(hcc, queryText, new PrintWriter(System.out, true));
+ public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText,
+ ILangCompilationProvider compilationProvider) {
+ this(hcc, queryText, new PrintWriter(System.out, true), compilationProvider);
}
public void compile() throws Exception {
@@ -69,8 +80,8 @@ public class AsterixJavaClient {
while ((ch = queryText.read()) != -1) {
builder.append((char) ch);
}
- AQLParser parser = new AQLParser(builder.toString());
- List<Statement> aqlStatements = parser.parse();
+ IParser parser = parserFactory.createParser(builder.toString());
+ List<Statement> statements = parser.parse();
MetadataManager.INSTANCE.init();
SessionConfig conf = new SessionConfig(writer, OutputFormat.ADM, optimize, true, generateBinaryRuntime);
@@ -79,17 +90,17 @@ public class AsterixJavaClient {
conf.set(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS, true);
}
- AqlTranslator aqlTranslator = new AqlTranslator(aqlStatements, conf);
- aqlTranslator.compileAndExecute(hcc, null, AqlTranslator.ResultDelivery.SYNC);
+ QueryTranslator translator = new QueryTranslator(statements, conf, compilationProvider);
+ translator.compileAndExecute(hcc, null, QueryTranslator.ResultDelivery.SYNC);
writer.flush();
}
public void execute() throws Exception {
if (dmlJobs != null) {
- APIFramework.executeJobArray(hcc, dmlJobs, writer);
+ apiFramework.executeJobArray(hcc, dmlJobs, writer);
}
if (queryJobSpec != null) {
- APIFramework.executeJobArray(hcc, new JobSpecification[] { queryJobSpec }, writer);
+ apiFramework.executeJobArray(hcc, new JobSpecification[] { queryJobSpec }, writer);
}
}