You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by kw...@apache.org on 2016/09/30 02:14:55 UTC
[38/61] [partial] incubator-impala git commit: IMPALA-3786: Replace
"cloudera" with "apache" (part 1)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/QueryStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/QueryStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/QueryStmt.java
deleted file mode 100644
index 791305b..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/QueryStmt.java
+++ /dev/null
@@ -1,458 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Set;
-
-import com.cloudera.impala.catalog.Type;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.common.TreeNode;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * Abstract base class for any statement that returns results
- * via a list of result expressions, for example a
- * SelectStmt or UnionStmt. Also maintains a map of expression substitutions
- * for replacing expressions from ORDER BY or GROUP BY clauses with
- * their corresponding result expressions.
- * Used for sharing members/methods and some of the analysis code, in particular the
- * analysis of the ORDER BY and LIMIT clauses.
- *
- */
-public abstract class QueryStmt extends StatementBase {
- /////////////////////////////////////////
- // BEGIN: Members that need to be reset()
-
- protected WithClause withClause_;
-
- protected ArrayList<OrderByElement> orderByElements_;
- protected LimitElement limitElement_;
-
- // For a select statment:
- // original list of exprs in select clause (star-expanded, ordinals and
- // aliases substituted, agg output substituted)
- // For a union statement:
- // list of slotrefs into the tuple materialized by the union.
- protected ArrayList<Expr> resultExprs_ = Lists.newArrayList();
-
- // For a select statment: select list exprs resolved to base tbl refs
- // For a union statement: same as resultExprs
- protected ArrayList<Expr> baseTblResultExprs_ = Lists.newArrayList();
-
- /**
- * Map of expression substitutions for replacing aliases
- * in "order by" or "group by" clauses with their corresponding result expr.
- */
- protected final ExprSubstitutionMap aliasSmap_;
-
- /**
- * Select list item alias does not have to be unique.
- * This list contains all the non-unique aliases. For example,
- * select int_col a, string_col a from alltypessmall;
- * Both columns are using the same alias "a".
- */
- protected final ArrayList<Expr> ambiguousAliasList_;
-
- protected SortInfo sortInfo_;
-
- // evaluateOrderBy_ is true if there is an order by clause that must be evaluated.
- // False for nested query stmts with an order-by clause without offset/limit.
- // sortInfo_ is still generated and used in analysis to ensure that the order-by clause
- // is well-formed.
- protected boolean evaluateOrderBy_;
-
- /////////////////////////////////////////
- // END: Members that need to be reset()
-
- QueryStmt(ArrayList<OrderByElement> orderByElements, LimitElement limitElement) {
- orderByElements_ = orderByElements;
- sortInfo_ = null;
- limitElement_ = limitElement == null ? new LimitElement(null, null) : limitElement;
- aliasSmap_ = new ExprSubstitutionMap();
- ambiguousAliasList_ = Lists.newArrayList();
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (isAnalyzed()) return;
- super.analyze(analyzer);
- analyzeLimit(analyzer);
- if (hasWithClause()) withClause_.analyze(analyzer);
- }
-
- /**
- * Returns a list containing all the materialized tuple ids that this stmt is
- * correlated with (i.e., those tuple ids from outer query blocks that TableRefs
- * inside this stmt are rooted at).
- *
- * Throws if this stmt contains an illegal mix of un/correlated table refs.
- * A statement is illegal if it contains a TableRef correlated with a parent query
- * block as well as a table ref with an absolute path (e.g. a BaseTabeRef). Such a
- * statement would generate a Subplan containing a base table scan (very expensive),
- * and should therefore be avoided.
- *
- * In other words, the following cases are legal:
- * (1) only uncorrelated table refs
- * (2) only correlated table refs
- * (3) a mix of correlated table refs and table refs rooted at those refs
- * (the statement is 'self-contained' with respect to correlation)
- */
- public List<TupleId> getCorrelatedTupleIds(Analyzer analyzer)
- throws AnalysisException {
- // Correlated tuple ids of this stmt.
- List<TupleId> correlatedTupleIds = Lists.newArrayList();
- // First correlated and absolute table refs. Used for error detection/reporting.
- // We pick the first ones for simplicity. Choosing arbitrary ones is equally valid.
- TableRef correlatedRef = null;
- TableRef absoluteRef = null;
- // Materialized tuple ids of the table refs checked so far.
- Set<TupleId> tblRefIds = Sets.newHashSet();
-
- List<TableRef> tblRefs = Lists.newArrayList();
- collectTableRefs(tblRefs);
- for (TableRef tblRef: tblRefs) {
- if (absoluteRef == null && !tblRef.isRelative()) absoluteRef = tblRef;
- if (tblRef.isCorrelated()) {
- // Check if the correlated table ref is rooted at a tuple descriptor from within
- // this query stmt. If so, the correlation is contained within this stmt
- // and the table ref does not conflict with absolute refs.
- CollectionTableRef t = (CollectionTableRef) tblRef;
- Preconditions.checkState(t.getResolvedPath().isRootedAtTuple());
- // This check relies on tblRefs being in depth-first order.
- if (!tblRefIds.contains(t.getResolvedPath().getRootDesc().getId())) {
- if (correlatedRef == null) correlatedRef = tblRef;
- correlatedTupleIds.add(t.getResolvedPath().getRootDesc().getId());
- }
- }
- if (correlatedRef != null && absoluteRef != null) {
- throw new AnalysisException(String.format(
- "Nested query is illegal because it contains a table reference '%s' " +
- "correlated with an outer block as well as an uncorrelated one '%s':\n%s",
- correlatedRef.tableRefToSql(), absoluteRef.tableRefToSql(), toSql()));
- }
- tblRefIds.add(tblRef.getId());
- }
- return correlatedTupleIds;
- }
-
- private void analyzeLimit(Analyzer analyzer) throws AnalysisException {
- if (limitElement_.getOffsetExpr() != null && !hasOrderByClause()) {
- throw new AnalysisException("OFFSET requires an ORDER BY clause: " +
- limitElement_.toSql().trim());
- }
- limitElement_.analyze(analyzer);
- }
-
- /**
- * Creates sortInfo by resolving aliases and ordinals in the orderingExprs.
- * If the query stmt is an inline view/union operand, then order-by with no
- * limit with offset is not allowed, since that requires a sort and merging-exchange,
- * and subsequent query execution would occur on a single machine.
- * Sets evaluateOrderBy_ to false for ignored order-by w/o limit/offset in nested
- * queries.
- */
- protected void createSortInfo(Analyzer analyzer) throws AnalysisException {
- // not computing order by
- if (orderByElements_ == null) {
- evaluateOrderBy_ = false;
- return;
- }
-
- ArrayList<Expr> orderingExprs = Lists.newArrayList();
- ArrayList<Boolean> isAscOrder = Lists.newArrayList();
- ArrayList<Boolean> nullsFirstParams = Lists.newArrayList();
-
- // extract exprs
- for (OrderByElement orderByElement: orderByElements_) {
- if (orderByElement.getExpr().contains(Predicates.instanceOf(Subquery.class))) {
- throw new AnalysisException(
- "Subqueries are not supported in the ORDER BY clause.");
- }
- // create copies, we don't want to modify the original parse node, in case
- // we need to print it
- orderingExprs.add(orderByElement.getExpr().clone());
- isAscOrder.add(Boolean.valueOf(orderByElement.isAsc()));
- nullsFirstParams.add(orderByElement.getNullsFirstParam());
- }
- substituteOrdinalsAliases(orderingExprs, "ORDER BY", analyzer);
-
- if (!analyzer.isRootAnalyzer() && hasOffset() && !hasLimit()) {
- throw new AnalysisException("Order-by with offset without limit not supported" +
- " in nested queries.");
- }
-
- sortInfo_ = new SortInfo(orderingExprs, isAscOrder, nullsFirstParams);
- // order by w/o limit and offset in inline views, union operands and insert statements
- // are ignored.
- if (!hasLimit() && !hasOffset() && !analyzer.isRootAnalyzer()) {
- evaluateOrderBy_ = false;
- // Return a warning that the order by was ignored.
- StringBuilder strBuilder = new StringBuilder();
- strBuilder.append("Ignoring ORDER BY clause without LIMIT or OFFSET: ");
- strBuilder.append("ORDER BY ");
- strBuilder.append(orderByElements_.get(0).toSql());
- for (int i = 1; i < orderByElements_.size(); ++i) {
- strBuilder.append(", ").append(orderByElements_.get(i).toSql());
- }
- strBuilder.append(".\nAn ORDER BY appearing in a view, subquery, union operand, ");
- strBuilder.append("or an insert/ctas statement has no effect on the query result ");
- strBuilder.append("unless a LIMIT and/or OFFSET is used in conjunction ");
- strBuilder.append("with the ORDER BY.");
- analyzer.addWarning(strBuilder.toString());
- } else {
- evaluateOrderBy_ = true;
- }
- }
-
- /**
- * Create a tuple descriptor for the single tuple that is materialized, sorted and
- * output by the exec node implementing the sort. Done by materializing slot refs in
- * the order-by and result expressions. Those SlotRefs in the ordering and result exprs
- * are substituted with SlotRefs into the new tuple. This simplifies sorting logic for
- * total (no limit) sorts.
- * Done after analyzeAggregation() since ordering and result exprs may refer to
- * the outputs of aggregation. Invoked for UnionStmt as well since
- * TODO: We could do something more sophisticated than simply copying input
- * slotrefs - e.g. compute some order-by expressions.
- */
- protected void createSortTupleInfo(Analyzer analyzer) throws AnalysisException {
- Preconditions.checkState(evaluateOrderBy_);
-
- for (Expr orderingExpr: sortInfo_.getOrderingExprs()) {
- if (orderingExpr.getType().isComplexType()) {
- throw new AnalysisException(String.format("ORDER BY expression '%s' with " +
- "complex type '%s' is not supported.", orderingExpr.toSql(),
- orderingExpr.getType().toSql()));
- }
- }
-
- // sourceSlots contains the slots from the input row to materialize.
- Set<SlotRef> sourceSlots = Sets.newHashSet();
- TreeNode.collect(resultExprs_, Predicates.instanceOf(SlotRef.class), sourceSlots);
- TreeNode.collect(sortInfo_.getOrderingExprs(), Predicates.instanceOf(SlotRef.class),
- sourceSlots);
-
- TupleDescriptor sortTupleDesc = analyzer.getDescTbl().createTupleDescriptor("sort");
- List<Expr> sortTupleExprs = Lists.newArrayList();
- sortTupleDesc.setIsMaterialized(true);
- // substOrderBy is the mapping from slot refs in the input row to slot refs in the
- // materialized sort tuple.
- ExprSubstitutionMap substOrderBy = new ExprSubstitutionMap();
- for (SlotRef origSlotRef: sourceSlots) {
- SlotDescriptor origSlotDesc = origSlotRef.getDesc();
- SlotDescriptor materializedDesc =
- analyzer.copySlotDescriptor(origSlotDesc, sortTupleDesc);
- SlotRef cloneRef = new SlotRef(materializedDesc);
- substOrderBy.put(origSlotRef, cloneRef);
- if (hasLimit()) {
- analyzer.registerValueTransfer(origSlotRef.getSlotId(), cloneRef.getSlotId());
- } else {
- analyzer.createAuxEquivPredicate(cloneRef, origSlotRef);
- }
- sortTupleExprs.add(origSlotRef);
- }
-
- resultExprs_ = Expr.substituteList(resultExprs_, substOrderBy, analyzer, false);
- sortInfo_.substituteOrderingExprs(substOrderBy, analyzer);
- sortInfo_.setMaterializedTupleInfo(sortTupleDesc, sortTupleExprs);
- }
-
- /**
- * Return the first expr in exprs that is a non-unique alias. Return null if none of
- * exprs is an ambiguous alias.
- */
- protected Expr getFirstAmbiguousAlias(List<Expr> exprs) {
- for (Expr exp: exprs) {
- if (ambiguousAliasList_.contains(exp)) return exp;
- }
- return null;
- }
-
- /**
- * Substitute exprs of the form "<number>" with the corresponding
- * expressions and any alias references in aliasSmap_.
- * Modifies exprs list in-place.
- */
- protected void substituteOrdinalsAliases(List<Expr> exprs, String errorPrefix,
- Analyzer analyzer) throws AnalysisException {
- Expr ambiguousAlias = getFirstAmbiguousAlias(exprs);
- if (ambiguousAlias != null) {
- throw new AnalysisException("Column '" + ambiguousAlias.toSql() +
- "' in " + errorPrefix + " clause is ambiguous");
- }
-
- ListIterator<Expr> i = exprs.listIterator();
- while (i.hasNext()) {
- Expr expr = i.next();
- // We can substitute either by ordinal or by alias.
- // If we substitute by ordinal, we should not replace any aliases, since
- // the new expression was copied from the select clause context, where
- // alias substitution is not performed in the same way.
- Expr substituteExpr = trySubstituteOrdinal(expr, errorPrefix, analyzer);
- if (substituteExpr == null) {
- substituteExpr = expr.trySubstitute(aliasSmap_, analyzer, false);
- }
- i.set(substituteExpr);
- }
- }
-
- // Attempt to replace an expression of form "<number>" with the corresponding
- // select list items. Return null if not an ordinal expression.
- private Expr trySubstituteOrdinal(Expr expr, String errorPrefix,
- Analyzer analyzer) throws AnalysisException {
- if (!(expr instanceof NumericLiteral)) return null;
- expr.analyze(analyzer);
- if (!expr.getType().isIntegerType()) return null;
- long pos = ((NumericLiteral) expr).getLongValue();
- if (pos < 1) {
- throw new AnalysisException(
- errorPrefix + ": ordinal must be >= 1: " + expr.toSql());
- }
- if (pos > resultExprs_.size()) {
- throw new AnalysisException(
- errorPrefix + ": ordinal exceeds number of items in select list: "
- + expr.toSql());
- }
-
- // Create copy to protect against accidentally shared state.
- return resultExprs_.get((int) pos - 1).clone();
- }
-
- /**
- * UnionStmt and SelectStmt have different implementations.
- */
- public abstract ArrayList<String> getColLabels();
-
- /**
- * Returns the materialized tuple ids of the output of this stmt.
- * Used in case this stmt is part of an @InlineViewRef,
- * since we need to know the materialized tupls ids of a TableRef.
- * This call must be idempotent because it may be called more than once for Union stmt.
- * TODO: The name of this function has become outdated due to analytics
- * producing logical (non-materialized) tuples. Re-think and clean up.
- */
- public abstract void getMaterializedTupleIds(ArrayList<TupleId> tupleIdList);
-
- /**
- * Returns all physical (non-inline-view) TableRefs of this statement and the nested
- * statements of inline views. The returned TableRefs are in depth-first order.
- */
- public abstract void collectTableRefs(List<TableRef> tblRefs);
-
- public void setWithClause(WithClause withClause) { this.withClause_ = withClause; }
- public boolean hasWithClause() { return withClause_ != null; }
- public WithClause getWithClause() { return withClause_; }
- public boolean hasOrderByClause() { return orderByElements_ != null; }
- public boolean hasLimit() { return limitElement_.getLimitExpr() != null; }
- public long getLimit() { return limitElement_.getLimit(); }
- public boolean hasOffset() { return limitElement_.getOffsetExpr() != null; }
- public long getOffset() { return limitElement_.getOffset(); }
- public SortInfo getSortInfo() { return sortInfo_; }
- public boolean evaluateOrderBy() { return evaluateOrderBy_; }
- public ArrayList<Expr> getResultExprs() { return resultExprs_; }
- public ArrayList<Expr> getBaseTblResultExprs() { return baseTblResultExprs_; }
- public void setLimit(long limit) throws AnalysisException {
- Preconditions.checkState(limit >= 0);
- long newLimit = hasLimit() ? Math.min(limit, getLimit()) : limit;
- limitElement_ = new LimitElement(new NumericLiteral(Long.toString(newLimit),
- Type.BIGINT), null);
- }
-
- /**
- * Mark all slots that need to be materialized for the execution of this stmt.
- * This excludes slots referenced in resultExprs (it depends on the consumer of
- * the output of the stmt whether they'll be accessed) and single-table predicates
- * (the PlanNode that materializes that tuple can decide whether evaluating those
- * predicates requires slot materialization).
- * This is called prior to plan tree generation and allows tuple-materializing
- * PlanNodes to compute their tuple's mem layout.
- */
- public abstract void materializeRequiredSlots(Analyzer analyzer);
-
- /**
- * Mark slots referenced in exprs as materialized.
- */
- protected void materializeSlots(Analyzer analyzer, List<Expr> exprs) {
- List<SlotId> slotIds = Lists.newArrayList();
- for (Expr e: exprs) {
- e.getIds(null, slotIds);
- }
- analyzer.getDescTbl().markSlotsMaterialized(slotIds);
- }
-
- /**
- * Substitutes the result expressions with smap. Preserves the original types of
- * those expressions during the substitution.
- */
- public void substituteResultExprs(ExprSubstitutionMap smap, Analyzer analyzer) {
- resultExprs_ = Expr.substituteList(resultExprs_, smap, analyzer, true);
- }
-
- public ArrayList<OrderByElement> cloneOrderByElements() {
- if (orderByElements_ == null) return null;
- ArrayList<OrderByElement> result =
- Lists.newArrayListWithCapacity(orderByElements_.size());
- for (OrderByElement o: orderByElements_) result.add(o.clone());
- return result;
- }
-
- public WithClause cloneWithClause() {
- return withClause_ != null ? withClause_.clone() : null;
- }
-
- /**
- * C'tor for cloning.
- */
- protected QueryStmt(QueryStmt other) {
- super(other);
- withClause_ = other.cloneWithClause();
- orderByElements_ = other.cloneOrderByElements();
- limitElement_ = other.limitElement_.clone();
- resultExprs_ = Expr.cloneList(other.resultExprs_);
- baseTblResultExprs_ = Expr.cloneList(other.baseTblResultExprs_);
- aliasSmap_ = other.aliasSmap_.clone();
- ambiguousAliasList_ = Expr.cloneList(other.ambiguousAliasList_);
- sortInfo_ = (other.sortInfo_ != null) ? other.sortInfo_.clone() : null;
- analyzer_ = other.analyzer_;
- evaluateOrderBy_ = other.evaluateOrderBy_;
- }
-
- @Override
- public void reset() {
- super.reset();
- if (orderByElements_ != null) {
- for (OrderByElement o: orderByElements_) o.getExpr().reset();
- }
- limitElement_.reset();
- resultExprs_.clear();
- baseTblResultExprs_.clear();
- aliasSmap_.clear();
- ambiguousAliasList_.clear();
- sortInfo_ = null;
- evaluateOrderBy_ = false;
- }
-
- @Override
- public abstract QueryStmt clone();
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java
deleted file mode 100644
index ac6c390..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/ResetMetadataStmt.java
+++ /dev/null
@@ -1,110 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import com.cloudera.impala.authorization.Privilege;
-import com.cloudera.impala.authorization.PrivilegeRequest;
-import com.cloudera.impala.authorization.PrivilegeRequestBuilder;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.thrift.TResetMetadataRequest;
-import com.cloudera.impala.thrift.TTableName;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-
-/**
- * Representation of a REFRESH/INVALIDATE METADATA statement.
- */
-public class ResetMetadataStmt extends StatementBase {
- // Updated during analysis. Null if invalidating the entire catalog.
- private TableName tableName_;
-
- // true if it is a REFRESH statement.
- private final boolean isRefresh_;
-
- // not null when refreshing a single partition
- private final PartitionSpec partitionSpec_;
-
- public ResetMetadataStmt(TableName name, boolean isRefresh,
- PartitionSpec partitionSpec) {
- Preconditions.checkArgument(!isRefresh || name != null);
- Preconditions.checkArgument(isRefresh || partitionSpec == null);
- this.tableName_ = name;
- this.isRefresh_ = isRefresh;
- this.partitionSpec_ = partitionSpec;
- if (partitionSpec_ != null) partitionSpec_.setTableName(tableName_);
- }
-
- public TableName getTableName() { return tableName_; }
- public boolean isRefresh() { return isRefresh_; }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (tableName_ != null) {
- String dbName = analyzer.getTargetDbName(tableName_);
- tableName_ = new TableName(dbName, tableName_.getTbl());
-
- if (isRefresh_) {
- // Verify the user has privileges to access this table. Will throw if the parent
- // database does not exists. Don't call getTable() to avoid loading the table
- // metadata if it is not yet in this impalad's catalog cache.
- if (!analyzer.dbContainsTable(dbName, tableName_.getTbl(), Privilege.ANY)) {
- // Only throw an exception when the table does not exist for refresh statements
- // since 'invalidate metadata' should add/remove tables created/dropped external
- // to Impala.
- throw new AnalysisException(Analyzer.TBL_DOES_NOT_EXIST_ERROR_MSG + tableName_);
- }
- if (partitionSpec_ != null) {
- partitionSpec_.setPrivilegeRequirement(Privilege.ANY);
- partitionSpec_.analyze(analyzer);
- }
- } else {
- // Verify the user has privileges to access this table.
- analyzer.registerPrivReq(new PrivilegeRequestBuilder()
- .onTable(dbName, tableName_.getTbl()).any().toRequest());
- }
- } else {
- analyzer.registerPrivReq(new PrivilegeRequest(Privilege.ALL));
- }
- }
-
- @Override
- public String toSql() {
- StringBuilder result = new StringBuilder();
- if (isRefresh_) {
- result.append("INVALIDATE METADATA");
- } else {
- result.append("REFRESH");
- }
-
- if (tableName_ != null) result.append(" ").append(tableName_);
- if (partitionSpec_ != null) result.append(" " + partitionSpec_.toSql());
- return result.toString();
- }
-
- public TResetMetadataRequest toThrift() {
- TResetMetadataRequest params = new TResetMetadataRequest();
- params.setIs_refresh(isRefresh_);
- if (tableName_ != null) {
- params.setTable_name(new TTableName(tableName_.getDb(), tableName_.getTbl()));
- }
- if (partitionSpec_ != null) {
- params.setPartition_spec(partitionSpec_.toThrift());
- }
- return params;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/SelectList.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/SelectList.java b/fe/src/main/java/com/cloudera/impala/analysis/SelectList.java
deleted file mode 100644
index 429d488..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/SelectList.java
+++ /dev/null
@@ -1,94 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.List;
-
-import com.google.common.collect.Lists;
-
-/**
- * Select list items plus optional distinct clause and optional plan hints.
- */
-public class SelectList {
- private List<String> planHints_;
- private boolean isDistinct_;
-
- /////////////////////////////////////////
- // BEGIN: Members that need to be reset()
-
- private final List<SelectListItem> items_;
-
- // END: Members that need to be reset()
- /////////////////////////////////////////
-
- public SelectList(List<SelectListItem> items) {
- isDistinct_ = false;
- items_ = items;
- }
-
- public SelectList() {
- isDistinct_ = false;
- items_ = Lists.newArrayList();
- }
-
- public SelectList(List<SelectListItem> items, boolean isDistinct,
- List<String> planHints) {
- isDistinct_ = isDistinct;
- items_ = items;
- planHints_ = planHints;
- }
-
- /**
- * C'tor for cloning.
- */
- public SelectList(SelectList other) {
- planHints_ =
- (other.planHints_ != null) ? Lists.newArrayList(other.planHints_) : null;
- items_ = Lists.newArrayList();
- for (SelectListItem item: other.items_) {
- items_.add(item.clone());
- }
- isDistinct_ = other.isDistinct_;
- }
-
- public List<SelectListItem> getItems() { return items_; }
- public void setPlanHints(List<String> planHints) { planHints_ = planHints; }
- public List<String> getPlanHints() { return planHints_; }
- public boolean isDistinct() { return isDistinct_; }
- public void setIsDistinct(boolean value) { isDistinct_ = value; }
- public boolean hasPlanHints() { return planHints_ != null; }
-
- public void analyzePlanHints(Analyzer analyzer) {
- if (planHints_ == null) return;
- for (String hint: planHints_) {
- if (!hint.equalsIgnoreCase("straight_join")) {
- analyzer.addWarning("PLAN hint not recognized: " + hint);
- }
- analyzer.setIsStraightJoin();
- }
- }
-
- @Override
- public SelectList clone() { return new SelectList(this); }
-
- public void reset() {
- for (SelectListItem item: items_) {
- if (!item.isStar()) item.getExpr().reset();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/SelectListItem.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/SelectListItem.java b/fe/src/main/java/com/cloudera/impala/analysis/SelectListItem.java
deleted file mode 100644
index 96831f3..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/SelectListItem.java
+++ /dev/null
@@ -1,129 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.List;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-
-class SelectListItem {
- private final Expr expr_;
- private String alias_;
-
- // for "[path.]*" (excludes trailing '*')
- private final List<String> rawPath_;
- private final boolean isStar_;
-
- public SelectListItem(Expr expr, String alias) {
- super();
- Preconditions.checkNotNull(expr);
- expr_ = expr;
- alias_ = alias;
- isStar_ = false;
- rawPath_ = null;
- }
-
- // select list item corresponding to path_to_struct.*
- static public SelectListItem createStarItem(List<String> rawPath) {
- return new SelectListItem(rawPath);
- }
-
- private SelectListItem(List<String> path) {
- super();
- expr_ = null;
- isStar_ = true;
- rawPath_ = path;
- }
-
- public Expr getExpr() { return expr_; }
- public boolean isStar() { return isStar_; }
- public String getAlias() { return alias_; }
- public void setAlias(String alias) { alias_ = alias; }
- public List<String> getRawPath() { return rawPath_; }
-
- @Override
- public String toString() {
- if (!isStar_) {
- Preconditions.checkNotNull(expr_);
- return expr_.toSql() + ((alias_ != null) ? " " + alias_ : "");
- } else if (rawPath_ != null) {
- Preconditions.checkState(isStar_);
- return Joiner.on(".").join(rawPath_) + ".*";
- } else {
- return "*";
- }
- }
-
- public String toSql() {
- if (!isStar_) {
- Preconditions.checkNotNull(expr_);
- // Enclose aliases in quotes if Hive cannot parse them without quotes.
- // This is needed for view compatibility between Impala and Hive.
- String aliasSql = null;
- if (alias_ != null) aliasSql = ToSqlUtils.getIdentSql(alias_);
- return expr_.toSql() + ((aliasSql != null) ? " " + aliasSql : "");
- } else if (rawPath_ != null) {
- Preconditions.checkState(isStar_);
- StringBuilder result = new StringBuilder();
- for (String p: rawPath_) {
- if (result.length() > 0) result.append(".");
- result.append(ToSqlUtils.getIdentSql(p.toLowerCase()));
- }
- result.append(".*");
- return result.toString();
- } else {
- return "*";
- }
- }
-
- /**
- * Returns a column label for this select list item.
- * If an alias was given, then the column label is the lower case alias.
- * If expr is a SlotRef then directly use its lower case column name.
- * Otherwise, the label is the lower case toSql() of expr or a Hive auto-generated
- * column name (depending on useHiveColLabels).
- * Hive's auto-generated column labels have a "_c" prefix and a select-list pos suffix,
- * e.g., "_c0", "_c1", "_c2", etc.
- *
- * Using auto-generated columns that are consistent with Hive is important
- * for view compatibility between Impala and Hive.
- */
- public String toColumnLabel(int selectListPos, boolean useHiveColLabels) {
- if (alias_ != null) return alias_.toLowerCase();
- if (expr_ instanceof SlotRef) {
- SlotRef slotRef = (SlotRef) expr_;
- return Joiner.on(".").join(slotRef.getResolvedPath().getRawPath());
- }
- // Optionally return auto-generated column label.
- if (useHiveColLabels) return "_c" + selectListPos;
- // Abbreviate the toSql() for analytic exprs.
- if (expr_ instanceof AnalyticExpr) {
- AnalyticExpr expr = (AnalyticExpr) expr_;
- return expr.getFnCall().toSql() + " OVER(...)";
- }
- return expr_.toSql().toLowerCase();
- }
-
- @Override
- public SelectListItem clone() {
- if (isStar_) return createStarItem(rawPath_);
- return new SelectListItem(expr_.clone(), alias_);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/SelectStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/SelectStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/SelectStmt.java
deleted file mode 100644
index 864dcc8..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/SelectStmt.java
+++ /dev/null
@@ -1,1021 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.impala.analysis.Path.PathType;
-import com.cloudera.impala.catalog.Column;
-import com.cloudera.impala.catalog.StructField;
-import com.cloudera.impala.catalog.StructType;
-import com.cloudera.impala.catalog.Table;
-import com.cloudera.impala.catalog.TableLoadingException;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.common.ColumnAliasGenerator;
-import com.cloudera.impala.common.TableAliasGenerator;
-import com.cloudera.impala.common.TreeNode;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * Representation of a single select block, including GROUP BY, ORDER BY and HAVING
- * clauses.
- */
-public class SelectStmt extends QueryStmt {
- private final static Logger LOG = LoggerFactory.getLogger(SelectStmt.class);
-
- /////////////////////////////////////////
- // BEGIN: Members that need to be reset()
-
- protected SelectList selectList_;
- protected final ArrayList<String> colLabels_; // lower case column labels
- protected final FromClause fromClause_;
- protected Expr whereClause_;
- protected ArrayList<Expr> groupingExprs_;
- protected final Expr havingClause_; // original having clause
-
- // havingClause with aliases and agg output resolved
- private Expr havingPred_;
-
- // set if we have any kind of aggregation operation, include SELECT DISTINCT
- private AggregateInfo aggInfo_;
-
- // set if we have AnalyticExprs in the select list/order by clause
- private AnalyticInfo analyticInfo_;
-
- // SQL string of this SelectStmt before inline-view expression substitution.
- // Set in analyze().
- protected String sqlString_;
-
- // substitutes all exprs in this select block to reference base tables
- // directly
- private ExprSubstitutionMap baseTblSmap_ = new ExprSubstitutionMap();
-
- // END: Members that need to be reset()
- /////////////////////////////////////////
-
- SelectStmt(SelectList selectList,
- FromClause fromClause,
- Expr wherePredicate, ArrayList<Expr> groupingExprs,
- Expr havingPredicate, ArrayList<OrderByElement> orderByElements,
- LimitElement limitElement) {
- super(orderByElements, limitElement);
- selectList_ = selectList;
- if (fromClause == null) {
- fromClause_ = new FromClause();
- } else {
- fromClause_ = fromClause;
- }
- whereClause_ = wherePredicate;
- groupingExprs_ = groupingExprs;
- havingClause_ = havingPredicate;
- colLabels_ = Lists.newArrayList();
- havingPred_ = null;
- aggInfo_ = null;
- sortInfo_ = null;
- }
-
- /**
- * @return the original select list items from the query
- */
- public SelectList getSelectList() { return selectList_; }
-
- /**
- * @return the HAVING clause post-analysis and with aliases resolved
- */
- public Expr getHavingPred() { return havingPred_; }
-
- public List<TableRef> getTableRefs() { return fromClause_.getTableRefs(); }
- public boolean hasWhereClause() { return whereClause_ != null; }
- public boolean hasGroupByClause() { return groupingExprs_ != null; }
- public Expr getWhereClause() { return whereClause_; }
- public void setWhereClause(Expr whereClause) { whereClause_ = whereClause; }
- public AggregateInfo getAggInfo() { return aggInfo_; }
- public boolean hasAggInfo() { return aggInfo_ != null; }
- public AnalyticInfo getAnalyticInfo() { return analyticInfo_; }
- public boolean hasAnalyticInfo() { return analyticInfo_ != null; }
- public boolean hasHavingClause() { return havingClause_ != null; }
- @Override
- public ArrayList<String> getColLabels() { return colLabels_; }
- public ExprSubstitutionMap getBaseTblSmap() { return baseTblSmap_; }
-
- // Column alias generator used during query rewriting.
- private ColumnAliasGenerator columnAliasGenerator_ = null;
- public ColumnAliasGenerator getColumnAliasGenerator() {
- if (columnAliasGenerator_ == null) {
- columnAliasGenerator_ = new ColumnAliasGenerator(colLabels_, null);
- }
- return columnAliasGenerator_;
- }
-
- // Table alias generator used during query rewriting.
- private TableAliasGenerator tableAliasGenerator_ = null;
- public TableAliasGenerator getTableAliasGenerator() {
- if (tableAliasGenerator_ == null) {
- tableAliasGenerator_ = new TableAliasGenerator(analyzer_, null);
- }
- return tableAliasGenerator_;
- }
-
- /**
- * Creates resultExprs and baseTblResultExprs.
- */
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (isAnalyzed()) return;
- super.analyze(analyzer);
-
- fromClause_.analyze(analyzer);
-
- // Generate !empty() predicates to filter out empty collections.
- // Skip this step when analyzing a WITH-clause because CollectionTableRefs
- // do not register collection slots in their parent in that context
- // (see CollectionTableRef.analyze()).
- if (!analyzer.isWithClause()) registerIsNotEmptyPredicates(analyzer);
-
- // analyze plan hints from select list
- selectList_.analyzePlanHints(analyzer);
-
- // populate resultExprs_, aliasSmap_, and colLabels_
- for (int i = 0; i < selectList_.getItems().size(); ++i) {
- SelectListItem item = selectList_.getItems().get(i);
- if (item.isStar()) {
- if (item.getRawPath() != null) {
- Path resolvedPath = analyzeStarPath(item.getRawPath(), analyzer);
- expandStar(resolvedPath, analyzer);
- } else {
- expandStar(analyzer);
- }
- } else {
- // Analyze the resultExpr before generating a label to ensure enforcement
- // of expr child and depth limits (toColumn() label may call toSql()).
- item.getExpr().analyze(analyzer);
- if (item.getExpr().contains(Predicates.instanceOf(Subquery.class))) {
- throw new AnalysisException(
- "Subqueries are not supported in the select list.");
- }
- resultExprs_.add(item.getExpr());
- String label = item.toColumnLabel(i, analyzer.useHiveColLabels());
- SlotRef aliasRef = new SlotRef(label);
- Expr existingAliasExpr = aliasSmap_.get(aliasRef);
- if (existingAliasExpr != null && !existingAliasExpr.equals(item.getExpr())) {
- // If we have already seen this alias, it refers to more than one column and
- // therefore is ambiguous.
- ambiguousAliasList_.add(aliasRef);
- }
- aliasSmap_.put(aliasRef, item.getExpr().clone());
- colLabels_.add(label);
- }
- }
-
- // Star exprs only expand to the scalar-typed columns/fields, so
- // the resultExprs_ could be empty.
- if (resultExprs_.isEmpty()) {
- throw new AnalysisException("The star exprs expanded to an empty select list " +
- "because the referenced tables only have complex-typed columns.\n" +
- "Star exprs only expand to scalar-typed columns because complex-typed exprs " +
- "are currently not supported in the select list.\n" +
- "Affected select statement:\n" + toSql());
- }
-
- for (Expr expr: resultExprs_) {
- // Complex types are currently not supported in the select list because we'd need
- // to serialize them in a meaningful way.
- if (expr.getType().isComplexType()) {
- throw new AnalysisException(String.format(
- "Expr '%s' in select list returns a complex type '%s'.\n" +
- "Only scalar types are allowed in the select list.",
- expr.toSql(), expr.getType().toSql()));
- }
- if (!expr.getType().isSupported()) {
- throw new AnalysisException("Unsupported type '"
- + expr.getType().toSql() + "' in '" + expr.toSql() + "'.");
- }
- }
-
- if (TreeNode.contains(resultExprs_, AnalyticExpr.class)) {
- if (fromClause_.isEmpty()) {
- throw new AnalysisException("Analytic expressions require FROM clause.");
- }
-
- // do this here, not after analyzeAggregation(), otherwise the AnalyticExprs
- // will get substituted away
- if (selectList_.isDistinct()) {
- throw new AnalysisException(
- "cannot combine SELECT DISTINCT with analytic functions");
- }
- }
-
- if (whereClause_ != null) {
- whereClause_.analyze(analyzer);
- if (whereClause_.contains(Expr.isAggregatePredicate())) {
- throw new AnalysisException(
- "aggregate function not allowed in WHERE clause");
- }
- whereClause_.checkReturnsBool("WHERE clause", false);
- Expr e = whereClause_.findFirstOf(AnalyticExpr.class);
- if (e != null) {
- throw new AnalysisException(
- "WHERE clause must not contain analytic expressions: " + e.toSql());
- }
- analyzer.registerConjuncts(whereClause_, false);
- }
-
- createSortInfo(analyzer);
- analyzeAggregation(analyzer);
- createAnalyticInfo(analyzer);
- if (evaluateOrderBy_) createSortTupleInfo(analyzer);
-
- // Remember the SQL string before inline-view expression substitution.
- sqlString_ = toSql();
- resolveInlineViewRefs(analyzer);
-
- // If this block's select-project-join portion returns an empty result set and the
- // block has no aggregation, then mark this block as returning an empty result set.
- if (analyzer.hasEmptySpjResultSet() && aggInfo_ == null) {
- analyzer.setHasEmptyResultSet();
- }
-
- ColumnLineageGraph graph = analyzer.getColumnLineageGraph();
- if (aggInfo_ != null && !aggInfo_.getAggregateExprs().isEmpty()) {
- graph.addDependencyPredicates(aggInfo_.getGroupingExprs());
- }
- if (sortInfo_ != null && hasLimit()) {
- // When there is a LIMIT clause in conjunction with an ORDER BY, the ordering exprs
- // must be added in the column lineage graph.
- graph.addDependencyPredicates(sortInfo_.getOrderingExprs());
- }
-
- if (aggInfo_ != null) LOG.debug("post-analysis " + aggInfo_.debugString());
- }
-
- /**
- * Generates and registers !empty() predicates to filter out empty collections directly
- * in the parent scan of collection table refs. This is a performance optimization to
- * avoid the expensive processing of empty collections inside a subplan that would
- * yield an empty result set.
- *
- * For correctness purposes, the predicates are generated in cases where we can ensure
- * that they will be assigned only to the parent scan, and no other plan node.
- *
- * The conditions are as follows:
- * - collection table ref is relative and non-correlated
- * - collection table ref represents the rhs of an inner/cross/semi join
- * - collection table ref's parent tuple is not outer joined
- *
- * TODO: In some cases, it is possible to generate !empty() predicates for a correlated
- * table ref, but in general, that is not correct for non-trivial query blocks.
- * For example, if the block with the correlated ref has an aggregation then adding a
- * !empty() predicate would incorrectly discard rows from the final result set.
- * TODO: Evaluating !empty() predicates at non-scan nodes interacts poorly with our BE
- * projection of collection slots. For example, rows could incorrectly be filtered if
- * a !empty() predicate is assigned to a plan node that comes after the unnest of the
- * collection that also performs the projection.
- */
- private void registerIsNotEmptyPredicates(Analyzer analyzer) throws AnalysisException {
- for (TableRef tblRef: fromClause_.getTableRefs()) {
- Preconditions.checkState(tblRef.isResolved());
- if (!(tblRef instanceof CollectionTableRef)) continue;
- CollectionTableRef ref = (CollectionTableRef) tblRef;
- // Skip non-relative and correlated refs.
- if (!ref.isRelative() || ref.isCorrelated()) continue;
- // Skip outer and anti joins.
- if (ref.getJoinOp().isOuterJoin() || ref.getJoinOp().isAntiJoin()) continue;
- // Do not generate a predicate if the parent tuple is outer joined.
- if (analyzer.isOuterJoined(ref.getResolvedPath().getRootDesc().getId())) continue;
- IsNotEmptyPredicate isNotEmptyPred =
- new IsNotEmptyPredicate(ref.getCollectionExpr().clone());
- isNotEmptyPred.analyze(analyzer);
- // Register the predicate as an On-clause conjunct because it should only
- // affect the result of this join and not the whole FROM clause.
- analyzer.registerOnClauseConjuncts(
- Lists.<Expr>newArrayList(isNotEmptyPred), ref);
- }
- }
-
- /**
- * Marks all unassigned join predicates as well as exprs in aggInfo and sortInfo.
- */
- @Override
- public void materializeRequiredSlots(Analyzer analyzer) {
- // Mark unassigned join predicates. Some predicates that must be evaluated by a join
- // can also be safely evaluated below the join (picked up by getBoundPredicates()).
- // Such predicates will be marked twice and that is ok.
- List<Expr> unassigned =
- analyzer.getUnassignedConjuncts(getTableRefIds(), true);
- List<Expr> unassignedJoinConjuncts = Lists.newArrayList();
- for (Expr e: unassigned) {
- if (analyzer.evalByJoin(e)) unassignedJoinConjuncts.add(e);
- }
- List<Expr> baseTblJoinConjuncts =
- Expr.substituteList(unassignedJoinConjuncts, baseTblSmap_, analyzer, false);
- materializeSlots(analyzer, baseTblJoinConjuncts);
-
- if (evaluateOrderBy_) {
- // mark ordering exprs before marking agg/analytic exprs because they could contain
- // agg/analytic exprs that are not referenced anywhere but the ORDER BY clause
- sortInfo_.materializeRequiredSlots(analyzer, baseTblSmap_);
- }
-
- if (hasAnalyticInfo()) {
- // Mark analytic exprs before marking agg exprs because they could contain agg
- // exprs that are not referenced anywhere but the analytic expr.
- // Gather unassigned predicates and mark their slots. It is not desirable
- // to account for propagated predicates because if an analytic expr is only
- // referenced by a propagated predicate, then it's better to not materialize the
- // analytic expr at all.
- ArrayList<TupleId> tids = Lists.newArrayList();
- getMaterializedTupleIds(tids); // includes the analytic tuple
- List<Expr> conjuncts = analyzer.getUnassignedConjuncts(tids, false);
- materializeSlots(analyzer, conjuncts);
- analyticInfo_.materializeRequiredSlots(analyzer, baseTblSmap_);
- }
-
- if (aggInfo_ != null) {
- // mark all agg exprs needed for HAVING pred and binding predicates as materialized
- // before calling AggregateInfo.materializeRequiredSlots(), otherwise they won't
- // show up in AggregateInfo.getMaterializedAggregateExprs()
- ArrayList<Expr> havingConjuncts = Lists.newArrayList();
- if (havingPred_ != null) havingConjuncts.add(havingPred_);
- // Ignore predicates bound to a group-by slot because those
- // are already evaluated below this agg node (e.g., in a scan).
- Set<SlotId> groupBySlots = Sets.newHashSet();
- for (int i = 0; i < aggInfo_.getGroupingExprs().size(); ++i) {
- groupBySlots.add(aggInfo_.getOutputTupleDesc().getSlots().get(i).getId());
- }
- // Binding predicates are assigned to the final output tuple of the aggregation,
- // which is the tuple of the 2nd phase agg for distinct aggs.
- ArrayList<Expr> bindingPredicates =
- analyzer.getBoundPredicates(aggInfo_.getResultTupleId(), groupBySlots, false);
- havingConjuncts.addAll(bindingPredicates);
- havingConjuncts.addAll(
- analyzer.getUnassignedConjuncts(aggInfo_.getResultTupleId().asList(), false));
- materializeSlots(analyzer, havingConjuncts);
- aggInfo_.materializeRequiredSlots(analyzer, baseTblSmap_);
- }
- }
-
- /**
- * Populates baseTblSmap_ with our combined inline view smap and creates
- * baseTblResultExprs.
- */
- protected void resolveInlineViewRefs(Analyzer analyzer)
- throws AnalysisException {
- // Gather the inline view substitution maps from the enclosed inline views
- for (TableRef tblRef: fromClause_) {
- if (tblRef instanceof InlineViewRef) {
- InlineViewRef inlineViewRef = (InlineViewRef) tblRef;
- baseTblSmap_ =
- ExprSubstitutionMap.combine(baseTblSmap_, inlineViewRef.getBaseTblSmap());
- }
- }
- baseTblResultExprs_ =
- Expr.trySubstituteList(resultExprs_, baseTblSmap_, analyzer, false);
- LOG.trace("baseTblSmap_: " + baseTblSmap_.debugString());
- LOG.trace("resultExprs: " + Expr.debugString(resultExprs_));
- LOG.trace("baseTblResultExprs: " + Expr.debugString(baseTblResultExprs_));
- }
-
- public List<TupleId> getTableRefIds() {
- List<TupleId> result = Lists.newArrayList();
- for (TableRef ref: fromClause_) {
- result.add(ref.getId());
- }
- return result;
- }
-
- /**
- * Resolves the given raw path as a STAR path and checks its legality.
- * Returns the resolved legal path, or throws if the raw path could not
- * be resolved or is an illegal star path.
- */
- private Path analyzeStarPath(List<String> rawPath, Analyzer analyzer)
- throws AnalysisException {
- Path resolvedPath = null;
- try {
- resolvedPath = analyzer.resolvePath(rawPath, PathType.STAR);
- } catch (TableLoadingException e) {
- // Should never happen because we only check registered table aliases.
- Preconditions.checkState(false);
- }
- Preconditions.checkNotNull(resolvedPath);
- return resolvedPath;
- }
-
- /**
- * Expand "*" select list item, ignoring semi-joined tables as well as
- * complex-typed fields because those are currently illegal in any select
- * list (even for inline views, etc.)
- */
- private void expandStar(Analyzer analyzer) throws AnalysisException {
- if (fromClause_.isEmpty()) {
- throw new AnalysisException("'*' expression in select list requires FROM clause.");
- }
- // expand in From clause order
- for (TableRef tableRef: fromClause_) {
- if (analyzer.isSemiJoined(tableRef.getId())) continue;
- Path resolvedPath = new Path(tableRef.getDesc(), Collections.<String>emptyList());
- Preconditions.checkState(resolvedPath.resolve());
- expandStar(resolvedPath, analyzer);
- }
- }
-
- /**
- * Expand "path.*" from a resolved path, ignoring complex-typed fields because those
- * are currently illegal in any select list (even for inline views, etc.)
- */
- private void expandStar(Path resolvedPath, Analyzer analyzer)
- throws AnalysisException {
- Preconditions.checkState(resolvedPath.isResolved());
- if (resolvedPath.destTupleDesc() != null &&
- resolvedPath.destTupleDesc().getTable() != null &&
- resolvedPath.destTupleDesc().getPath().getMatchedTypes().isEmpty()) {
- // The resolved path targets a registered tuple descriptor of a catalog
- // table. Expand the '*' based on the Hive-column order.
- TupleDescriptor tupleDesc = resolvedPath.destTupleDesc();
- Table table = tupleDesc.getTable();
- for (Column c: table.getColumnsInHiveOrder()) {
- addStarResultExpr(resolvedPath, analyzer, c.getName());
- }
- } else {
- // The resolved path does not target the descriptor of a catalog table.
- // Expand '*' based on the destination type of the resolved path.
- Preconditions.checkState(resolvedPath.destType().isStructType());
- StructType structType = (StructType) resolvedPath.destType();
- Preconditions.checkNotNull(structType);
-
- // Star expansion for references to nested collections.
- // Collection Type Star Expansion
- // array<int> --> item
- // array<struct<f1,f2,...,fn>> --> f1, f2, ..., fn
- // map<int,int> --> key, value
- // map<int,struct<f1,f2,...,fn>> --> key, f1, f2, ..., fn
- if (structType instanceof CollectionStructType) {
- CollectionStructType cst = (CollectionStructType) structType;
- if (cst.isMapStruct()) {
- addStarResultExpr(resolvedPath, analyzer, Path.MAP_KEY_FIELD_NAME);
- }
- if (cst.getOptionalField().getType().isStructType()) {
- structType = (StructType) cst.getOptionalField().getType();
- for (StructField f: structType.getFields()) {
- addStarResultExpr(
- resolvedPath, analyzer, cst.getOptionalField().getName(), f.getName());
- }
- } else if (cst.isMapStruct()) {
- addStarResultExpr(resolvedPath, analyzer, Path.MAP_VALUE_FIELD_NAME);
- } else {
- addStarResultExpr(resolvedPath, analyzer, Path.ARRAY_ITEM_FIELD_NAME);
- }
- } else {
- // Default star expansion.
- for (StructField f: structType.getFields()) {
- addStarResultExpr(resolvedPath, analyzer, f.getName());
- }
- }
- }
- }
-
- /**
- * Helper function used during star expansion to add a single result expr
- * based on a given raw path to be resolved relative to an existing path.
- * Ignores paths with a complex-typed destination because they are currently
- * illegal in any select list (even for inline views, etc.)
- */
- private void addStarResultExpr(Path resolvedPath, Analyzer analyzer,
- String... relRawPath) throws AnalysisException {
- Path p = Path.createRelPath(resolvedPath, relRawPath);
- Preconditions.checkState(p.resolve());
- if (p.destType().isComplexType()) return;
- SlotDescriptor slotDesc = analyzer.registerSlotRef(p);
- SlotRef slotRef = new SlotRef(slotDesc);
- slotRef.analyze(analyzer);
- resultExprs_.add(slotRef);
- colLabels_.add(relRawPath[relRawPath.length - 1]);
- }
-
- /**
- * Analyze aggregation-relevant components of the select block (Group By clause,
- * select list, Order By clause), substitute AVG with SUM/COUNT, create the
- * AggregationInfo, including the agg output tuple, and transform all post-agg exprs
- * given AggregationInfo's smap.
- */
- private void analyzeAggregation(Analyzer analyzer) throws AnalysisException {
- // Analyze the HAVING clause first so we can check if it contains aggregates.
- // We need to analyze/register it even if we are not computing aggregates.
- if (havingClause_ != null) {
- if (havingClause_.contains(Predicates.instanceOf(Subquery.class))) {
- throw new AnalysisException(
- "Subqueries are not supported in the HAVING clause.");
- }
- // substitute aliases in place (ordinals not allowed in having clause)
- havingPred_ = havingClause_.substitute(aliasSmap_, analyzer, false);
- havingPred_.checkReturnsBool("HAVING clause", true);
- // can't contain analytic exprs
- Expr analyticExpr = havingPred_.findFirstOf(AnalyticExpr.class);
- if (analyticExpr != null) {
- throw new AnalysisException(
- "HAVING clause must not contain analytic expressions: "
- + analyticExpr.toSql());
- }
- }
-
- if (groupingExprs_ == null && !selectList_.isDistinct()
- && !TreeNode.contains(resultExprs_, Expr.isAggregatePredicate())
- && (havingPred_ == null
- || !havingPred_.contains(Expr.isAggregatePredicate()))
- && (sortInfo_ == null
- || !TreeNode.contains(sortInfo_.getOrderingExprs(),
- Expr.isAggregatePredicate()))) {
- // We're not computing aggregates but we still need to register the HAVING
- // clause which could, e.g., contain a constant expression evaluating to false.
- if (havingPred_ != null) analyzer.registerConjuncts(havingPred_, true);
- return;
- }
-
- // If we're computing an aggregate, we must have a FROM clause.
- if (fromClause_.isEmpty()) {
- throw new AnalysisException(
- "aggregation without a FROM clause is not allowed");
- }
-
- if (selectList_.isDistinct()
- && (groupingExprs_ != null
- || TreeNode.contains(resultExprs_, Expr.isAggregatePredicate())
- || (havingPred_ != null
- && havingPred_.contains(Expr.isAggregatePredicate())))) {
- throw new AnalysisException(
- "cannot combine SELECT DISTINCT with aggregate functions or GROUP BY");
- }
-
- // Disallow '*' with explicit GROUP BY or aggregation function (we can't group by
- // '*', and if you need to name all star-expanded cols in the group by clause you
- // might as well do it in the select list).
- if (groupingExprs_ != null ||
- TreeNode.contains(resultExprs_, Expr.isAggregatePredicate())) {
- for (SelectListItem item : selectList_.getItems()) {
- if (item.isStar()) {
- throw new AnalysisException(
- "cannot combine '*' in select list with grouping or aggregation");
- }
- }
- }
-
- // disallow subqueries in the GROUP BY clause
- if (groupingExprs_ != null) {
- for (Expr expr: groupingExprs_) {
- if (expr.contains(Predicates.instanceOf(Subquery.class))) {
- throw new AnalysisException(
- "Subqueries are not supported in the GROUP BY clause.");
- }
- }
- }
-
- // analyze grouping exprs
- ArrayList<Expr> groupingExprsCopy = Lists.newArrayList();
- if (groupingExprs_ != null) {
- // make a deep copy here, we don't want to modify the original
- // exprs during analysis (in case we need to print them later)
- groupingExprsCopy = Expr.cloneList(groupingExprs_);
-
- substituteOrdinalsAliases(groupingExprsCopy, "GROUP BY", analyzer);
-
- for (int i = 0; i < groupingExprsCopy.size(); ++i) {
- groupingExprsCopy.get(i).analyze(analyzer);
- if (groupingExprsCopy.get(i).contains(Expr.isAggregatePredicate())) {
- // reference the original expr in the error msg
- throw new AnalysisException(
- "GROUP BY expression must not contain aggregate functions: "
- + groupingExprs_.get(i).toSql());
- }
- if (groupingExprsCopy.get(i).contains(AnalyticExpr.class)) {
- // reference the original expr in the error msg
- throw new AnalysisException(
- "GROUP BY expression must not contain analytic expressions: "
- + groupingExprsCopy.get(i).toSql());
- }
- }
- }
-
- // Collect the aggregate expressions from the SELECT, HAVING and ORDER BY clauses
- // of this statement.
- ArrayList<FunctionCallExpr> aggExprs = Lists.newArrayList();
- TreeNode.collect(resultExprs_, Expr.isAggregatePredicate(), aggExprs);
- if (havingPred_ != null) {
- havingPred_.collect(Expr.isAggregatePredicate(), aggExprs);
- }
- if (sortInfo_ != null) {
- // TODO: Avoid evaluating aggs in ignored order-bys
- TreeNode.collect(sortInfo_.getOrderingExprs(), Expr.isAggregatePredicate(),
- aggExprs);
- }
-
- // Optionally rewrite all count(distinct <expr>) into equivalent NDV() calls.
- ExprSubstitutionMap ndvSmap = null;
- if (analyzer.getQueryCtx().getRequest().query_options.appx_count_distinct) {
- ndvSmap = new ExprSubstitutionMap();
- for (FunctionCallExpr aggExpr: aggExprs) {
- if (!aggExpr.isDistinct()
- || !aggExpr.getFnName().getFunction().equals("count")
- || aggExpr.getParams().size() != 1) {
- continue;
- }
- FunctionCallExpr ndvFnCall =
- new FunctionCallExpr("ndv", aggExpr.getParams().exprs());
- ndvFnCall.analyzeNoThrow(analyzer);
- Preconditions.checkState(ndvFnCall.getType().equals(aggExpr.getType()));
- ndvSmap.put(aggExpr, ndvFnCall);
- }
- // Replace all count(distinct <expr>) with NDV(<expr>).
- List<Expr> substAggExprs = Expr.substituteList(aggExprs, ndvSmap, analyzer, false);
- aggExprs.clear();
- for (Expr aggExpr: substAggExprs) {
- Preconditions.checkState(aggExpr instanceof FunctionCallExpr);
- aggExprs.add((FunctionCallExpr) aggExpr);
- }
- }
-
- // When DISTINCT aggregates are present, non-distinct (i.e. ALL) aggregates are
- // evaluated in two phases (see AggregateInfo for more details). In particular,
- // COUNT(c) in "SELECT COUNT(c), AGG(DISTINCT d) from R" is transformed to
- // "SELECT SUM(cnt) FROM (SELECT COUNT(c) as cnt from R group by d ) S".
- // Since a group-by expression is added to the inner query it returns no rows if
- // R is empty, in which case the SUM of COUNTs will return NULL.
- // However the original COUNT(c) should have returned 0 instead of NULL in this case.
- // Therefore, COUNT([ALL]) is transformed into zeroifnull(COUNT([ALL]) if
- // i) There is no GROUP-BY clause, and
- // ii) Other DISTINCT aggregates are present.
- ExprSubstitutionMap countAllMap = createCountAllMap(aggExprs, analyzer);
- countAllMap = ExprSubstitutionMap.compose(ndvSmap, countAllMap, analyzer);
- List<Expr> substitutedAggs =
- Expr.substituteList(aggExprs, countAllMap, analyzer, false);
- aggExprs.clear();
- TreeNode.collect(substitutedAggs, Expr.isAggregatePredicate(), aggExprs);
- createAggInfo(groupingExprsCopy, aggExprs, analyzer);
-
- // combine avg smap with the one that produces the final agg output
- AggregateInfo finalAggInfo =
- aggInfo_.getSecondPhaseDistinctAggInfo() != null
- ? aggInfo_.getSecondPhaseDistinctAggInfo()
- : aggInfo_;
-
- ExprSubstitutionMap combinedSmap =
- ExprSubstitutionMap.compose(countAllMap, finalAggInfo.getOutputSmap(), analyzer);
- LOG.trace("combined smap: " + combinedSmap.debugString());
-
- // change select list, having and ordering exprs to point to agg output. We need
- // to reanalyze the exprs at this point.
- LOG.trace("desctbl: " + analyzer.getDescTbl().debugString());
- LOG.trace("resultexprs: " + Expr.debugString(resultExprs_));
- resultExprs_ = Expr.substituteList(resultExprs_, combinedSmap, analyzer, false);
- LOG.trace("post-agg selectListExprs: " + Expr.debugString(resultExprs_));
- if (havingPred_ != null) {
- // Make sure the predicate in the HAVING clause does not contain a
- // subquery.
- Preconditions.checkState(!havingPred_.contains(
- Predicates.instanceOf(Subquery.class)));
- havingPred_ = havingPred_.substitute(combinedSmap, analyzer, false);
- analyzer.registerConjuncts(havingPred_, true);
- LOG.debug("post-agg havingPred: " + havingPred_.debugString());
- }
- if (sortInfo_ != null) {
- sortInfo_.substituteOrderingExprs(combinedSmap, analyzer);
- LOG.debug("post-agg orderingExprs: " +
- Expr.debugString(sortInfo_.getOrderingExprs()));
- }
-
- // check that all post-agg exprs point to agg output
- for (int i = 0; i < selectList_.getItems().size(); ++i) {
- if (!resultExprs_.get(i).isBound(finalAggInfo.getOutputTupleId())) {
- SelectListItem selectListItem = selectList_.getItems().get(i);
- Preconditions.checkState(!selectListItem.isStar());
- throw new AnalysisException(
- "select list expression not produced by aggregation output "
- + "(missing from GROUP BY clause?): "
- + selectListItem.getExpr().toSql());
- }
- }
- if (orderByElements_ != null) {
- for (int i = 0; i < orderByElements_.size(); ++i) {
- if (!sortInfo_.getOrderingExprs().get(i).isBound(
- finalAggInfo.getOutputTupleId())) {
- throw new AnalysisException(
- "ORDER BY expression not produced by aggregation output "
- + "(missing from GROUP BY clause?): "
- + orderByElements_.get(i).getExpr().toSql());
- }
- }
- }
- if (havingPred_ != null) {
- if (!havingPred_.isBound(finalAggInfo.getOutputTupleId())) {
- throw new AnalysisException(
- "HAVING clause not produced by aggregation output "
- + "(missing from GROUP BY clause?): "
- + havingClause_.toSql());
- }
- }
- }
-
- /**
- * Create a map from COUNT([ALL]) -> zeroifnull(COUNT([ALL])) if
- * i) There is no GROUP-BY, and
- * ii) There are other distinct aggregates to be evaluated.
- * This transformation is necessary for COUNT to correctly return 0 for empty
- * input relations.
- */
- private ExprSubstitutionMap createCountAllMap(
- List<FunctionCallExpr> aggExprs, Analyzer analyzer)
- throws AnalysisException {
- ExprSubstitutionMap scalarCountAllMap = new ExprSubstitutionMap();
-
- if (groupingExprs_ != null && !groupingExprs_.isEmpty()) {
- // There are grouping expressions, so no substitution needs to be done.
- return scalarCountAllMap;
- }
-
- com.google.common.base.Predicate<FunctionCallExpr> isNotDistinctPred =
- new com.google.common.base.Predicate<FunctionCallExpr>() {
- public boolean apply(FunctionCallExpr expr) {
- return !expr.isDistinct();
- }
- };
- if (Iterables.all(aggExprs, isNotDistinctPred)) {
- // Only [ALL] aggs, so no substitution needs to be done.
- return scalarCountAllMap;
- }
-
- com.google.common.base.Predicate<FunctionCallExpr> isCountPred =
- new com.google.common.base.Predicate<FunctionCallExpr>() {
- public boolean apply(FunctionCallExpr expr) {
- return expr.getFnName().getFunction().equals("count");
- }
- };
-
- Iterable<FunctionCallExpr> countAllAggs =
- Iterables.filter(aggExprs, Predicates.and(isCountPred, isNotDistinctPred));
- for (FunctionCallExpr countAllAgg: countAllAggs) {
- // Replace COUNT(ALL) with zeroifnull(COUNT(ALL))
- ArrayList<Expr> zeroIfNullParam = Lists.newArrayList(countAllAgg.clone());
- FunctionCallExpr zeroIfNull =
- new FunctionCallExpr("zeroifnull", zeroIfNullParam);
- zeroIfNull.analyze(analyzer);
- scalarCountAllMap.put(countAllAgg, zeroIfNull);
- }
-
- return scalarCountAllMap;
- }
-
- /**
- * Create aggInfo for the given grouping and agg exprs.
- */
- private void createAggInfo(ArrayList<Expr> groupingExprs,
- ArrayList<FunctionCallExpr> aggExprs, Analyzer analyzer)
- throws AnalysisException {
- if (selectList_.isDistinct()) {
- // Create aggInfo for SELECT DISTINCT ... stmt:
- // - all select list items turn into grouping exprs
- // - there are no aggregate exprs
- Preconditions.checkState(groupingExprs.isEmpty());
- Preconditions.checkState(aggExprs.isEmpty());
- ArrayList<Expr> distinctGroupingExprs = Expr.cloneList(resultExprs_);
- aggInfo_ =
- AggregateInfo.create(distinctGroupingExprs, null, null, analyzer);
- } else {
- aggInfo_ = AggregateInfo.create(groupingExprs, aggExprs, null, analyzer);
- }
- }
-
- /**
- * If the select list contains AnalyticExprs, create AnalyticInfo and substitute
- * AnalyticExprs using the AnalyticInfo's smap.
- */
- private void createAnalyticInfo(Analyzer analyzer)
- throws AnalysisException {
- // collect AnalyticExprs from the SELECT and ORDER BY clauses
- ArrayList<Expr> analyticExprs = Lists.newArrayList();
- TreeNode.collect(resultExprs_, AnalyticExpr.class, analyticExprs);
- if (sortInfo_ != null) {
- TreeNode.collect(sortInfo_.getOrderingExprs(), AnalyticExpr.class,
- analyticExprs);
- }
- if (analyticExprs.isEmpty()) return;
- ExprSubstitutionMap rewriteSmap = new ExprSubstitutionMap();
- for (Expr expr: analyticExprs) {
- AnalyticExpr toRewrite = (AnalyticExpr)expr;
- Expr newExpr = AnalyticExpr.rewrite(toRewrite);
- if (newExpr != null) {
- newExpr.analyze(analyzer);
- if (!rewriteSmap.containsMappingFor(toRewrite)) {
- rewriteSmap.put(toRewrite, newExpr);
- }
- }
- }
- if (rewriteSmap.size() > 0) {
- // Substitute the exprs with their rewritten versions.
- ArrayList<Expr> updatedAnalyticExprs =
- Expr.substituteList(analyticExprs, rewriteSmap, analyzer, false);
- // This is to get rid the original exprs which have been rewritten.
- analyticExprs.clear();
- // Collect the new exprs introduced through the rewrite and the non-rewrite exprs.
- TreeNode.collect(updatedAnalyticExprs, AnalyticExpr.class, analyticExprs);
- }
-
- analyticInfo_ = AnalyticInfo.create(analyticExprs, analyzer);
-
- ExprSubstitutionMap smap = analyticInfo_.getSmap();
- // If 'exprRewritten' is true, we have to compose the new smap with the existing one.
- if (rewriteSmap.size() > 0) {
- smap = ExprSubstitutionMap.compose(
- rewriteSmap, analyticInfo_.getSmap(), analyzer);
- }
- // change select list and ordering exprs to point to analytic output. We need
- // to reanalyze the exprs at this point.
- resultExprs_ = Expr.substituteList(resultExprs_, smap, analyzer,
- false);
- LOG.trace("post-analytic selectListExprs: " + Expr.debugString(resultExprs_));
- if (sortInfo_ != null) {
- sortInfo_.substituteOrderingExprs(smap, analyzer);
- LOG.trace("post-analytic orderingExprs: " +
- Expr.debugString(sortInfo_.getOrderingExprs()));
- }
- }
-
- /**
- * Returns the SQL string corresponding to this SelectStmt.
- */
- @Override
- public String toSql() {
- // Return the SQL string before inline-view expression substitution.
- if (sqlString_ != null) return sqlString_;
-
- StringBuilder strBuilder = new StringBuilder();
- if (withClause_ != null) {
- strBuilder.append(withClause_.toSql());
- strBuilder.append(" ");
- }
-
- // Select list
- strBuilder.append("SELECT ");
- if (selectList_.isDistinct()) {
- strBuilder.append("DISTINCT ");
- }
- if (selectList_.hasPlanHints()) {
- strBuilder.append(ToSqlUtils.getPlanHintsSql(selectList_.getPlanHints()) + " ");
- }
- for (int i = 0; i < selectList_.getItems().size(); ++i) {
- strBuilder.append(selectList_.getItems().get(i).toSql());
- strBuilder.append((i+1 != selectList_.getItems().size()) ? ", " : "");
- }
- // From clause
- if (!fromClause_.isEmpty()) { strBuilder.append(fromClause_.toSql()); }
- // Where clause
- if (whereClause_ != null) {
- strBuilder.append(" WHERE ");
- strBuilder.append(whereClause_.toSql());
- }
- // Group By clause
- if (groupingExprs_ != null) {
- strBuilder.append(" GROUP BY ");
- for (int i = 0; i < groupingExprs_.size(); ++i) {
- strBuilder.append(groupingExprs_.get(i).toSql());
- strBuilder.append((i+1 != groupingExprs_.size()) ? ", " : "");
- }
- }
- // Having clause
- if (havingClause_ != null) {
- strBuilder.append(" HAVING ");
- strBuilder.append(havingClause_.toSql());
- }
- // Order By clause
- if (orderByElements_ != null) {
- strBuilder.append(" ORDER BY ");
- for (int i = 0; i < orderByElements_.size(); ++i) {
- strBuilder.append(orderByElements_.get(i).toSql());
- strBuilder.append((i+1 != orderByElements_.size()) ? ", " : "");
- }
- }
- // Limit clause.
- strBuilder.append(limitElement_.toSql());
- return strBuilder.toString();
- }
-
- /**
- * If the select statement has a sort/top that is evaluated, then the sort tuple
- * is materialized. Else, if there is aggregation then the aggregate tuple id is
- * materialized. Otherwise, all referenced tables are materialized as long as they are
- * not semi-joined. If there are analytics and no sort, then the returned tuple
- * ids also include the logical analytic output tuple.
- */
- @Override
- public void getMaterializedTupleIds(ArrayList<TupleId> tupleIdList) {
- if (evaluateOrderBy_) {
- tupleIdList.add(sortInfo_.getSortTupleDescriptor().getId());
- } else if (aggInfo_ != null) {
- // Return the tuple id produced in the final aggregation step.
- tupleIdList.add(aggInfo_.getResultTupleId());
- } else {
- for (TableRef tblRef: fromClause_) {
- // Don't include materialized tuple ids from semi-joined table
- // refs (see IMPALA-1526)
- if (tblRef.getJoinOp().isLeftSemiJoin()) continue;
- // Remove the materialized tuple ids of all the table refs that
- // are semi-joined by the right semi/anti join.
- if (tblRef.getJoinOp().isRightSemiJoin()) tupleIdList.clear();
- tupleIdList.addAll(tblRef.getMaterializedTupleIds());
- }
- }
- // We materialize the agg tuple or the table refs together with the analytic tuple.
- if (hasAnalyticInfo() && !evaluateOrderBy_) {
- tupleIdList.add(analyticInfo_.getOutputTupleId());
- }
- }
-
- /**
- * C'tor for cloning.
- */
- private SelectStmt(SelectStmt other) {
- super(other);
- selectList_ = other.selectList_.clone();
- fromClause_ = other.fromClause_.clone();
- whereClause_ = (other.whereClause_ != null) ? other.whereClause_.clone() : null;
- groupingExprs_ =
- (other.groupingExprs_ != null) ? Expr.cloneList(other.groupingExprs_) : null;
- havingClause_ = (other.havingClause_ != null) ? other.havingClause_.clone() : null;
- colLabels_ = Lists.newArrayList(other.colLabels_);
- aggInfo_ = (other.aggInfo_ != null) ? other.aggInfo_.clone() : null;
- analyticInfo_ = (other.analyticInfo_ != null) ? other.analyticInfo_.clone() : null;
- sqlString_ = (other.sqlString_ != null) ? new String(other.sqlString_) : null;
- baseTblSmap_ = other.baseTblSmap_.clone();
- }
-
- @Override
- public void collectTableRefs(List<TableRef> tblRefs) {
- for (TableRef tblRef: fromClause_) {
- if (tblRef instanceof InlineViewRef) {
- InlineViewRef inlineViewRef = (InlineViewRef) tblRef;
- inlineViewRef.getViewStmt().collectTableRefs(tblRefs);
- } else {
- tblRefs.add(tblRef);
- }
- }
- }
-
- @Override
- public void reset() {
- super.reset();
- selectList_.reset();
- colLabels_.clear();
- fromClause_.reset();
- baseTblSmap_.clear();
- if (whereClause_ != null) whereClause_.reset();
- if (groupingExprs_ != null) Expr.resetList(groupingExprs_);
- if (havingClause_ != null) havingClause_.reset();
- }
-
- @Override
- public SelectStmt clone() { return new SelectStmt(this); }
-
- /**
- * Check if the stmt returns a single row. This can happen
- * in the following cases:
- * 1. select stmt with a 'limit 1' clause
- * 2. select stmt with an aggregate function and no group by.
- * 3. select stmt with no from clause.
- *
- * This function may produce false negatives because the cardinality of the
- * result set also depends on the data a stmt is processing.
- */
- public boolean returnsSingleRow() {
- // limit 1 clause
- if (limitElement_ != null && limitElement_.getLimit() == 1) return true;
- // No from clause (base tables or inline views)
- if (fromClause_.isEmpty()) return true;
- // Aggregation with no group by and no DISTINCT
- if (hasAggInfo() && !hasGroupByClause() && !selectList_.isDistinct()) return true;
- // In all other cases, return false.
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/SetStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/SetStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/SetStmt.java
deleted file mode 100644
index d44f5b4..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/SetStmt.java
+++ /dev/null
@@ -1,69 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import com.cloudera.impala.thrift.TSetQueryOptionRequest;
-import com.google.common.base.Preconditions;
-
-/**
- * Representation of a SET query options statement.
- */
-public class SetStmt extends StatementBase {
- private final String key_;
- private final String value_;
-
- // This key is deprecated in CDH5.2; COMPRESSION_CODEC_KEY replaces this
- private static final String DEPRECATED_PARQUET_CODEC_KEY = "PARQUET_COMPRESSION_CODEC";
- private static final String COMPRESSION_CODEC_KEY = "COMPRESSION_CODEC";
-
- // maps the given key name to a key defined in the thrift file
- private static String resolveThriftKey(String key) {
- if (key.toLowerCase().equals(DEPRECATED_PARQUET_CODEC_KEY.toLowerCase())) {
- return COMPRESSION_CODEC_KEY;
- }
- return key;
- }
-
- public SetStmt(String key, String value) {
- Preconditions.checkArgument((key == null) == (value == null));
- Preconditions.checkArgument(key == null || !key.isEmpty());
- key_ = key;
- value_ = value;
- }
-
- @Override
- public String toSql() {
- if (key_ == null) return "SET";
- Preconditions.checkNotNull(value_);
- return "SET " + ToSqlUtils.getIdentSql(key_) + "='" + value_ + "'";
- }
-
- @Override
- public void analyze(Analyzer analyzer) {
- // Query option key is validated by the backend.
- }
-
- public TSetQueryOptionRequest toThrift() {
- TSetQueryOptionRequest request = new TSetQueryOptionRequest();
- if (key_ != null) {
- request.setKey(resolveThriftKey(key_));
- request.setValue(value_);
- }
- return request;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateFunctionStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateFunctionStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateFunctionStmt.java
deleted file mode 100644
index e3ac551..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateFunctionStmt.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.List;
-
-import com.cloudera.impala.analysis.FunctionName;
-import com.cloudera.impala.authorization.Privilege;
-import com.cloudera.impala.catalog.Db;
-import com.cloudera.impala.catalog.Function;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.thrift.TGetFunctionsParams;
-import com.cloudera.impala.thrift.TFunctionCategory;
-import com.google.common.base.Preconditions;
-
-/**
- * Representation of a SHOW CREATE FUNCTION statement which returns the "CREATE FUNCTION ..."
- * string that re-creates the function.
- *
- * Syntax: SHOW CREATE [AGGREGATE] FUNCTION [<db_name>.]<function_name>
- */
-public class ShowCreateFunctionStmt extends StatementBase {
- private final FunctionName functionName_;
- private final TFunctionCategory category_;
-
- public ShowCreateFunctionStmt(FunctionName functionName, TFunctionCategory category) {
- Preconditions.checkNotNull(functionName);
- Preconditions.checkArgument(category == TFunctionCategory.SCALAR ||
- category == TFunctionCategory.AGGREGATE);
- functionName_ = functionName;
- category_ = category;
- }
-
- @Override
- public String toSql() {
- return "SHOW CREATE " +
- (category_ == TFunctionCategory.AGGREGATE ? "AGGREGATE " : "") +
- "FUNCTION " + functionName_;
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- functionName_.analyze(analyzer);
- Db db = analyzer.getDb(functionName_.getDb(), Privilege.VIEW_METADATA);
- List<Function> functions = db.getFunctions(category_, functionName_.getFunction());
- if (functions.isEmpty()) {
- throw new AnalysisException("Function " + functionName_.getFunction() + "() " +
- "does not exist in database " + functionName_.getDb());
- }
- }
-
- public TGetFunctionsParams toThrift() {
- TGetFunctionsParams params = new TGetFunctionsParams();
- params.setCategory(category_);
- params.setDb(functionName_.getDb());
- params.setPattern(functionName_.getFunction());
- return params;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateTableStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateTableStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateTableStmt.java
deleted file mode 100644
index 68b593c..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/ShowCreateTableStmt.java
+++ /dev/null
@@ -1,80 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import com.cloudera.impala.authorization.Privilege;
-import com.cloudera.impala.catalog.Table;
-import com.cloudera.impala.catalog.View;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.thrift.TCatalogObjectType;
-import com.cloudera.impala.thrift.TTableName;
-import com.google.common.base.Preconditions;
-
-/**
- * Representation of a SHOW CREATE TABLE statement which returns either the
- * "CREATE TABLE ..." string that re-creates the table or the "CREATE VIEW ..."
- * string that re-creates the view as appropriate.
- *
- * Syntax: SHOW CREATE (TABLE|VIEW) <table or view>
- */
-public class ShowCreateTableStmt extends StatementBase {
- private TableName tableName_;
-
- // The object type keyword used, e.g. TABLE or VIEW, needed to output matching SQL.
- private TCatalogObjectType objectType_;
-
- public ShowCreateTableStmt(TableName table, TCatalogObjectType objectType) {
- Preconditions.checkNotNull(table);
- this.tableName_ = table;
- this.objectType_ = objectType;
- }
-
- @Override
- public String toSql() {
- return "SHOW CREATE " + objectType_.name() + " " + tableName_;
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- tableName_ = analyzer.getFqTableName(tableName_);
- Table table = analyzer.getTable(tableName_, Privilege.VIEW_METADATA);
- if (table instanceof View) {
- View view = (View)table;
- // Analyze the view query statement with its own analyzer for authorization.
- Analyzer viewAnalyzer = new Analyzer(analyzer);
- // Only show the view's query if the user has permissions to execute the query, to
- // avoid revealing information, e.g. tables the user does not have access to.
- // Report a masked authorization message if authorization fails.
- viewAnalyzer.setAuthErrMsg(String.format("User '%s' does not have privileges to " +
- "see the definition of view '%s'.",
- analyzer.getUser().getName(), view.getFullName()));
- QueryStmt viewQuery = view.getQueryStmt().clone();
- // Views from the Hive metastore may rely on Hive's column naming if the SQL
- // statement references a column by its implicitly defined column names.
- viewAnalyzer.setUseHiveColLabels(true);
- viewQuery.analyze(viewAnalyzer);
- }
- }
-
- public TTableName toThrift() {
- TTableName params = new TTableName();
- params.setTable_name(tableName_.getTbl());
- params.setDb_name(tableName_.getDb());
- return params;
- }
-}