You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by kw...@apache.org on 2016/09/30 02:14:58 UTC
[41/61] [partial] incubator-impala git commit: IMPALA-3786: Replace
"cloudera" with "apache" (part 1)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java
deleted file mode 100644
index d53aa9e..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/FunctionCallExpr.java
+++ /dev/null
@@ -1,516 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.List;
-
-import com.cloudera.impala.authorization.Privilege;
-import com.cloudera.impala.catalog.AggregateFunction;
-import com.cloudera.impala.catalog.Catalog;
-import com.cloudera.impala.catalog.Db;
-import com.cloudera.impala.catalog.Function;
-import com.cloudera.impala.catalog.ScalarFunction;
-import com.cloudera.impala.catalog.ScalarType;
-import com.cloudera.impala.catalog.Type;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.common.TreeNode;
-import com.cloudera.impala.thrift.TAggregateExpr;
-import com.cloudera.impala.thrift.TExprNode;
-import com.cloudera.impala.thrift.TExprNodeType;
-import com.cloudera.impala.thrift.TFunctionBinaryType;
-import com.google.common.base.Joiner;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class FunctionCallExpr extends Expr {
- private final FunctionName fnName_;
- private final FunctionParams params_;
- private boolean isAnalyticFnCall_ = false;
- private boolean isInternalFnCall_ = false;
-
- // Indicates whether this is a merge aggregation function that should use the merge
- // instead of the update symbol. This flag also affects the behavior of
- // resetAnalysisState() which is used during expr substitution.
- private final boolean isMergeAggFn_;
-
- // Printed in toSqlImpl(), if set. Used for merge agg fns.
- private String label_;
-
- public FunctionCallExpr(String functionName, List<Expr> params) {
- this(new FunctionName(functionName), new FunctionParams(false, params));
- }
-
- public FunctionCallExpr(FunctionName fnName, List<Expr> params) {
- this(fnName, new FunctionParams(false, params));
- }
-
- public FunctionCallExpr(FunctionName fnName, FunctionParams params) {
- this(fnName, params, false);
- }
-
- private FunctionCallExpr(
- FunctionName fnName, FunctionParams params, boolean isMergeAggFn) {
- super();
- fnName_ = fnName;
- params_ = params;
- isMergeAggFn_ = isMergeAggFn;
- if (params.exprs() != null) children_ = Lists.newArrayList(params_.exprs());
- }
-
- /**
- * Returns an Expr that evaluates the function call <fnName>(<params>). The returned
- * Expr is not necessarily a FunctionCallExpr (example: DECODE())
- */
- public static Expr createExpr(FunctionName fnName, FunctionParams params) {
- FunctionCallExpr functionCallExpr = new FunctionCallExpr(fnName, params);
- if (fnName.getFnNamePath().size() == 1
- && fnName.getFnNamePath().get(0).equalsIgnoreCase("decode")
- || fnName.getFnNamePath().size() == 2
- && fnName.getFnNamePath().get(0).equalsIgnoreCase(Catalog.BUILTINS_DB)
- && fnName.getFnNamePath().get(1).equalsIgnoreCase("decode")) {
- return new CaseExpr(functionCallExpr);
- }
- return functionCallExpr;
- }
-
- /**
- * Returns a new function call expr on the given params for performing the merge()
- * step of the given aggregate function.
- */
- public static FunctionCallExpr createMergeAggCall(
- FunctionCallExpr agg, List<Expr> params) {
- Preconditions.checkState(agg.isAnalyzed_);
- Preconditions.checkState(agg.isAggregateFunction());
- FunctionCallExpr result = new FunctionCallExpr(
- agg.fnName_, new FunctionParams(false, params), true);
- // Inherit the function object from 'agg'.
- result.fn_ = agg.fn_;
- result.type_ = agg.type_;
- // Set an explicit label based on the input agg.
- if (agg.isMergeAggFn_) {
- result.label_ = agg.label_;
- } else {
- // fn(input) becomes fn:merge(input).
- result.label_ = agg.toSql().replaceFirst(agg.fnName_.toString(),
- agg.fnName_.toString() + ":merge");
- }
- Preconditions.checkState(!result.type_.isWildcardDecimal());
- return result;
- }
-
- /**
- * Copy c'tor used in clone().
- */
- protected FunctionCallExpr(FunctionCallExpr other) {
- super(other);
- fnName_ = other.fnName_;
- isAnalyticFnCall_ = other.isAnalyticFnCall_;
- isInternalFnCall_ = other.isInternalFnCall_;
- isMergeAggFn_ = other.isMergeAggFn_;
- // Clone the params in a way that keeps the children_ and the params.exprs()
- // in sync. The children have already been cloned in the super c'tor.
- if (other.params_.isStar()) {
- Preconditions.checkState(children_.isEmpty());
- params_ = FunctionParams.createStarParam();
- } else {
- params_ = new FunctionParams(other.params_.isDistinct(), children_);
- }
- label_ = other.label_;
- }
-
- public boolean isMergeAggFn() { return isMergeAggFn_; }
-
- @Override
- public void resetAnalysisState() {
- isAnalyzed_ = false;
- // Resolving merge agg functions after substitution may fail e.g., if the
- // intermediate agg type is not the same as the output type. Preserve the original
- // fn_ such that analyze() hits the special-case code for merge agg fns that
- // handles this case.
- if (!isMergeAggFn_) fn_ = null;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!super.equals(obj)) return false;
- FunctionCallExpr o = (FunctionCallExpr)obj;
- return fnName_.equals(o.fnName_) &&
- params_.isDistinct() == o.params_.isDistinct() &&
- params_.isStar() == o.params_.isStar();
- }
-
- @Override
- public String toSqlImpl() {
- if (label_ != null) return label_;
- // Merge agg fns should have an explicit label.
- Preconditions.checkState(!isMergeAggFn_);
- StringBuilder sb = new StringBuilder();
- sb.append(fnName_).append("(");
- if (params_.isStar()) sb.append("*");
- if (params_.isDistinct()) sb.append("DISTINCT ");
- sb.append(Joiner.on(", ").join(childrenToSql())).append(")");
- return sb.toString();
- }
-
- @Override
- public String debugString() {
- return Objects.toStringHelper(this)
- .add("name", fnName_)
- .add("isStar", params_.isStar())
- .add("isDistinct", params_.isDistinct())
- .addValue(super.debugString())
- .toString();
- }
-
- public FunctionParams getParams() { return params_; }
- public boolean isScalarFunction() {
- Preconditions.checkNotNull(fn_);
- return fn_ instanceof ScalarFunction ;
- }
-
- public Type getReturnType() {
- Preconditions.checkNotNull(fn_);
- return fn_.getReturnType();
- }
-
- /**
- * Returns true if this is a call to a non-analytic aggregate function.
- */
- public boolean isAggregateFunction() {
- Preconditions.checkNotNull(fn_);
- return fn_ instanceof AggregateFunction && !isAnalyticFnCall_;
- }
-
- /**
- * Returns true if this is a call to an aggregate function that returns
- * non-null on an empty input (e.g. count).
- */
- public boolean returnsNonNullOnEmpty() {
- Preconditions.checkNotNull(fn_);
- return fn_ instanceof AggregateFunction &&
- ((AggregateFunction)fn_).returnsNonNullOnEmpty();
- }
-
- public boolean isDistinct() {
- Preconditions.checkState(isAggregateFunction());
- return params_.isDistinct();
- }
-
- public boolean ignoresDistinct() {
- Preconditions.checkState(isAggregateFunction());
- return ((AggregateFunction)fn_).ignoresDistinct();
- }
-
- public FunctionName getFnName() { return fnName_; }
- public void setIsAnalyticFnCall(boolean v) { isAnalyticFnCall_ = v; }
- public void setIsInternalFnCall(boolean v) { isInternalFnCall_ = v; }
-
- @Override
- protected void toThrift(TExprNode msg) {
- if (isAggregateFunction() || isAnalyticFnCall_) {
- msg.node_type = TExprNodeType.AGGREGATE_EXPR;
- if (!isAnalyticFnCall_) msg.setAgg_expr(new TAggregateExpr(isMergeAggFn_));
- } else {
- msg.node_type = TExprNodeType.FUNCTION_CALL;
- }
- }
-
- /**
- * Aggregate functions are never constant.
- */
- @Override
- public boolean isConstant() {
- if (fn_ != null && fn_ instanceof AggregateFunction) return false;
- return super.isConstant();
- }
-
- // Provide better error message for some aggregate builtins. These can be
- // a bit more user friendly than a generic function not found.
- // TODO: should we bother to do this? We could also improve the general
- // error messages. For example, listing the alternatives.
- protected String getFunctionNotFoundError(Type[] argTypes) {
- if (fnName_.isBuiltin()) {
- // Some custom error message for builtins
- if (params_.isStar()) {
- return "'*' can only be used in conjunction with COUNT";
- }
- if (fnName_.getFunction().equalsIgnoreCase("count")) {
- if (!params_.isDistinct() && argTypes.length > 1) {
- return "COUNT must have DISTINCT for multiple arguments: " + toSql();
- }
- }
- if (fnName_.getFunction().equalsIgnoreCase("sum")) {
- return "SUM requires a numeric parameter: " + toSql();
- }
- if (fnName_.getFunction().equalsIgnoreCase("avg")) {
- return "AVG requires a numeric or timestamp parameter: " + toSql();
- }
- }
-
- String[] argTypesSql = new String[argTypes.length];
- for (int i = 0; i < argTypes.length; ++i) {
- argTypesSql[i] = argTypes[i].toSql();
- }
- return String.format(
- "No matching function with signature: %s(%s).",
- fnName_, params_.isStar() ? "*" : Joiner.on(", ").join(argTypesSql));
- }
-
- /**
- * Builtins that return decimals are specified as the wildcard decimal(decimal(*,*))
- * and the specific decimal can only be determined based on the inputs. We currently
- * don't have a mechanism to specify this with the UDF interface. Until we add
- * that (i.e. allowing UDFs to participate in the planning phase), we will
- * manually resolve the wildcard types for the few functions that need it.
- * This can only be called for functions that return wildcard decimals and the first
- * argument is a wildcard decimal.
- * TODO: this prevents UDFs from using wildcard decimals and is in general not scalable.
- * We should add a prepare_fn() to UDFs for doing this.
- */
- private Type resolveDecimalReturnType(Analyzer analyzer) throws AnalysisException {
- Preconditions.checkState(type_.isWildcardDecimal());
- Preconditions.checkState(fn_.getBinaryType() == TFunctionBinaryType.BUILTIN);
- Preconditions.checkState(children_.size() > 0);
-
- // Find first decimal input (some functions, such as if(), begin with non-decimal
- // arguments).
- ScalarType childType = null;
- for (Expr child : children_) {
- if (child.type_.isDecimal()) {
- childType = (ScalarType) child.type_;
- break;
- }
- }
- Preconditions.checkState(childType != null && !childType.isWildcardDecimal());
- Type returnType = childType;
-
- if (fnName_.getFunction().equalsIgnoreCase("sum")) {
- return childType.getMaxResolutionType();
- }
-
- int digitsBefore = childType.decimalPrecision() - childType.decimalScale();
- int digitsAfter = childType.decimalScale();
- if (fnName_.getFunction().equalsIgnoreCase("ceil") ||
- fnName_.getFunction().equalsIgnoreCase("ceiling") ||
- fnName_.getFunction().equals("floor") ||
- fnName_.getFunction().equals("dfloor")) {
- // These functions just return with scale 0 but can trigger rounding. We need
- // to increase the precision by 1 to handle that.
- ++digitsBefore;
- digitsAfter = 0;
- } else if (fnName_.getFunction().equalsIgnoreCase("truncate") ||
- fnName_.getFunction().equalsIgnoreCase("dtrunc") ||
- fnName_.getFunction().equalsIgnoreCase("round") ||
- fnName_.getFunction().equalsIgnoreCase("dround")) {
- if (children_.size() > 1) {
- // The second argument to these functions is the desired scale, otherwise
- // the default is 0.
- Preconditions.checkState(children_.size() == 2);
- if (children_.get(1).isNullLiteral()) {
- throw new AnalysisException(fnName_.getFunction() +
- "() cannot be called with a NULL second argument.");
- }
-
- if (!children_.get(1).isConstant()) {
- // We don't allow calling truncate or round with a non-constant second
- // (desired scale) argument. e.g. select round(col1, col2). This would
- // mean we don't know the scale of the resulting type and would need some
- // kind of dynamic type handling which is not yet possible. This seems like
- // a reasonable restriction.
- throw new AnalysisException(fnName_.getFunction() +
- "() must be called with a constant second argument.");
- }
- NumericLiteral scaleLiteral = (NumericLiteral) LiteralExpr.create(
- children_.get(1), analyzer.getQueryCtx());
- digitsAfter = (int)scaleLiteral.getLongValue();
- if (Math.abs(digitsAfter) > ScalarType.MAX_SCALE) {
- throw new AnalysisException("Cannot round/truncate to scales greater than " +
- ScalarType.MAX_SCALE + ".");
- }
- // Round/Truncate to a negative scale means to round to the digit before
- // the decimal e.g. round(1234.56, -2) would be 1200.
- // The resulting scale is always 0.
- digitsAfter = Math.max(digitsAfter, 0);
- } else {
- // Round()/Truncate() with no second argument.
- digitsAfter = 0;
- }
-
- if ((fnName_.getFunction().equalsIgnoreCase("round") ||
- fnName_.getFunction().equalsIgnoreCase("dround")) &&
- digitsAfter < childType.decimalScale()) {
- // If we are rounding to fewer decimal places, it's possible we need another
- // digit before the decimal.
- ++digitsBefore;
- }
- }
- Preconditions.checkState(returnType.isDecimal() && !returnType.isWildcardDecimal());
- return ScalarType.createDecimalTypeInternal(digitsBefore + digitsAfter, digitsAfter);
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (isAnalyzed_) return;
- super.analyze(analyzer);
- fnName_.analyze(analyzer);
-
- if (isMergeAggFn_) {
- // This is the function call expr after splitting up to a merge aggregation.
- // The function has already been analyzed so just do the minimal sanity
- // check here.
- AggregateFunction aggFn = (AggregateFunction)fn_;
- Preconditions.checkNotNull(aggFn);
- Type intermediateType = aggFn.getIntermediateType();
- if (intermediateType == null) intermediateType = type_;
- Preconditions.checkState(!type_.isWildcardDecimal());
- return;
- }
-
- Type[] argTypes = collectChildReturnTypes();
-
- // User needs DB access.
- Db db = analyzer.getDb(fnName_.getDb(), Privilege.VIEW_METADATA, true);
- if (!db.containsFunction(fnName_.getFunction())) {
- throw new AnalysisException(fnName_ + "() unknown");
- }
-
- if (fnName_.getFunction().equals("count") && params_.isDistinct()) {
- // Treat COUNT(DISTINCT ...) special because of how we do the rewrite.
- // There is no version of COUNT() that takes more than 1 argument but after
- // the rewrite, we only need count(*).
- // TODO: fix how we rewrite count distinct.
- argTypes = new Type[0];
- Function searchDesc = new Function(fnName_, argTypes, Type.INVALID, false);
- fn_ = db.getFunction(searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
- type_ = fn_.getReturnType();
- // Make sure BE doesn't see any TYPE_NULL exprs
- for (int i = 0; i < children_.size(); ++i) {
- if (getChild(i).getType().isNull()) {
- uncheckedCastChild(ScalarType.BOOLEAN, i);
- }
- }
- return;
- }
-
- // TODO: We allow implicit cast from string->timestamp but only
- // support avg(timestamp). This means avg(string_col) would work
- // from our casting rules. This is not right.
- // We need to revisit where implicit casts are allowed for string
- // to timestamp
- if (fnName_.getFunction().equalsIgnoreCase("avg") &&
- children_.size() == 1 && children_.get(0).getType().isStringType()) {
- throw new AnalysisException(
- "AVG requires a numeric or timestamp parameter: " + toSql());
- }
-
- Function searchDesc = new Function(fnName_, argTypes, Type.INVALID, false);
- fn_ = db.getFunction(searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
- if (fn_ == null || (!isInternalFnCall_ && !fn_.userVisible())) {
- throw new AnalysisException(getFunctionNotFoundError(argTypes));
- }
-
- if (isAggregateFunction()) {
- // subexprs must not contain aggregates
- if (TreeNode.contains(children_, Expr.isAggregatePredicate())) {
- throw new AnalysisException(
- "aggregate function must not contain aggregate parameters: " + this.toSql());
- }
-
- // .. or analytic exprs
- if (Expr.contains(children_, AnalyticExpr.class)) {
- throw new AnalysisException(
- "aggregate function must not contain analytic parameters: " + this.toSql());
- }
-
- // The catalog contains count() with no arguments to handle count(*) but don't
- // accept count().
- // TODO: can this be handled more cleanly. It does seem like a special case since
- // no other aggregate functions (currently) can accept '*'.
- if (fnName_.getFunction().equalsIgnoreCase("count") &&
- !params_.isStar() && children_.size() == 0) {
- throw new AnalysisException("count() is not allowed.");
- }
-
- // TODO: the distinct rewrite does not handle this but why?
- if (params_.isDistinct()) {
- // The second argument in group_concat(distinct) must be a constant expr that
- // returns a string.
- if (fnName_.getFunction().equalsIgnoreCase("group_concat")
- && getChildren().size() == 2
- && !getChild(1).isConstant()) {
- throw new AnalysisException("Second parameter in GROUP_CONCAT(DISTINCT)" +
- " must be a constant expression that returns a string.");
- }
- if (fn_.getBinaryType() != TFunctionBinaryType.BUILTIN) {
- throw new AnalysisException("User defined aggregates do not support DISTINCT.");
- }
- }
-
- AggregateFunction aggFn = (AggregateFunction)fn_;
- if (aggFn.ignoresDistinct()) params_.setIsDistinct(false);
- }
-
- if (params_.isIgnoreNulls() && !isAnalyticFnCall_) {
- throw new AnalysisException("Function " + fnName_.getFunction().toUpperCase()
- + " does not accept the keyword IGNORE NULLS.");
- }
-
- if (isScalarFunction()) validateScalarFnParams(params_);
- if (fn_ instanceof AggregateFunction
- && ((AggregateFunction) fn_).isAnalyticFn()
- && !((AggregateFunction) fn_).isAggregateFn()
- && !isAnalyticFnCall_) {
- throw new AnalysisException(
- "Analytic function requires an OVER clause: " + toSql());
- }
-
- castForFunctionCall(false);
- type_ = fn_.getReturnType();
- if (type_.isDecimal() && type_.isWildcardDecimal()) {
- type_ = resolveDecimalReturnType(analyzer);
- }
-
- // We do not allow any function to return a type CHAR or VARCHAR
- // TODO add support for CHAR(N) and VARCHAR(N) return values in post 2.0,
- // support for this was not added to the backend in 2.0
- if (type_.isWildcardChar() || type_.isWildcardVarchar()) {
- type_ = ScalarType.STRING;
- }
-
- // TODO(tmarshall): Differentiate based on the specific function.
- if (hasChildCosts()) evalCost_ = getChildCosts() + FUNCTION_CALL_COST;
- }
-
- /**
- * Checks that no special aggregate params are included in 'params' that would be
- * invalid for a scalar function. Analysis of the param exprs is not done.
- */
- static void validateScalarFnParams(FunctionParams params)
- throws AnalysisException {
- if (params.isStar()) {
- throw new AnalysisException("Cannot pass '*' to scalar function.");
- }
- if (params.isDistinct()) {
- throw new AnalysisException("Cannot pass 'DISTINCT' to scalar function.");
- }
- }
-
- @Override
- public Expr clone() { return new FunctionCallExpr(this); }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java b/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java
deleted file mode 100644
index 5609578..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/FunctionName.java
+++ /dev/null
@@ -1,148 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.ArrayList;
-
-import com.cloudera.impala.catalog.Catalog;
-import com.cloudera.impala.catalog.Db;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.thrift.TFunctionName;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-
-/**
- * Class to represent a function name. Function names are specified as
- * db.function_name.
- */
-public class FunctionName {
- // Only set for parsed function names.
- private final ArrayList<String> fnNamePath_;
-
- // Set/validated during analysis.
- private String db_;
- private String fn_;
- private boolean isBuiltin_ = false;
- private boolean isAnalyzed_ = false;
-
- /**
- * C'tor for parsed function names. The function names could be invalid. The validity
- * is checked during analysis.
- */
- public FunctionName(ArrayList<String> fnNamePath) {
- fnNamePath_ = fnNamePath;
- }
-
- public FunctionName(String dbName, String fn) {
- db_ = (dbName != null) ? dbName.toLowerCase() : null;
- fn_ = fn.toLowerCase();
- fnNamePath_ = null;
- }
-
- public FunctionName(String fn) {
- this(null, fn);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (!(obj instanceof FunctionName)) return false;
- FunctionName o = (FunctionName)obj;
- if ((db_ == null || o.db_ == null) && (db_ != o.db_)) {
- if (db_ == null && o.db_ != null) return false;
- if (db_ != null && o.db_ == null) return false;
- if (!db_.equalsIgnoreCase(o.db_)) return false;
- }
- return fn_.equalsIgnoreCase(o.fn_);
- }
-
- public String getDb() { return db_; }
- public String getFunction() { return fn_; }
- public boolean isFullyQualified() { return db_ != null; }
- public boolean isBuiltin() { return isBuiltin_; }
- public ArrayList<String> getFnNamePath() { return fnNamePath_; }
-
- @Override
- public String toString() {
- // The fnNamePath_ is not always set.
- if (!isAnalyzed_ && fnNamePath_ != null) return Joiner.on(".").join(fnNamePath_);
- if (db_ == null || isBuiltin_) return fn_;
- return db_ + "." + fn_;
- }
-
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (isAnalyzed_) return;
- analyzeFnNamePath();
- if (fn_.isEmpty()) throw new AnalysisException("Function name cannot be empty.");
- for (int i = 0; i < fn_.length(); ++i) {
- if (!isValidCharacter(fn_.charAt(i))) {
- throw new AnalysisException(
- "Function names must be all alphanumeric or underscore. " +
- "Invalid name: " + fn_);
- }
- }
- if (Character.isDigit(fn_.charAt(0))) {
- throw new AnalysisException("Function cannot start with a digit: " + fn_);
- }
-
- // Resolve the database for this function.
- if (!isFullyQualified()) {
- Db builtinDb = analyzer.getCatalog().getBuiltinsDb();
- if (builtinDb.containsFunction(fn_)) {
- // If it isn't fully qualified and is the same name as a builtin, use
- // the builtin.
- db_ = Catalog.BUILTINS_DB;
- isBuiltin_ = true;
- } else {
- db_ = analyzer.getDefaultDb();
- isBuiltin_ = false;
- }
- } else {
- isBuiltin_ = db_.equals(Catalog.BUILTINS_DB);
- }
- isAnalyzed_ = true;
- }
-
- private void analyzeFnNamePath() throws AnalysisException {
- if (fnNamePath_ == null) return;
- if (fnNamePath_.size() > 2 || fnNamePath_.isEmpty()) {
- throw new AnalysisException(
- String.format("Invalid function name: '%s'. Expected [dbname].funcname.",
- Joiner.on(".").join(fnNamePath_)));
- } else if (fnNamePath_.size() > 1) {
- db_ = fnNamePath_.get(0);
- fn_ = fnNamePath_.get(1).toLowerCase();
- } else {
- Preconditions.checkState(fnNamePath_.size() == 1);
- fn_ = fnNamePath_.get(0).toLowerCase();
- }
- }
-
- private boolean isValidCharacter(char c) {
- return Character.isLetterOrDigit(c) || c == '_';
- }
-
- public TFunctionName toThrift() {
- TFunctionName name = new TFunctionName(fn_);
- name.setDb_name(db_);
- return name;
- }
-
- public static FunctionName fromThrift(TFunctionName fnName) {
- return new FunctionName(fnName.getDb_name(), fnName.getFunction_name());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java b/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java
deleted file mode 100644
index e6854a0..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/FunctionParams.java
+++ /dev/null
@@ -1,68 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.List;
-
-/**
- * Return value of the grammar production that parses function
- * parameters. These parameters can be for scalar or aggregate functions.
- */
-class FunctionParams implements Cloneable {
- private final boolean isStar_;
- private boolean isDistinct_;
- private boolean isIgnoreNulls_;
- private final List<Expr> exprs_;
-
- // c'tor for non-star params
- public FunctionParams(boolean isDistinct, boolean isIgnoreNulls, List<Expr> exprs) {
- this.isStar_ = false;
- this.isDistinct_ = isDistinct;
- this.isIgnoreNulls_ = isIgnoreNulls;
- this.exprs_ = exprs;
- }
-
- // c'tor for non-star, non-ignore-nulls params
- public FunctionParams(boolean isDistinct, List<Expr> exprs) {
- this(isDistinct, false, exprs);
- }
-
- // c'tor for non-star, non-distinct, non-ignore-nulls params
- public FunctionParams(List<Expr> exprs) {
- this(false, false, exprs);
- }
-
- static public FunctionParams createStarParam() {
- return new FunctionParams();
- }
-
- public boolean isStar() { return isStar_; }
- public boolean isDistinct() { return isDistinct_; }
- public boolean isIgnoreNulls() { return isIgnoreNulls_; }
- public List<Expr> exprs() { return exprs_; }
- public void setIsDistinct(boolean v) { isDistinct_ = v; }
- public int size() { return exprs_ == null ? 0 : exprs_.size(); }
-
- // c'tor for <agg>(*)
- private FunctionParams() {
- exprs_ = null;
- isStar_ = true;
- isDistinct_ = false;
- isIgnoreNulls_ = false;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java
deleted file mode 100644
index edaf22a..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokePrivStmt.java
+++ /dev/null
@@ -1,94 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.List;
-
-import com.cloudera.impala.catalog.Role;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.thrift.TGrantRevokePrivParams;
-import com.cloudera.impala.thrift.TPrivilege;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-
-/**
- * Represents a "GRANT/REVOKE PRIVILEGE" statement.
- * All privilege checks on catalog objects are skipped when executing
- * GRANT/REVOKE statements. This is because we need to be able to create
- * privileges on an object before any privileges actually exist.
- * The GRANT/REVOKE statement itself will be authorized (currently by
- * the Sentry Service).
- */
-public class GrantRevokePrivStmt extends AuthorizationStmt {
- private final PrivilegeSpec privilegeSpec_;
- private final String roleName_;
- private final boolean isGrantPrivStmt_;
- private final boolean hasGrantOpt_;
-
- // Set/modified during analysis
- private Role role_;
-
- public GrantRevokePrivStmt(String roleName, PrivilegeSpec privilegeSpec,
- boolean isGrantPrivStmt, boolean hasGrantOpt) {
- Preconditions.checkNotNull(privilegeSpec);
- Preconditions.checkNotNull(roleName);
- privilegeSpec_ = privilegeSpec;
- roleName_ = roleName;
- isGrantPrivStmt_ = isGrantPrivStmt;
- hasGrantOpt_ = hasGrantOpt;
- }
-
- public TGrantRevokePrivParams toThrift() {
- TGrantRevokePrivParams params = new TGrantRevokePrivParams();
- params.setRole_name(roleName_);
- params.setIs_grant(isGrantPrivStmt_);
- List<TPrivilege> privileges = privilegeSpec_.toThrift();
- for (TPrivilege privilege: privileges) {
- privilege.setRole_id(role_.getId());
- privilege.setHas_grant_opt(hasGrantOpt_);
- }
- params.setHas_grant_opt(hasGrantOpt_);
- params.setPrivileges(privileges);
- return params;
- }
-
- @Override
- public String toSql() {
- StringBuilder sb = new StringBuilder(isGrantPrivStmt_ ? "GRANT " : "REVOKE ");
- if (!isGrantPrivStmt_ && hasGrantOpt_) sb.append("GRANT OPTION FOR ");
- sb.append(privilegeSpec_.toSql());
- sb.append(isGrantPrivStmt_ ? " TO " : " FROM ");
- sb.append(roleName_);
- if (isGrantPrivStmt_ && hasGrantOpt_) sb.append(" WITH GRANT OPTION");
- return sb.toString();
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- super.analyze(analyzer);
- if (Strings.isNullOrEmpty(roleName_)) {
- throw new AnalysisException("Role name in GRANT/REVOKE privilege cannot be " +
- "empty.");
- }
- role_ = analyzer.getCatalog().getAuthPolicy().getRole(roleName_);
- if (role_ == null) {
- throw new AnalysisException(String.format("Role '%s' does not exist.", roleName_));
- }
- privilegeSpec_.analyze(analyzer);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java b/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java
deleted file mode 100644
index 73240dc..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/GrantRevokeRoleStmt.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.thrift.TGrantRevokeRoleParams;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-/**
- * Represents a "GRANT/REVOKE ROLE" statement.
- */
-public class GrantRevokeRoleStmt extends AuthorizationStmt {
- private final String roleName_;
- private final String groupName_;
- private final boolean isGrantStmt_;
-
- public GrantRevokeRoleStmt(String roleName, String groupName, boolean isGrantStmt) {
- Preconditions.checkNotNull(roleName);
- Preconditions.checkNotNull(groupName);
- roleName_ = roleName;
- groupName_ = groupName;
- isGrantStmt_ = isGrantStmt;
- }
-
- @Override
- public String toSql() {
- if (isGrantStmt_) {
- return String.format("GRANT ROLE %s TO %s", roleName_, groupName_);
- } else {
- return String.format("REVOKE ROLE %s FROM %s", roleName_, groupName_);
- }
- }
-
- public TGrantRevokeRoleParams toThrift() {
- TGrantRevokeRoleParams params = new TGrantRevokeRoleParams();
- params.setRole_names(Lists.newArrayList(roleName_));
- params.setGroup_names(Lists.newArrayList(groupName_));
- params.setIs_grant(isGrantStmt_);
- return params;
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- super.analyze(analyzer);
- if (analyzer.getCatalog().getAuthPolicy().getRole(roleName_) == null) {
- throw new AnalysisException(String.format("Role '%s' does not exist.", roleName_));
- }
- if (Strings.isNullOrEmpty(roleName_)) {
- throw new AnalysisException("Role name in GRANT/REVOKE ROLE cannot be empty.");
- }
- if (Strings.isNullOrEmpty(groupName_)) {
- throw new AnalysisException("Group name in GRANT/REVOKE ROLE cannot be empty.");
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java b/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java
deleted file mode 100644
index 1b6cff2..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/HdfsCachingOp.java
+++ /dev/null
@@ -1,93 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.math.BigDecimal;
-
-import com.cloudera.impala.catalog.HdfsCachePool;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.thrift.THdfsCachingOp;
-import com.google.common.base.Preconditions;
-
-/**
- * Represents the partial SQL statement of specifying whether a table/partition
- * should or should not be marked as cached.
- */
-public class HdfsCachingOp implements ParseNode {
- private final THdfsCachingOp cacheOp_;
- private final BigDecimal parsedReplication_;
-
- /**
- * Creates an HdfsCachingOp that specifies the target should be uncached
- */
- public HdfsCachingOp() {
- cacheOp_ = new THdfsCachingOp(false);
- parsedReplication_ = null;
- }
-
- /**
- * Creates an HdfsCachingOp that specifies the target should be cached in cachePoolName
- * with an optional replication factor
- */
- public HdfsCachingOp(String cachePoolName, BigDecimal replication) {
- cacheOp_ = new THdfsCachingOp(true);
- cacheOp_.setCache_pool_name(cachePoolName);
- parsedReplication_ = replication;
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (cacheOp_.isSet_cached()) {
- String poolName = cacheOp_.getCache_pool_name();
- Preconditions.checkNotNull(poolName);
- if (poolName.isEmpty()) {
- throw new AnalysisException("Cache pool name cannot be empty.");
- }
-
- HdfsCachePool cachePool = analyzer.getCatalog().getHdfsCachePool(poolName);
- if (cachePool == null) {
- throw new AnalysisException(
- "The specified cache pool does not exist: " + poolName);
- }
-
- if (parsedReplication_ != null && (parsedReplication_.longValue() <= 0 ||
- parsedReplication_.longValue() > Short.MAX_VALUE)) {
- throw new AnalysisException(
- "Cache replication factor must be between 0 and Short.MAX_VALUE");
- }
-
- if (parsedReplication_ != null) {
- cacheOp_.setReplication(parsedReplication_.shortValue());
- }
- }
- }
-
- @Override
- public String toSql() {
- return !shouldCache() ? "UNCACHED" : "CACHED IN '" + getCachePoolName() + "' WITH " +
- "REPLICATION = " + parsedReplication_.longValue();
- }
-
- public THdfsCachingOp toThrift() { return cacheOp_; }
-
- public boolean shouldCache() { return cacheOp_.isSet_cached(); }
-
- public String getCachePoolName() {
- return shouldCache() ? cacheOp_.getCache_pool_name() : null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java b/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java
deleted file mode 100644
index 9fbe467..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/HdfsUri.java
+++ /dev/null
@@ -1,128 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-
-import com.cloudera.impala.authorization.AuthorizeableUri;
-import com.cloudera.impala.authorization.Privilege;
-import com.cloudera.impala.authorization.PrivilegeRequest;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.common.FileSystemUtil;
-import com.cloudera.impala.util.FsPermissionChecker;
-import com.google.common.base.Preconditions;
-
-/**
- * Represents a Hadoop FileSystem URI in a SQL statement.
- */
-public class HdfsUri {
- private final String location_;
-
- // Set during analysis
- private Path uriPath_;
-
- public HdfsUri(String location) {
- Preconditions.checkNotNull(location);
- this.location_ = location.trim();
- }
-
- public Path getPath() {
- Preconditions.checkNotNull(uriPath_);
- return uriPath_;
- }
-
- public void analyze(Analyzer analyzer, Privilege privilege)
- throws AnalysisException {
- analyze(analyzer, privilege, FsAction.NONE, true);
- }
-
- public void analyze(Analyzer analyzer, Privilege privilege, FsAction perm)
- throws AnalysisException {
- analyze(analyzer, privilege, perm, true);
- }
-
- public void analyze(Analyzer analyzer, Privilege privilege, boolean registerPrivReq)
- throws AnalysisException {
- analyze(analyzer, privilege, FsAction.NONE, registerPrivReq);
- }
-
- /**
- * Analyzes the URI.
- * Optionally check location path permission, issue warning if impala user doesn't
- * have sufficient access rights.
- * Optionally register a privilege request. Used by GRANT/REVOKE privilege statements.
- */
- public void analyze(Analyzer analyzer, Privilege privilege, FsAction perm,
- boolean registerPrivReq) throws AnalysisException {
- if (location_.isEmpty()) {
- throw new AnalysisException("URI path cannot be empty.");
- }
-
- uriPath_ = new Path(location_);
- if (!uriPath_.isUriPathAbsolute()) {
- throw new AnalysisException("URI path must be absolute: " + uriPath_);
- }
-
- uriPath_ = FileSystemUtil.createFullyQualifiedPath(uriPath_);
-
- // Check if parent path exists and if impala is allowed to access it.
- Path parentPath = uriPath_.getParent();
- try {
- FileSystem fs = uriPath_.getFileSystem(FileSystemUtil.getConfiguration());
- boolean pathExists = false;
- StringBuilder errorMsg = new StringBuilder();
- try {
- pathExists = fs.exists(parentPath);
- if (!pathExists) errorMsg.append("Path does not exist.");
- } catch (Exception e) {
- errorMsg.append(e.getMessage());
- }
- if (!pathExists) {
- analyzer.addWarning(String.format("Path '%s' cannot be reached: %s",
- parentPath, errorMsg.toString()));
- } else if (perm != FsAction.NONE) {
- FsPermissionChecker checker = FsPermissionChecker.getInstance();
- if (!checker.getPermissions(fs, parentPath).checkPermissions(perm)) {
- analyzer.addWarning(String.format(
- "Impala does not have %s access to path '%s'",
- perm.toString(), parentPath));
- }
- }
- } catch (IOException e) {
- throw new AnalysisException(e.getMessage(), e);
- }
-
- if (registerPrivReq) {
- analyzer.registerPrivReq(new PrivilegeRequest(
- new AuthorizeableUri(uriPath_.toString()), privilege));
- }
- }
-
- @Override
- public String toString() {
- // If uriPath is null (this HdfsURI has not been analyzed yet) just return the raw
- // location string the caller passed in.
- return uriPath_ == null ? location_ : uriPath_.toString();
- }
-
- public String getLocation() { return location_; }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java b/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java
deleted file mode 100644
index 28d8f12..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/InPredicate.java
+++ /dev/null
@@ -1,234 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.cloudera.impala.catalog.Db;
-import com.cloudera.impala.catalog.Function.CompareMode;
-import com.cloudera.impala.catalog.PrimitiveType;
-import com.cloudera.impala.catalog.ScalarFunction;
-import com.cloudera.impala.catalog.Type;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.common.Reference;
-import com.cloudera.impala.thrift.TExprNode;
-import com.cloudera.impala.thrift.TExprNodeType;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * Class representing a [NOT] IN predicate. It determines if a specified value
- * (first child) matches any value in a subquery (second child) or a list
- * of values (remaining children).
- */
-public class InPredicate extends Predicate {
- private static final String IN_SET_LOOKUP = "in_set_lookup";
- private static final String NOT_IN_SET_LOOKUP = "not_in_set_lookup";
- private static final String IN_ITERATE= "in_iterate";
- private static final String NOT_IN_ITERATE = "not_in_iterate";
- private final boolean isNotIn_;
-
- public boolean isNotIn() { return isNotIn_; }
-
- public static void initBuiltins(Db db) {
- for (Type t: Type.getSupportedTypes()) {
- if (t.isNull()) continue;
- // TODO we do not support codegen for CHAR and the In predicate must be codegened
- // because it has variable number of arguments. This will force CHARs to be
- // cast up to strings; meaning that "in" comparisons will not have CHAR comparison
- // semantics.
- if (t.getPrimitiveType() == PrimitiveType.CHAR) continue;
-
- String typeString = t.getPrimitiveType().toString().toLowerCase();
- if (t.isScalarType(PrimitiveType.VARCHAR)) typeString = "string";
-
- db.addBuiltin(ScalarFunction.createBuiltin(IN_ITERATE,
- Lists.newArrayList(t, t), true, Type.BOOLEAN,
- "impala::InPredicate::InIterate", null, null, false));
- db.addBuiltin(ScalarFunction.createBuiltin(NOT_IN_ITERATE,
- Lists.newArrayList(t, t), true, Type.BOOLEAN,
- "impala::InPredicate::NotInIterate", null, null, false));
-
- String prepareFn = "impala::InPredicate::SetLookupPrepare_" + typeString;
- String closeFn = "impala::InPredicate::SetLookupClose_" + typeString;
-
- db.addBuiltin(ScalarFunction.createBuiltin(IN_SET_LOOKUP,
- Lists.newArrayList(t, t), true, Type.BOOLEAN,
- "impala::InPredicate::InSetLookup", prepareFn, closeFn, false));
- db.addBuiltin(ScalarFunction.createBuiltin(NOT_IN_SET_LOOKUP,
- Lists.newArrayList(t, t), true, Type.BOOLEAN,
- "impala::InPredicate::NotInSetLookup", prepareFn, closeFn, false));
-
- }
- }
-
- // First child is the comparison expr for which we
- // should check membership in the inList (the remaining children).
- public InPredicate(Expr compareExpr, List<Expr> inList, boolean isNotIn) {
- children_.add(compareExpr);
- children_.addAll(inList);
- isNotIn_ = isNotIn;
- }
-
- // C'tor for initializing an [NOT] IN predicate with a subquery child.
- public InPredicate(Expr compareExpr, Expr subquery, boolean isNotIn) {
- Preconditions.checkNotNull(compareExpr);
- Preconditions.checkNotNull(subquery);
- children_.add(compareExpr);
- children_.add(subquery);
- isNotIn_ = isNotIn;
- }
-
- /**
- * Copy c'tor used in clone().
- */
- protected InPredicate(InPredicate other) {
- super(other);
- isNotIn_ = other.isNotIn_;
- }
-
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (isAnalyzed_) return;
- super.analyze(analyzer);
-
- if (contains(Subquery.class)) {
- // An [NOT] IN predicate with a subquery must contain two children, the second of
- // which is a Subquery.
- if (children_.size() != 2 || !(getChild(1) instanceof Subquery)) {
- throw new AnalysisException("Unsupported IN predicate with a subquery: " +
- toSqlImpl());
- }
- Subquery subquery = (Subquery)getChild(1);
- if (!subquery.returnsScalarColumn()) {
- throw new AnalysisException("Subquery must return a single column: " +
- subquery.toSql());
- }
-
- // Ensure that the column in the lhs of the IN predicate and the result of
- // the subquery are type compatible. No need to perform any
- // casting at this point. Any casting needed will be performed when the
- // subquery is unnested.
- ArrayList<Expr> subqueryExprs = subquery.getStatement().getResultExprs();
- Expr compareExpr = children_.get(0);
- Expr subqueryExpr = subqueryExprs.get(0);
- analyzer.getCompatibleType(compareExpr.getType(), compareExpr, subqueryExpr);
- } else {
- Preconditions.checkState(getChildren().size() >= 2);
- analyzer.castAllToCompatibleType(children_);
- Type childType = children_.get(0).getType();
-
- if (childType.isNull()) {
- // Make sure the BE never sees TYPE_NULL by picking an arbitrary type
- for (int i = 0; i < children_.size(); ++i) {
- uncheckedCastChild(Type.BOOLEAN, i);
- }
- }
-
- // Choose SetLookup or Iterate strategy. SetLookup can be used if all the exprs in
- // the IN list are constant, and is faster than iterating if the IN list is big
- // enough.
- boolean allConstant = true;
- for (int i = 1; i < children_.size(); ++i) {
- if (!children_.get(i).isConstant()) {
- allConstant = false;
- break;
- }
- }
- boolean useSetLookup = allConstant;
- // Threshold based on InPredicateBenchmark results
- int setLookupThreshold = children_.get(0).getType().isStringType() ? 6 : 2;
- if (children_.size() - 1 < setLookupThreshold) useSetLookup = false;
-
- // Only lookup fn_ if all subqueries have been rewritten. If the second child is a
- // subquery, it will have type ArrayType, which cannot be resolved to a builtin
- // function and will fail analysis.
- Type[] argTypes = {getChild(0).type_, getChild(1).type_};
- if (useSetLookup) {
- fn_ = getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_SET_LOOKUP : IN_SET_LOOKUP,
- argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
- } else {
- fn_ = getBuiltinFunction(analyzer, isNotIn_ ? NOT_IN_ITERATE : IN_ITERATE,
- argTypes, CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
- }
- Preconditions.checkNotNull(fn_);
- Preconditions.checkState(fn_.getReturnType().isBoolean());
- castForFunctionCall(false);
- }
-
- // TODO: Fix selectivity_ for nested predicate
- Reference<SlotRef> slotRefRef = new Reference<SlotRef>();
- Reference<Integer> idxRef = new Reference<Integer>();
- if (isSingleColumnPredicate(slotRefRef, idxRef)
- && idxRef.getRef() == 0
- && slotRefRef.getRef().getNumDistinctValues() > 0) {
- selectivity_ = (double) (getChildren().size() - 1)
- / (double) slotRefRef.getRef().getNumDistinctValues();
- selectivity_ = Math.max(0.0, Math.min(1.0, selectivity_));
- }
-
- if (hasChildCosts()) {
- // BINARY_PREDICATE_COST accounts for the cost of performing the comparison.
- evalCost_ = getChildCosts() + BINARY_PREDICATE_COST * (children_.size() - 1);
- }
- }
-
- @Override
- protected void toThrift(TExprNode msg) {
- // Can't serialize a predicate with a subquery
- Preconditions.checkState(!contains(Subquery.class));
- msg.node_type = TExprNodeType.FUNCTION_CALL;
- }
-
- @Override
- public String toSqlImpl() {
- StringBuilder strBuilder = new StringBuilder();
- String notStr = (isNotIn_) ? "NOT " : "";
- strBuilder.append(getChild(0).toSql() + " " + notStr + "IN ");
- boolean hasSubquery = contains(Subquery.class);
- if (!hasSubquery) strBuilder.append("(");
- for (int i = 1; i < children_.size(); ++i) {
- strBuilder.append(getChild(i).toSql());
- strBuilder.append((i+1 != children_.size()) ? ", " : "");
- }
- if (!hasSubquery) strBuilder.append(")");
- return strBuilder.toString();
- }
-
- /**
- * If predicate is of the form "<SlotRef> [NOT] IN", returns the
- * SlotRef.
- */
- @Override
- public SlotRef getBoundSlot() {
- return getChild(0).unwrapSlotRef(true);
- }
-
- /**
- * Negates an InPredicate.
- */
- @Override
- public Expr negate() {
- return new InPredicate(getChild(0), children_.subList(1, children_.size()),
- !isNotIn_);
- }
-
- @Override
- public Expr clone() { return new InPredicate(this); }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java b/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java
deleted file mode 100644
index a6c62b0..0000000
--- a/fe/src/main/java/com/cloudera/impala/analysis/InlineViewRef.java
+++ /dev/null
@@ -1,339 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.analysis;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.impala.catalog.ColumnStats;
-import com.cloudera.impala.catalog.StructField;
-import com.cloudera.impala.catalog.StructType;
-import com.cloudera.impala.catalog.View;
-import com.cloudera.impala.common.AnalysisException;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-/**
- * An inline view is a query statement with an alias. Inline views can be parsed directly
- * from a query string or represent a reference to a local or catalog view.
- */
-public class InlineViewRef extends TableRef {
- private final static Logger LOG = LoggerFactory.getLogger(SelectStmt.class);
-
- // Catalog or local view that is referenced.
- // Null for inline views parsed directly from a query string.
- private final View view_;
-
- // If not null, these will serve as the column labels for the inline view. This provides
- // a layer of separation between column labels visible from outside the inline view
- // and column labels used in the query definition. Either all or none of the column
- // labels must be overridden.
- private List<String> explicitColLabels_;
-
- /////////////////////////////////////////
- // BEGIN: Members that need to be reset()
-
- // The select or union statement of the inline view
- protected QueryStmt queryStmt_;
-
- // queryStmt has its own analysis context
- protected Analyzer inlineViewAnalyzer_;
-
- // list of tuple ids materialized by queryStmt
- protected final ArrayList<TupleId> materializedTupleIds_ = Lists.newArrayList();
-
- // Map inline view's output slots to the corresponding resultExpr of queryStmt.
- protected final ExprSubstitutionMap smap_;
-
- // Map inline view's output slots to the corresponding baseTblResultExpr of queryStmt.
- protected final ExprSubstitutionMap baseTblSmap_;
-
- // END: Members that need to be reset()
- /////////////////////////////////////////
-
- /**
- * C'tor for creating inline views parsed directly from the a query string.
- */
- public InlineViewRef(String alias, QueryStmt queryStmt) {
- super(null, alias);
- Preconditions.checkNotNull(queryStmt);
- queryStmt_ = queryStmt;
- view_ = null;
- smap_ = new ExprSubstitutionMap();
- baseTblSmap_ = new ExprSubstitutionMap();
- }
-
- public InlineViewRef(String alias, QueryStmt queryStmt, List<String> colLabels) {
- this(alias, queryStmt);
- explicitColLabels_ = Lists.newArrayList(colLabels);
- }
-
- /**
- * C'tor for creating inline views that replace a local or catalog view ref.
- */
- public InlineViewRef(View view, TableRef origTblRef) {
- super(view.getTableName().toPath(), origTblRef.getExplicitAlias());
- queryStmt_ = view.getQueryStmt().clone();
- queryStmt_.reset();
- if (view.isLocalView()) queryStmt_.reset();
- view_ = view;
- smap_ = new ExprSubstitutionMap();
- baseTblSmap_ = new ExprSubstitutionMap();
- setJoinAttrs(origTblRef);
- explicitColLabels_ = view.getColLabels();
- // Set implicit aliases if no explicit one was given.
- if (hasExplicitAlias()) return;
- aliases_ = new String[] {
- view_.getTableName().toString().toLowerCase(), view_.getName().toLowerCase()
- };
- }
-
- /**
- * C'tor for cloning.
- */
- public InlineViewRef(InlineViewRef other) {
- super(other);
- Preconditions.checkNotNull(other.queryStmt_);
- view_ = other.view_;
- queryStmt_ = other.queryStmt_.clone();
- inlineViewAnalyzer_ = other.inlineViewAnalyzer_;
- if (other.explicitColLabels_ != null) {
- explicitColLabels_ = Lists.newArrayList(other.explicitColLabels_);
- }
- materializedTupleIds_.addAll(other.materializedTupleIds_);
- smap_ = other.smap_.clone();
- baseTblSmap_ = other.baseTblSmap_.clone();
- }
-
- /**
- * Analyzes the inline view query block in a child analyzer of 'analyzer', creates
- * a new tuple descriptor for the inline view and registers auxiliary eq predicates
- * between the slots of that descriptor and the select list exprs of the inline view;
- * then performs join clause analysis.
- */
- @Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
- if (isAnalyzed_) return;
-
- // Analyze the inline view query statement with its own analyzer
- inlineViewAnalyzer_ = new Analyzer(analyzer);
-
- // Catalog views refs require special analysis settings for authorization.
- boolean isCatalogView = (view_ != null && !view_.isLocalView());
- if (isCatalogView) {
- analyzer.registerAuthAndAuditEvent(view_, analyzer);
- if (inlineViewAnalyzer_.isExplain()) {
- // If the user does not have privileges on the view's definition
- // then we report a masked authorization error so as not to reveal
- // privileged information (e.g., the existence of a table).
- inlineViewAnalyzer_.setAuthErrMsg(
- String.format("User '%s' does not have privileges to " +
- "EXPLAIN this statement.", analyzer.getUser().getName()));
- } else {
- // If this is not an EXPLAIN statement, auth checks for the view
- // definition should be disabled.
- inlineViewAnalyzer_.setEnablePrivChecks(false);
- }
- }
-
- inlineViewAnalyzer_.setUseHiveColLabels(
- isCatalogView ? true : analyzer.useHiveColLabels());
- queryStmt_.analyze(inlineViewAnalyzer_);
- correlatedTupleIds_.addAll(queryStmt_.getCorrelatedTupleIds(inlineViewAnalyzer_));
- if (explicitColLabels_ != null) {
- Preconditions.checkState(
- explicitColLabels_.size() == queryStmt_.getColLabels().size());
- }
-
- inlineViewAnalyzer_.setHasLimitOffsetClause(
- queryStmt_.hasLimit() || queryStmt_.hasOffset());
- queryStmt_.getMaterializedTupleIds(materializedTupleIds_);
- desc_ = analyzer.registerTableRef(this);
- isAnalyzed_ = true; // true now that we have assigned desc
-
- // For constant selects we materialize its exprs into a tuple.
- if (materializedTupleIds_.isEmpty()) {
- Preconditions.checkState(queryStmt_ instanceof SelectStmt);
- Preconditions.checkState(((SelectStmt) queryStmt_).getTableRefs().isEmpty());
- desc_.setIsMaterialized(true);
- materializedTupleIds_.add(desc_.getId());
- }
-
- // create smap_ and baseTblSmap_ and register auxiliary eq predicates between our
- // tuple descriptor's slots and our *unresolved* select list exprs;
- // we create these auxiliary predicates so that the analyzer can compute the value
- // transfer graph through this inline view correctly (ie, predicates can get
- // propagated through the view);
- // if the view stmt contains analytic functions, we cannot propagate predicates
- // into the view, unless the predicates are compatible with the analytic
- // function's partition by clause, because those extra filters
- // would alter the results of the analytic functions (see IMPALA-1243)
- // TODO: relax this a bit by allowing propagation out of the inline view (but
- // not into it)
- for (int i = 0; i < getColLabels().size(); ++i) {
- String colName = getColLabels().get(i).toLowerCase();
- Expr colExpr = queryStmt_.getResultExprs().get(i);
- Path p = new Path(desc_, Lists.newArrayList(colName));
- Preconditions.checkState(p.resolve());
- SlotDescriptor slotDesc = analyzer.registerSlotRef(p);
- slotDesc.setSourceExpr(colExpr);
- slotDesc.setStats(ColumnStats.fromExpr(colExpr));
- SlotRef slotRef = new SlotRef(slotDesc);
- smap_.put(slotRef, colExpr);
- baseTblSmap_.put(slotRef, queryStmt_.getBaseTblResultExprs().get(i));
- if (createAuxPredicate(colExpr)) {
- analyzer.createAuxEquivPredicate(new SlotRef(slotDesc), colExpr.clone());
- }
- }
- LOG.trace("inline view " + getUniqueAlias() + " smap: " + smap_.debugString());
- LOG.trace("inline view " + getUniqueAlias() + " baseTblSmap: " +
- baseTblSmap_.debugString());
-
- analyzeHints(analyzer);
- // Now do the remaining join analysis
- analyzeJoin(analyzer);
- }
-
- /**
- * Checks if an auxiliary predicate should be created for an expr. Returns False if the
- * inline view has a SELECT stmt with analytic functions and the expr is not in the
- * common partition exprs of all the analytic functions computed by this inline view.
- */
- public boolean createAuxPredicate(Expr e) {
- if (!(queryStmt_ instanceof SelectStmt)
- || !((SelectStmt) queryStmt_).hasAnalyticInfo()) {
- return true;
- }
- AnalyticInfo analyticInfo = ((SelectStmt) queryStmt_).getAnalyticInfo();
- return analyticInfo.getCommonPartitionExprs().contains(e);
- }
-
- /**
- * Create and register a non-materialized tuple descriptor for this inline view.
- * This method is called from the analyzer when registering this inline view.
- * Create a non-materialized tuple descriptor for this inline view.
- */
- @Override
- public TupleDescriptor createTupleDescriptor(Analyzer analyzer)
- throws AnalysisException {
- int numColLabels = getColLabels().size();
- Preconditions.checkState(numColLabels > 0);
- HashSet<String> uniqueColAliases = Sets.newHashSetWithExpectedSize(numColLabels);
- ArrayList<StructField> fields = Lists.newArrayListWithCapacity(numColLabels);
- for (int i = 0; i < numColLabels; ++i) {
- // inline view select statement has been analyzed. Col label should be filled.
- Expr selectItemExpr = queryStmt_.getResultExprs().get(i);
- String colAlias = getColLabels().get(i).toLowerCase();
-
- // inline view col cannot have duplicate name
- if (!uniqueColAliases.add(colAlias)) {
- throw new AnalysisException("duplicated inline view column alias: '" +
- colAlias + "'" + " in inline view " + "'" + getUniqueAlias() + "'");
- }
- fields.add(new StructField(colAlias, selectItemExpr.getType(), null));
- }
-
- // Create the non-materialized tuple and set its type.
- TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor(
- getClass().getSimpleName() + " " + getUniqueAlias());
- result.setIsMaterialized(false);
- result.setType(new StructType(fields));
- return result;
- }
-
- @Override
- public List<TupleId> getMaterializedTupleIds() {
- Preconditions.checkState(isAnalyzed_);
- Preconditions.checkState(materializedTupleIds_.size() > 0);
- return materializedTupleIds_;
- }
-
- public Analyzer getAnalyzer() {
- Preconditions.checkState(isAnalyzed_);
- return inlineViewAnalyzer_;
- }
-
- public ExprSubstitutionMap getSmap() {
- Preconditions.checkState(isAnalyzed_);
- return smap_;
- }
-
- public ExprSubstitutionMap getBaseTblSmap() {
- Preconditions.checkState(isAnalyzed_);
- return baseTblSmap_;
- }
-
- public QueryStmt getViewStmt() { return queryStmt_; }
- public void setRewrittenViewStmt(QueryStmt stmt) {
- Preconditions.checkState(getAnalyzer().containsSubquery());
- queryStmt_ = stmt;
- }
-
- public List<String> getExplicitColLabels() { return explicitColLabels_; }
-
- public List<String> getColLabels() {
- if (explicitColLabels_ != null) return explicitColLabels_;
- return queryStmt_.getColLabels();
- }
-
- @Override
- protected TableRef clone() { return new InlineViewRef(this); }
-
- @Override
- public void reset() {
- super.reset();
- queryStmt_.reset();
- inlineViewAnalyzer_ = null;
- materializedTupleIds_.clear();
- smap_.clear();
- baseTblSmap_.clear();
- }
-
- @Override
- protected String tableRefToSql() {
- // Enclose the alias in quotes if Hive cannot parse it without quotes.
- // This is needed for view compatibility between Impala and Hive.
- String aliasSql = null;
- String alias = getExplicitAlias();
- if (alias != null) aliasSql = ToSqlUtils.getIdentSql(alias);
- if (view_ != null) {
- return view_.getTableName().toSql() + (aliasSql == null ? "" : " " + aliasSql);
- }
- Preconditions.checkNotNull(aliasSql);
- StringBuilder sql = new StringBuilder()
- .append("(")
- .append(queryStmt_.toSql())
- .append(") ")
- .append(aliasSql);
- // Add explicit col labels for debugging even though this syntax isn't supported.
- if (explicitColLabels_ != null) {
- sql.append(" (");
- for (int i = 0; i < getExplicitColLabels().size(); i++) {
- if (i > 0) sql.append(", ");
- sql.append(ToSqlUtils.getIdentSql(getExplicitColLabels().get(i)));
- }
- sql.append(")");
- }
- return sql.toString();
- }
-}