You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/02/22 17:58:56 UTC
[3/6] impala git commit: IMPALA-5152: Introduce metadata loading phase
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
index 80ba630..d3eeb19 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeExprsTest.java
@@ -27,7 +27,6 @@ import java.util.Arrays;
import java.util.List;
import org.apache.impala.analysis.TimestampArithmeticExpr.TimeUnit;
-import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Catalog;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.Column;
@@ -1363,8 +1362,8 @@ public class AnalyzeExprsTest extends AnalyzerTest {
/**
* Get the result type of a select statement with a single select list element.
*/
- Type getReturnType(String stmt, Analyzer analyzer) {
- SelectStmt select = (SelectStmt) AnalyzesOk(stmt, analyzer, null);
+ Type getReturnType(String stmt, AnalysisContext ctx) {
+ SelectStmt select = (SelectStmt) AnalyzesOk(stmt, ctx);
List<Expr> selectListExprs = select.getResultExprs();
assertNotNull(selectListExprs);
assertEquals(selectListExprs.size(), 1);
@@ -1373,13 +1372,13 @@ public class AnalyzeExprsTest extends AnalyzerTest {
return expr.getType();
}
- private void checkReturnType(String stmt, Type resultType, Analyzer analyzer) {
- Type exprType = getReturnType(stmt, analyzer);
+ private void checkReturnType(String stmt, Type resultType, AnalysisContext ctx) {
+ Type exprType = getReturnType(stmt, ctx);
assertEquals("Expected: " + resultType + " != " + exprType, resultType, exprType);
}
private void checkReturnType(String stmt, Type resultType) {
- checkReturnType(stmt, resultType, createAnalyzer(Catalog.DEFAULT_DB));
+ checkReturnType(stmt, resultType, createAnalysisCtx(Catalog.DEFAULT_DB));
}
/**
@@ -1388,12 +1387,12 @@ public class AnalyzeExprsTest extends AnalyzerTest {
*/
private void checkDecimalReturnType(String stmt, Type decimalV1ResultType,
Type decimalV2ResultType) {
- Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
- analyzer.getQueryOptions().setDecimal_v2(false);
- checkReturnType(stmt, decimalV1ResultType, analyzer);
- analyzer = createAnalyzer(Catalog.DEFAULT_DB);
- analyzer.getQueryOptions().setDecimal_v2(true);
- checkReturnType(stmt, decimalV2ResultType, analyzer);
+ AnalysisContext ctx = createAnalysisCtx(Catalog.DEFAULT_DB);
+ ctx.getQueryOptions().setDecimal_v2(false);
+ checkReturnType(stmt, decimalV1ResultType, ctx);
+ ctx = createAnalysisCtx(Catalog.DEFAULT_DB);
+ ctx.getQueryOptions().setDecimal_v2(true);
+ checkReturnType(stmt, decimalV2ResultType, ctx);
}
/**
@@ -2214,10 +2213,11 @@ public class AnalyzeExprsTest extends AnalyzerTest {
// decimal v2.
private void testDecimalExpr(String expr,
Type decimalV1ExpectedType, Type decimalV2ExpectedType) {
- Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
+ TQueryOptions queryOpts = new TQueryOptions();
- analyzer.getQueryOptions().setDecimal_v2(false);
- SelectStmt selectStmt = (SelectStmt) AnalyzesOk("select " + expr, analyzer);
+ queryOpts.setDecimal_v2(false);
+ SelectStmt selectStmt =
+ (SelectStmt) AnalyzesOk("select " + expr, createAnalysisCtx(queryOpts));
Expr root = selectStmt.resultExprs_.get(0);
Type actualType = root.getType();
Assert.assertTrue(
@@ -2225,8 +2225,8 @@ public class AnalyzeExprsTest extends AnalyzerTest {
" Expected: " + decimalV1ExpectedType + " Actual: " + actualType,
decimalV1ExpectedType.equals(actualType));
- analyzer.getQueryOptions().setDecimal_v2(true);
- selectStmt = (SelectStmt) AnalyzesOk("select " + expr, analyzer);
+ queryOpts.setDecimal_v2(true);
+ selectStmt = (SelectStmt) AnalyzesOk("select " + expr, createAnalysisCtx(queryOpts));
root = selectStmt.resultExprs_.get(0);
actualType = root.getType();
Assert.assertTrue(
@@ -2563,16 +2563,16 @@ public class AnalyzeExprsTest extends AnalyzerTest {
"select count(distinct %s), sum(distinct smallint_col), " +
"avg(float_col), min(%s) " +
"from functional.alltypes",
- colName, colName), createAnalyzer(queryOptions));
+ colName, colName), createAnalysisCtx(queryOptions));
countDistinctFns.add(String.format("count(distinct %s)", colName));
}
// Test a single query with a count(distinct) on all columns of alltypesTbl.
AnalyzesOk(String.format("select %s from functional.alltypes",
- Joiner.on(",").join(countDistinctFns)), createAnalyzer(queryOptions));
+ Joiner.on(",").join(countDistinctFns)), createAnalysisCtx(queryOptions));
allCountDistinctFns.addAll(countDistinctFns);
countDistinctFns.clear();
- Table decimalTbl = catalog_.getTable("functional", "decimal_tbl");
+ Table decimalTbl = catalog_.getOrLoadTable("functional", "decimal_tbl");
for (Column col: decimalTbl.getColumns()) {
String colName = col.getName();
// Test a single count(distinct) with some other aggs.
@@ -2580,12 +2580,12 @@ public class AnalyzeExprsTest extends AnalyzerTest {
"select count(distinct %s), sum(distinct d1), " +
"avg(d2), min(%s) " +
"from functional.decimal_tbl",
- colName, colName), createAnalyzer(queryOptions));
+ colName, colName), createAnalysisCtx(queryOptions));
countDistinctFns.add(String.format("count(distinct %s)", colName));
}
// Test a single query with a count(distinct) on all columns of decimalTbl.
AnalyzesOk(String.format("select %s from functional.decimal_tbl",
- Joiner.on(",").join(countDistinctFns)), createAnalyzer(queryOptions));
+ Joiner.on(",").join(countDistinctFns)), createAnalysisCtx(queryOptions));
allCountDistinctFns.addAll(countDistinctFns);
@@ -2593,19 +2593,19 @@ public class AnalyzeExprsTest extends AnalyzerTest {
// alltypes/decimalTbl.
AnalyzesOk(String.format(
"select %s from functional.alltypes cross join functional.decimal_tbl",
- Joiner.on(",").join(countDistinctFns)), createAnalyzer(queryOptions));
+ Joiner.on(",").join(countDistinctFns)), createAnalysisCtx(queryOptions));
// The rewrite does not work for multiple count() arguments.
AnalysisError("select count(distinct int_col, bigint_col), " +
"count(distinct string_col, float_col) from functional.alltypes",
- createAnalyzer(queryOptions),
+ createAnalysisCtx(queryOptions),
"all DISTINCT aggregate functions need to have the same set of parameters as " +
"count(DISTINCT int_col, bigint_col); deviating function: " +
"count(DISTINCT string_col, float_col)");
// The rewrite only applies to the count() function.
AnalysisError(
"select avg(distinct int_col), sum(distinct float_col) from functional.alltypes",
- createAnalyzer(queryOptions),
+ createAnalysisCtx(queryOptions),
"all DISTINCT aggregate functions need to have the same set of parameters as " +
"avg(DISTINCT int_col); deviating function: sum(DISTINCT");
}
@@ -2623,8 +2623,7 @@ public class AnalyzeExprsTest extends AnalyzerTest {
Assert.assertTrue(tinyIntFn.compare(decimalFn,
CompareMode.IS_NONSTRICT_SUPERTYPE_OF));
// Check that this resolves to the decimal version of the function.
- Analyzer analyzer = createAnalyzer(Catalog.BUILTINS_DB);
- Db db = analyzer.getDb(Catalog.BUILTINS_DB, Privilege.VIEW_METADATA, true);
+ Db db = catalog_.getDb(Catalog.BUILTINS_DB);
Function foundFn = db.getFunction(decimalFn, CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
assertNotNull(foundFn);
Assert.assertTrue(foundFn.getArgs()[0].isDecimal());
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AnalyzeModifyStmtsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeModifyStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeModifyStmtsTest.java
index 152add6..3f6d72d 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeModifyStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeModifyStmtsTest.java
@@ -17,9 +17,8 @@
package org.apache.impala.analysis;
-import org.junit.Test;
-
import org.apache.impala.testutil.TestUtils;
+import org.junit.Test;
/**
* Tests analysis phase of the ModifyStmt and its sub-classes.
@@ -42,12 +41,12 @@ public class AnalyzeModifyStmtsTest extends AnalyzerTest {
AnalyzesOk("update a set a.name = 'values' from functional_kudu.testtbl a " +
"where a.zip in (select zip from functional.testtbl limit 10)");
AnalyzesOk("update functional_kudu.dimtbl set name = 'Oskar' FROM dimtbl",
- createAnalyzer("functional_kudu"));
+ createAnalysisCtx("functional_kudu"));
AnalysisError("update a set b.name = 'Oskar' FROM dimtbl b",
- createAnalyzer("functional_kudu"),
+ createAnalysisCtx("functional_kudu"),
"'a' is not a valid table alias or reference.");
AnalyzesOk("update a set a.name = 'Oskar' FROM dimtbl a",
- createAnalyzer("functional_kudu"));
+ createAnalysisCtx("functional_kudu"));
// Table name is an implicit alias
AnalyzesOk(
"update functional_kudu.dimtbl set name = 'Oskar' FROM functional_kudu.dimtbl");
@@ -65,7 +64,7 @@ public class AnalyzeModifyStmtsTest extends AnalyzerTest {
// Location of the kudu table doesnt matter
AnalyzesOk(
"update a set a.name = 'Oskar' from functional.testtbl b, dimtbl a where b.id =" +
- " a.id ", createAnalyzer("functional_kudu"));
+ " a.id ", createAnalysisCtx("functional_kudu"));
AnalyzesOk("update a set name = 'Oskar' from functional_kudu.testtbl a");
AnalysisError(
"update functional_kudu.testtbl set name = 'Oskar' from functional_kudu.dimtbl",
@@ -87,7 +86,8 @@ public class AnalyzeModifyStmtsTest extends AnalyzerTest {
TestUtils.assumeKuduIsSupported();
AnalyzesOk("update functional_kudu.dimtbl set name = 'Oskar'");
// Correct default database resolution
- AnalyzesOk("update dimtbl set name = 'Oskar'", createAnalyzer("functional_kudu"));
+ AnalyzesOk("update dimtbl set name = 'Oskar'",
+ createAnalysisCtx("functional_kudu"));
// Correct table alias resolution
AnalyzesOk("update functional_kudu.dimtbl set name = '10'");
// Check type compatibility, zip is int, 4711 is smallint
@@ -239,5 +239,4 @@ public class AnalyzeModifyStmtsTest extends AnalyzerTest {
"'functional.allcomplextypes.int_array_col' is not a valid table alias or " +
"reference.");
}
-
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index b4c2e43..36d6970 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -122,7 +122,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
// Parent/collection join requires the child to use an alias of the parent.
AnalysisError(String.format(
"select %s from allcomplextypes, %s", collectionField, collectionTable),
- createAnalyzer("functional"),
+ createAnalysisCtx("functional"),
String.format("Could not resolve table reference: '%s'", collectionTable));
AnalysisError(String.format(
"select %s from functional.allcomplextypes, %s",
@@ -315,7 +315,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalysisError("select alltypes.smallint_col, functional.alltypes.int_col " +
"from alltypes inner join functional.alltypes " +
"on (alltypes.id = functional.alltypes.id)",
- createAnalyzer("functional"),
+ createAnalysisCtx("functional"),
"Duplicate table alias: 'functional.alltypes'");
}
@@ -410,7 +410,6 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
}
List<List<Integer>> expectedPaths = Lists.newArrayList(expectedAbsPaths);
Assert.assertEquals("Mismatched absolute paths.", expectedPaths, actualAbsPaths);
-
}
/**
@@ -805,13 +804,13 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalyzesOk("select 1 from a.a");
AnalyzesOk("select 1 from a.a.a");
AnalyzesOk("select 1 from a.a.a.a");
- AnalyzesOk("select 1 from a", createAnalyzer("a"));
- AnalyzesOk("select 1 from a.a.a.a", createAnalyzer("a"));
+ AnalyzesOk("select 1 from a", createAnalysisCtx("a"));
+ AnalyzesOk("select 1 from a.a.a.a", createAnalysisCtx("a"));
// Table paths are ambiguous.
- AnalysisError("select 1 from a.a", createAnalyzer("a"),
+ AnalysisError("select 1 from a.a", createAnalysisCtx("a"),
"Table reference is ambiguous: 'a.a'");
- AnalysisError("select 1 from a.a.a", createAnalyzer("a"),
+ AnalysisError("select 1 from a.a.a", createAnalysisCtx("a"),
"Table reference is ambiguous: 'a.a.a'");
// Ambiguous reference to registered table aliases.
@@ -1211,18 +1210,18 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
// test auto-generated column labels by enforcing their use in inline views
AnalyzesOk("select _c0, a, int_col, _c3 from " +
"(select int_col * 1, int_col as a, int_col, !bool_col, concat(string_col) " +
- "from functional.alltypes) t", createAnalyzerUsingHiveColLabels());
+ "from functional.alltypes) t", createAnalysisCtxUsingHiveColLabels());
// test auto-generated column labels in group by and order by
AnalyzesOk("select _c0, count(a), count(int_col), _c3 from " +
"(select int_col * 1, int_col as a, int_col, !bool_col, concat(string_col) " +
"from functional.alltypes) t group by _c0, _c3 order by _c0 limit 10",
- createAnalyzerUsingHiveColLabels());
+ createAnalysisCtxUsingHiveColLabels());
// test auto-generated column labels in multiple scopes
AnalyzesOk("select x.front, x._c1, x._c2 from " +
"(select y.back as front, y._c0 * 10, y._c2 + 2 from " +
"(select int_col * 10, int_col as back, int_col + 2 from " +
"functional.alltypestiny) y) x",
- createAnalyzerUsingHiveColLabels());
+ createAnalysisCtxUsingHiveColLabels());
// IMPALA-3537: Test that auto-generated column labels are only applied in
// the appropriate child query blocks.
SelectStmt colLabelsStmt =
@@ -1236,13 +1235,13 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
"(select int_col * 2, id from functional.alltypes) a inner join " +
"(select int_col + 6, id from functional.alltypes) b " +
"on (a.id = b.id)",
- createAnalyzerUsingHiveColLabels(),
+ createAnalysisCtxUsingHiveColLabels(),
"Column/field reference is ambiguous: '_c0'");
// auto-generated column doesn't exist
AnalysisError("select _c0, a, _c2, _c3 from " +
"(select int_col * 1, int_col as a, int_col, !bool_col, concat(string_col) " +
"from functional.alltypes) t",
- createAnalyzerUsingHiveColLabels(),
+ createAnalysisCtxUsingHiveColLabels(),
"Could not resolve column/field reference: '_c2'");
// Regression test for IMPALA-984.
@@ -1332,7 +1331,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalyzesOk("select cnt from functional.allcomplextypes, " +
"(select count(1) cnt from functional.allcomplextypes) v");
AnalyzesOk("select cnt from functional.allcomplextypes, " +
- "(select count(1) cnt from allcomplextypes) v", createAnalyzer("functional"));
+ "(select count(1) cnt from allcomplextypes) v", createAnalysisCtx("functional"));
// Illegal correlated reference.
AnalysisError("select cnt from functional.allcomplextypes t, " +
"(select count(1) cnt from t) v",
@@ -2843,19 +2842,19 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
AnalyzesOk("with t1 as (select int_col x, bigint_col y from alltypes), " +
"alltypes as (select x a, y b from t1)" +
"select a, b from alltypes",
- createAnalyzer("functional"));
+ createAnalysisCtx("functional"));
// Recursion is prevented because of scoping rules. The inner 'complex_view'
// refers to a view in the catalog.
AnalyzesOk("with t1 as (select abc x, xyz y from complex_view), " +
"complex_view as (select x a, y b from t1)" +
"select a, b from complex_view",
- createAnalyzer("functional"));
+ createAnalysisCtx("functional"));
// Nested WITH clauses. Scoping prevents recursion.
AnalyzesOk("with t1 as (with t1 as (select int_col x, bigint_col y from alltypes) " +
"select x, y from t1), " +
"alltypes as (select x a, y b from t1) " +
"select a, b from alltypes",
- createAnalyzer("functional"));
+ createAnalysisCtx("functional"));
// Nested WITH clause inside a subquery.
AnalyzesOk("with t1 as " +
"(select * from (with t2 as (select * from functional.alltypes) " +
@@ -2928,7 +2927,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
// The inner alltypes_view gets resolved to the catalog view.
AnalyzesOk("with alltypes_view as (select int_col x from alltypes_view) " +
"select x from alltypes_view",
- createAnalyzer("functional"));
+ createAnalysisCtx("functional"));
// The inner 't' is resolved to a non-existent base table.
AnalysisError("with t as (select int_col x, bigint_col y from t1) " +
"select x, y from t",
@@ -3381,7 +3380,8 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
// Unknown target DB
AnalysisError("INSERT " + qualifier + " table UNKNOWNDB.alltypesnopart SELECT * " +
- "from functional.alltypesnopart", "Database does not exist: UNKNOWNDB");
+ "from functional.alltypesnopart",
+ "Database does not exist: UNKNOWNDB");
}
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AnalyzeSubqueriesTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeSubqueriesTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeSubqueriesTest.java
index e49e289..de8632e 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeSubqueriesTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeSubqueriesTest.java
@@ -17,10 +17,9 @@
package org.apache.impala.analysis;
-import org.junit.Test;
-
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
+import org.junit.Test;
public class AnalyzeSubqueriesTest extends AnalyzerTest {
private static String cmpOperators[] = {"=", "!=", "<=", ">=", ">", "<"};
@@ -1198,7 +1197,7 @@ public class AnalyzeSubqueriesTest extends AnalyzerTest {
"where id in (select id from functional.allcomplextypes)");
AnalyzesOk("select id from functional.allcomplextypes " +
"where id < (select count(1) cnt from allcomplextypes)",
- createAnalyzer("functional"));
+ createAnalysisCtx("functional"));
// Illegal correlated table references.
AnalysisError("select id from (select * from functional.alltypestiny) t " +
"where t.int_col = (select count(*) from t)",
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
index d6cd804..fc7b51a 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeUpsertStmtTest.java
@@ -17,9 +17,8 @@
package org.apache.impala.analysis;
-import org.junit.Test;
-
import org.apache.impala.testutil.TestUtils;
+import org.junit.Test;
public class AnalyzeUpsertStmtTest extends AnalyzerTest {
@Test
@@ -101,7 +100,8 @@ public class AnalyzeUpsertStmtTest extends AnalyzerTest {
"UPSERT is only supported for Kudu tables");
// Unknown target DB
AnalysisError("upsert into UNKNOWNDB.testtbl select * " +
- "from functional.alltypesnopart", "Database does not exist: UNKNOWNDB");
+ "from functional.alltypesnopart",
+ "Database does not exist: UNKNOWNDB");
// WITH-clause tables cannot be upserted into
AnalysisError("with t1 as (select 'a' x) upsert into t1 values('b' x)",
"Table does not exist: default.t1");
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
index aab2dba..7fc8768 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
@@ -114,7 +114,7 @@ public class AnalyzerTest extends FrontendTestBase {
Preconditions.checkState(tbl.isFullyQualified());
Preconditions.checkState(query.contains("$TBL"));
String uqQuery = query.replace("$TBL", tbl.getTbl());
- AnalyzesOk(uqQuery, createAnalyzer(tbl.getDb()));
+ AnalyzesOk(uqQuery, createAnalysisCtx(tbl.getDb()));
String fqQuery = query.replace("$TBL", tbl.toString());
AnalyzesOk(fqQuery);
}
@@ -128,7 +128,7 @@ public class AnalyzerTest extends FrontendTestBase {
Preconditions.checkState(tbl.isFullyQualified());
Preconditions.checkState(query.contains("$TBL"));
String uqQuery = query.replace("$TBL", tbl.getTbl());
- AnalysisError(uqQuery, createAnalyzer(tbl.getDb()), expectedError);
+ AnalysisError(uqQuery, createAnalysisCtx(tbl.getDb()), expectedError);
String fqQuery = query.replace("$TBL", tbl.toString());
AnalysisError(fqQuery, expectedError);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AuditingTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuditingTest.java b/fe/src/test/java/org/apache/impala/analysis/AuditingTest.java
index a14fbc9..610bf56 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuditingTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuditingTest.java
@@ -19,27 +19,28 @@ package org.apache.impala.analysis;
import java.util.Set;
-import org.junit.Assert;
-import org.junit.Test;
-
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.catalog.AuthorizationException;
import org.apache.impala.catalog.Catalog;
import org.apache.impala.catalog.ImpaladCatalog;
import org.apache.impala.common.AnalysisException;
-import org.apache.impala.common.InternalException;
+import org.apache.impala.common.FrontendTestBase;
+import org.apache.impala.common.ImpalaException;
import org.apache.impala.service.Frontend;
import org.apache.impala.testutil.ImpaladTestCatalog;
import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TAccessEvent;
import org.apache.impala.thrift.TCatalogObjectType;
+import org.junit.Assert;
+import org.junit.Test;
+
import com.google.common.collect.Sets;
/**
* Tests that auditing access events are properly captured during analysis for all
* statement types.
*/
-public class AuditingTest extends AnalyzerTest {
+public class AuditingTest extends FrontendTestBase {
@Test
public void TestSelect() throws AuthorizationException, AnalysisException {
// Simple select from a table.
@@ -354,22 +355,20 @@ public class AuditingTest extends AnalyzerTest {
@Test
public void TestAccessEventsOnAuthFailure() throws AuthorizationException,
- AnalysisException, InternalException {
+ ImpalaException {
// The policy file doesn't exist so all operations will result in
// an AuthorizationError
AuthorizationConfig config = AuthorizationConfig.createHadoopGroupAuthConfig(
"server1", "/does/not/exist", "");
ImpaladCatalog catalog = new ImpaladTestCatalog(config);
Frontend fe = new Frontend(config, catalog);
- AnalysisContext analysisContext =
- new AnalysisContext(catalog, TestUtils.createQueryContext(), config);
+ AnalysisContext analysisCtx = createAnalysisCtx(config);
// We should get an audit event even when an authorization failure occurs.
try {
- analysisContext.analyze("create table foo_does_not_exist(i int)");
- analysisContext.authorize(fe.getAuthzChecker());
+ parseAndAnalyze("create table foo_does_not_exist(i int)", analysisCtx, fe);
Assert.fail("Expected AuthorizationException");
} catch (AuthorizationException e) {
- Assert.assertEquals(1, analysisContext.getAnalyzer().getAccessEvents().size());
+ Assert.assertEquals(1, analysisCtx.getAnalyzer().getAccessEvents().size());
}
}
@@ -468,8 +467,8 @@ public class AuditingTest extends AnalyzerTest {
private Set<TAccessEvent> AnalyzeAccessEvents(String stmt, String db)
throws AuthorizationException, AnalysisException {
- Analyzer analyzer = createAnalyzer(db);
- AnalyzesOk(stmt, analyzer);
- return analyzer.getAccessEvents();
+ AnalysisContext ctx = createAnalysisCtx(db);
+ AnalyzesOk(stmt, ctx);
+ return ctx.getAnalyzer().getAccessEvents();
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
index 9140e52..bcea229 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
@@ -17,6 +17,7 @@
package org.apache.impala.analysis;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -29,39 +30,24 @@ import java.util.Map;
import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import org.apache.sentry.provider.common.ResourceAuthorizationProvider;
import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
import org.apache.hive.service.rpc.thrift.TGetTablesReq;
-import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.AuthorizeableTable;
import org.apache.impala.authorization.User;
import org.apache.impala.catalog.AuthorizationException;
-import org.apache.impala.catalog.Catalog;
import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.ImpaladCatalog;
import org.apache.impala.catalog.ScalarFunction;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
+import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.service.Frontend;
import org.apache.impala.testutil.ImpaladTestCatalog;
-import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TFunctionBinaryType;
import org.apache.impala.thrift.TMetadataOpRequest;
import org.apache.impala.thrift.TMetadataOpcode;
@@ -69,17 +55,27 @@ import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TPrivilegeLevel;
import org.apache.impala.thrift.TPrivilegeScope;
-import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TResultSet;
import org.apache.impala.thrift.TSessionState;
import org.apache.impala.util.PatternMatcher;
import org.apache.impala.util.SentryPolicyService;
+import org.apache.sentry.provider.common.ResourceAuthorizationProvider;
+import org.apache.sentry.provider.file.LocalGroupResourceAuthorizationProvider;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@RunWith(Parameterized.class)
-public class AuthorizationTest {
+public class AuthorizationTest extends FrontendTestBase {
// Policy file has defined current user and 'test_user' have:
// ALL permission on 'tpch' database and 'newdb' database
// ALL permission on 'functional_seq_snap' database
@@ -123,7 +119,6 @@ public class AuthorizationTest {
}
private final TestContext ctx_;
- private final TQueryCtx queryCtx_;
private final AnalysisContext analysisContext_;
private final Frontend fe_;
@@ -177,8 +172,7 @@ public class AuthorizationTest {
public AuthorizationTest(TestContext ctx) throws Exception {
ctx_ = ctx;
- queryCtx_ = TestUtils.createQueryContext(Catalog.DEFAULT_DB, USER.getName());
- analysisContext_ = new AnalysisContext(ctx_.catalog, queryCtx_, ctx_.authzConfig);
+ analysisContext_ = createAnalysisCtx(ctx_.authzConfig, USER.getName());
fe_ = new Frontend(ctx_.authzConfig, ctx_.catalog);
}
@@ -1860,17 +1854,14 @@ public class AuthorizationTest {
new User(USER.getName() + "/abc.host.com@REAL.COM"),
new User(USER.getName() + "@REAL.COM"));
for (User user: users) {
- AnalysisContext context = new AnalysisContext(ctx_.catalog,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB, user.getName()),
- ctx_.authzConfig);
+ AnalysisContext ctx = createAnalysisCtx(ctx_.authzConfig, user.getName());
// Can select from table that user has privileges on.
- AuthzOk(context, "select * from functional.alltypesagg");
+ AuthzOk(ctx, "select * from functional.alltypesagg");
// Unqualified table name.
- AuthzError(context, "select * from alltypes",
- "User '%s' does not have privileges to execute 'SELECT' on: default.alltypes",
- user);
+ AuthzError(ctx, "select * from alltypes",
+ "User '%s' does not have privileges to execute 'SELECT' on: default.alltypes");
}
// If the first character is '/', the short username should be the same as
// the full username.
@@ -1904,16 +1895,14 @@ public class AuthorizationTest {
User.setRulesForTesting(
new Configuration().get(HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT"));
User user = new User("authtest/hostname@REALM.COM");
- AnalysisContext context = new AnalysisContext(catalog,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB, user.getName()), authzConfig);
+ AnalysisContext ctx = createAnalysisCtx(authzConfig, user.getName());
Frontend fe = new Frontend(authzConfig, catalog);
// Can select from table that user has privileges on.
- AuthzOk(fe, context, "select * from tpcds.customer");
+ AuthzOk(fe, ctx, "select * from tpcds.customer");
// Does not have privileges to execute a query
- AuthzError(fe, context, "select * from functional.alltypes",
- "User '%s' does not have privileges to execute 'SELECT' on: functional.alltypes",
- user);
+ AuthzError(fe, ctx, "select * from functional.alltypes",
+ "User '%s' does not have privileges to execute 'SELECT' on: functional.alltypes");
// Unit tests for User#getShortName()
// Different auth_to_local rules to apply on the username.
@@ -1951,31 +1940,28 @@ public class AuthorizationTest {
@Test
public void TestFunction() throws Exception {
// First try with the less privileged user.
- User currentUser = USER;
- AnalysisContext context = new AnalysisContext(ctx_.catalog,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB, currentUser.getName()),
- ctx_.authzConfig);
- AuthzError(context, "show functions",
- "User '%s' does not have privileges to access: default", currentUser);
- AuthzOk(context, "show functions in tpch");
-
- AuthzError(context, "create function f() returns int location " +
+ AnalysisContext ctx = createAnalysisCtx(ctx_.authzConfig, USER.getName());
+ AuthzError(ctx, "show functions",
+ "User '%s' does not have privileges to access: default");
+ AuthzOk(ctx, "show functions in tpch");
+
+ AuthzError(ctx, "create function f() returns int location " +
"'/test-warehouse/libTestUdfs.so' symbol='NoArgs'",
- "User '%s' does not have privileges to CREATE/DROP functions.", currentUser);
+ "User '%s' does not have privileges to CREATE/DROP functions.");
- AuthzError(context, "create function tpch.f() returns int location " +
+ AuthzError(ctx, "create function tpch.f() returns int location " +
"'/test-warehouse/libTestUdfs.so' symbol='NoArgs'",
- "User '%s' does not have privileges to CREATE/DROP functions.", currentUser);
+ "User '%s' does not have privileges to CREATE/DROP functions.");
- AuthzError(context, "create function notdb.f() returns int location " +
+ AuthzError(ctx, "create function notdb.f() returns int location " +
"'/test-warehouse/libTestUdfs.so' symbol='NoArgs'",
- "User '%s' does not have privileges to CREATE/DROP functions.", currentUser);
+ "User '%s' does not have privileges to CREATE/DROP functions.");
- AuthzError(context, "drop function if exists f()",
- "User '%s' does not have privileges to CREATE/DROP functions.", currentUser);
+ AuthzError(ctx, "drop function if exists f()",
+ "User '%s' does not have privileges to CREATE/DROP functions.");
- AuthzError(context, "drop function notdb.f()",
- "User '%s' does not have privileges to CREATE/DROP functions.", currentUser);
+ AuthzError(ctx, "drop function notdb.f()",
+ "User '%s' does not have privileges to CREATE/DROP functions.");
// TODO: Add test support for dynamically changing privileges for
// file-based policy.
@@ -2017,9 +2003,9 @@ public class AuthorizationTest {
sentryService.revokeRoleFromGroup(USER, "admin", USER.getName());
ctx_.catalog.reset();
- AuthzError(context, "create function tpch.f() returns int location " +
+ AuthzError(ctx, "create function tpch.f() returns int location " +
"'/test-warehouse/libTestUdfs.so' symbol='NoArgs'",
- "User '%s' does not have privileges to CREATE/DROP functions.", currentUser);
+ "User '%s' does not have privileges to CREATE/DROP functions.");
// Couldn't create tpch.f() but can run it.
AuthzOk("select tpch.f()");
@@ -2036,8 +2022,7 @@ public class AuthorizationTest {
}
@Test
- public void TestServerNameAuthorized()
- throws AnalysisException, InternalException {
+ public void TestServerNameAuthorized() throws ImpalaException {
if (ctx_.authzConfig.isFileBasedPolicy()) {
// Authorization config that has a different server name from policy file.
TestWithIncorrectConfig(AuthorizationConfig.createHadoopGroupAuthConfig(
@@ -2047,8 +2032,7 @@ public class AuthorizationTest {
}
@Test
- public void TestNoPermissionsWhenPolicyFileDoesNotExist()
- throws AnalysisException, InternalException {
+ public void TestNoPermissionsWhenPolicyFileDoesNotExist() throws ImpalaException {
// Test doesn't make sense except for file based policies.
if (!ctx_.authzConfig.isFileBasedPolicy()) return;
@@ -2172,47 +2156,42 @@ public class AuthorizationTest {
// Create an analysis context + FE with the test user (as defined in the policy file)
User user = new User("test_user");
- AnalysisContext context = new AnalysisContext(catalog,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB, user.getName()), authzConfig);
+ AnalysisContext ctx = createAnalysisCtx(authzConfig, user.getName());
Frontend fe = new Frontend(authzConfig, catalog);
// Can select from table that user has privileges on.
- AuthzOk(fe, context, "select * from functional.alltypesagg");
+ AuthzOk(fe, ctx, "select * from functional.alltypesagg");
// Does not have privileges to execute a query
- AuthzError(fe, context, "select * from functional.alltypes",
- "User '%s' does not have privileges to execute 'SELECT' on: functional.alltypes",
- user);
+ AuthzError(fe, ctx, "select * from functional.alltypes",
+ "User '%s' does not have privileges to execute 'SELECT' on: functional.alltypes");
// Verify with the admin user
user = new User("admin_user");
- context = new AnalysisContext(catalog,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB, user.getName()), authzConfig);
+ ctx = createAnalysisCtx(authzConfig, user.getName());
fe = new Frontend(authzConfig, catalog);
// Admin user should have privileges to do anything
- AuthzOk(fe, context, "select * from functional.alltypesagg");
- AuthzOk(fe, context, "select * from functional.alltypes");
- AuthzOk(fe, context, "invalidate metadata");
- AuthzOk(fe, context, "create external table tpch.kudu_tbl stored as kudu " +
+ AuthzOk(fe, ctx, "select * from functional.alltypesagg");
+ AuthzOk(fe, ctx, "select * from functional.alltypes");
+ AuthzOk(fe, ctx, "invalidate metadata");
+ AuthzOk(fe, ctx, "create external table tpch.kudu_tbl stored as kudu " +
"TBLPROPERTIES ('kudu.master_addresses'='127.0.0.1', 'kudu.table_name'='tbl')");
}
private void TestWithIncorrectConfig(AuthorizationConfig authzConfig, User user)
- throws AnalysisException, InternalException {
+ throws ImpalaException {
Frontend fe = new Frontend(authzConfig, ctx_.catalog);
- AnalysisContext ac = new AnalysisContext(new ImpaladTestCatalog(),
- TestUtils.createQueryContext(Catalog.DEFAULT_DB, user.getName()), authzConfig);
- AuthzError(fe, ac, "select * from functional.alltypesagg",
+ AnalysisContext ctx = createAnalysisCtx(authzConfig, user.getName());
+ AuthzError(fe, ctx, "select * from functional.alltypesagg",
"User '%s' does not have privileges to execute 'SELECT' on: " +
- "functional.alltypesagg", user);
- AuthzError(fe, ac, "ALTER TABLE functional_seq_snap.alltypes ADD COLUMNS (c1 int)",
+ "functional.alltypesagg");
+ AuthzError(fe, ctx, "ALTER TABLE functional_seq_snap.alltypes ADD COLUMNS (c1 int)",
"User '%s' does not have privileges to execute 'ALTER' on: " +
- "functional_seq_snap.alltypes", user);
- AuthzError(fe, ac, "drop table tpch.lineitem",
- "User '%s' does not have privileges to execute 'DROP' on: tpch.lineitem",
- user);
- AuthzError(fe, ac, "show tables in functional",
- "User '%s' does not have privileges to access: functional.*", user);
+ "functional_seq_snap.alltypes");
+ AuthzError(fe, ctx, "drop table tpch.lineitem",
+ "User '%s' does not have privileges to execute 'DROP' on: tpch.lineitem");
+ AuthzError(fe, ctx, "show tables in functional",
+ "User '%s' does not have privileges to access: functional.*");
}
private void AuthzOk(String stmt) throws ImpalaException {
@@ -2223,10 +2202,9 @@ public class AuthorizationTest {
AuthzOk(fe_, context, stmt);
}
- private static void AuthzOk(Frontend fe, AnalysisContext context, String stmt)
+ private void AuthzOk(Frontend fe, AnalysisContext context, String stmt)
throws ImpalaException {
- context.analyze(stmt);
- context.authorize(fe.getAuthzChecker());
+ parseAndAnalyze(stmt, context, fe);
}
/**
@@ -2234,29 +2212,24 @@ public class AuthorizationTest {
* string matches.
*/
private void AuthzError(String stmt, String expectedErrorString)
- throws AnalysisException, InternalException {
- AuthzError(analysisContext_, stmt, expectedErrorString, USER);
+ throws ImpalaException {
+ AuthzError(analysisContext_, stmt, expectedErrorString);
}
- private void AuthzError(AnalysisContext analysisContext,
- String stmt, String expectedErrorString, User user)
- throws AnalysisException, InternalException {
- AuthzError(fe_, analysisContext, stmt, expectedErrorString, user);
+ private void AuthzError(AnalysisContext ctx, String stmt, String expectedErrorString)
+ throws ImpalaException {
+ AuthzError(fe_, ctx, stmt, expectedErrorString);
}
- private static void AuthzError(Frontend fe, AnalysisContext analysisContext,
- String stmt, String expectedErrorString, User user)
- throws AnalysisException, InternalException {
+ private void AuthzError(Frontend fe, AnalysisContext ctx,
+ String stmt, String expectedErrorString)
+ throws ImpalaException {
Preconditions.checkNotNull(expectedErrorString);
try {
- try {
- analysisContext.analyze(stmt);
- } finally {
- analysisContext.authorize(fe.getAuthzChecker());
- }
+ parseAndAnalyze(stmt, ctx, fe);
} catch (AuthorizationException e) {
// Insert the username into the error.
- expectedErrorString = String.format(expectedErrorString, user.getName());
+ expectedErrorString = String.format(expectedErrorString, ctx.getUser());
String errorString = e.getMessage();
Assert.assertTrue(
"got error:\n" + errorString + "\nexpected:\n" + expectedErrorString,
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/ExprNdvTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ExprNdvTest.java b/fe/src/test/java/org/apache/impala/analysis/ExprNdvTest.java
index e49347e..88e1160 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ExprNdvTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ExprNdvTest.java
@@ -17,9 +17,9 @@
package org.apache.impala.analysis;
-import org.apache.impala.catalog.Catalog;
-import org.apache.impala.common.AnalysisException;
+import org.apache.impala.analysis.AnalysisContext.AnalysisResult;
import org.apache.impala.common.FrontendTestBase;
+import org.apache.impala.common.ImpalaException;
import org.junit.Assert;
import org.junit.Test;
@@ -29,7 +29,7 @@ import org.junit.Test;
public class ExprNdvTest extends FrontendTestBase {
public void verifyNdv(String expr, long expectedNdv)
- throws AnalysisException {
+ throws ImpalaException {
String stmtStr = "select " + expr + " from functional.alltypes";
verifyNdvStmt(stmtStr, expectedNdv);
}
@@ -40,18 +40,17 @@ public class ExprNdvTest extends FrontendTestBase {
* functional.tinytable (tiny) does not
*/
public void verifyNdvTwoTable(String expr, long expectedNdv)
- throws AnalysisException {
+ throws ImpalaException {
String stmtStr = "select " + expr + " from functional.alltypes a, " +
"functional.tinytable tiny";
verifyNdvStmt(stmtStr, expectedNdv);
}
- public void verifyNdvStmt(String stmtStr, long expectedNdv)
- throws AnalysisException {
- SelectStmt stmt = (SelectStmt) ParsesOk(stmtStr);
- Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
- stmt.analyze(analyzer);
- Expr analyzedExpr = stmt.getSelectList().getItems().get(0).getExpr();
+ public void verifyNdvStmt(String stmt, long expectedNdv) throws ImpalaException {
+ AnalysisContext ctx = createAnalysisCtx();
+ AnalysisResult result = parseAndAnalyze(stmt, ctx);
+ SelectStmt parsedStmt = (SelectStmt) result.getStmt();
+ Expr analyzedExpr = parsedStmt.getSelectList().getItems().get(0).getExpr();
long calculatedNdv = analyzedExpr.getNumDistinctValues();
assertEquals(expectedNdv, calculatedNdv);
}
@@ -66,7 +65,7 @@ public class ExprNdvTest extends FrontendTestBase {
}
@Test
- public void TestCaseExprBasic() throws AnalysisException {
+ public void TestCaseExprBasic() throws ImpalaException {
// All constants tests
verifyNdv("case when id = 1 then 'yes' else 'no' end", 2);
verifyNdv("case when id = 1 then 'yes' " +
@@ -92,7 +91,7 @@ public class ExprNdvTest extends FrontendTestBase {
}
@Test
- public void TestCaseExprMissingStats() throws AnalysisException {
+ public void TestCaseExprMissingStats() throws ImpalaException {
// Consts still work
verifyNdvTwoTable("case when a.id = 1 then 'yes' " +
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java b/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
index fa5bc05..209e69d 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ExprRewriteRulesTest.java
@@ -19,12 +19,10 @@ package org.apache.impala.analysis;
import java.util.List;
-import org.apache.impala.catalog.Catalog;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FrontendTestBase;
+import org.apache.impala.common.ImpalaException;
import org.apache.impala.rewrite.BetweenToCompoundRule;
-import org.apache.impala.rewrite.RemoveRedundantStringCast;
-import org.apache.impala.rewrite.SimplifyDistinctFromRule;
import org.apache.impala.rewrite.EqualityDisjunctsToInRule;
import org.apache.impala.rewrite.ExprRewriteRule;
import org.apache.impala.rewrite.ExprRewriter;
@@ -33,7 +31,9 @@ import org.apache.impala.rewrite.FoldConstantsRule;
import org.apache.impala.rewrite.NormalizeBinaryPredicatesRule;
import org.apache.impala.rewrite.NormalizeCountStarRule;
import org.apache.impala.rewrite.NormalizeExprsRule;
+import org.apache.impala.rewrite.RemoveRedundantStringCast;
import org.apache.impala.rewrite.SimplifyConditionalsRule;
+import org.apache.impala.rewrite.SimplifyDistinctFromRule;
import org.junit.Assert;
import org.junit.Test;
@@ -62,50 +62,51 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
public Expr RewritesOk(String exprStr, ExprRewriteRule rule, String expectedExprStr)
- throws AnalysisException {
+ throws ImpalaException {
return RewritesOk("functional.alltypessmall", exprStr, rule, expectedExprStr);
}
public Expr RewritesOk(String tableName, String exprStr, ExprRewriteRule rule, String expectedExprStr)
- throws AnalysisException {
+ throws ImpalaException {
return RewritesOk(tableName, exprStr, Lists.newArrayList(rule), expectedExprStr);
}
public Expr RewritesOk(String exprStr, List<ExprRewriteRule> rules, String expectedExprStr)
- throws AnalysisException {
+ throws ImpalaException {
return RewritesOk("functional.alltypessmall", exprStr, rules, expectedExprStr);
}
public Expr RewritesOk(String tableName, String exprStr, List<ExprRewriteRule> rules,
- String expectedExprStr) throws AnalysisException {
+ String expectedExprStr) throws ImpalaException {
String stmtStr = "select " + exprStr + " from " + tableName;
+ // Analyze without rewrites since that's what we want to test here.
SelectStmt stmt = (SelectStmt) ParsesOk(stmtStr);
- Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
- stmt.analyze(analyzer);
+ AnalyzesOkNoRewrite(stmt);
Expr origExpr = stmt.getSelectList().getItems().get(0).getExpr();
- Expr rewrittenExpr = verifyExprEquivalence(origExpr, expectedExprStr, rules, analyzer);
+ Expr rewrittenExpr =
+ verifyExprEquivalence(origExpr, expectedExprStr, rules, stmt.getAnalyzer());
return rewrittenExpr;
}
public Expr RewritesOkWhereExpr(String exprStr, ExprRewriteRule rule, String expectedExprStr)
- throws AnalysisException {
+ throws ImpalaException {
return RewritesOkWhereExpr("functional.alltypessmall", exprStr, rule, expectedExprStr);
}
public Expr RewritesOkWhereExpr(String tableName, String exprStr, ExprRewriteRule rule, String expectedExprStr)
- throws AnalysisException {
+ throws ImpalaException {
return RewritesOkWhereExpr(tableName, exprStr, Lists.newArrayList(rule), expectedExprStr);
}
public Expr RewritesOkWhereExpr(String tableName, String exprStr, List<ExprRewriteRule> rules,
- String expectedExprStr) throws AnalysisException {
+ String expectedExprStr) throws ImpalaException {
String stmtStr = "select count(1) from " + tableName + " where " + exprStr;
- System.out.println(stmtStr);
+ // Analyze without rewrites since that's what we want to test here.
SelectStmt stmt = (SelectStmt) ParsesOk(stmtStr);
- Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
- stmt.analyze(analyzer);
+ AnalyzesOkNoRewrite(stmt);
Expr origExpr = stmt.getWhereClause();
- Expr rewrittenExpr = verifyExprEquivalence(origExpr, expectedExprStr, rules, analyzer);
+ Expr rewrittenExpr =
+ verifyExprEquivalence(origExpr, expectedExprStr, rules, stmt.getAnalyzer());
return rewrittenExpr;
}
@@ -151,7 +152,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestBetweenToCompoundRule() throws AnalysisException {
+ public void TestBetweenToCompoundRule() throws ImpalaException {
ExprRewriteRule rule = BetweenToCompoundRule.INSTANCE;
// Basic BETWEEN predicates.
@@ -190,7 +191,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestExtractCommonConjunctsRule() throws AnalysisException {
+ public void TestExtractCommonConjunctsRule() throws ImpalaException {
ExprRewriteRule rule = ExtractCommonConjunctRule.INSTANCE;
// One common conjunct: int_col < 10
@@ -285,7 +286,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
* testing is done in expr-test.cc.
*/
@Test
- public void TestFoldConstantsRule() throws AnalysisException {
+ public void TestFoldConstantsRule() throws ImpalaException {
ExprRewriteRule rule = FoldConstantsRule.INSTANCE;
RewritesOk("1 + 1", rule, "2");
@@ -312,7 +313,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestSimplifyConditionalsRule() throws AnalysisException {
+ public void TestSimplifyConditionalsRule() throws ImpalaException {
ExprRewriteRule rule = SimplifyConditionalsRule.INSTANCE;
// IF
@@ -448,7 +449,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestNormalizeExprsRule() throws AnalysisException {
+ public void TestNormalizeExprsRule() throws ImpalaException {
ExprRewriteRule rule = NormalizeExprsRule.INSTANCE;
// CompoundPredicate
@@ -461,7 +462,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestNormalizeBinaryPredicatesRule() throws AnalysisException {
+ public void TestNormalizeBinaryPredicatesRule() throws ImpalaException {
ExprRewriteRule rule = NormalizeBinaryPredicatesRule.INSTANCE;
RewritesOk("0 = id", rule, "id = 0");
@@ -483,7 +484,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestEqualityDisjunctsToInRule() throws AnalysisException {
+ public void TestEqualityDisjunctsToInRule() throws ImpalaException {
ExprRewriteRule edToInrule = EqualityDisjunctsToInRule.INSTANCE;
ExprRewriteRule normalizeRule = NormalizeBinaryPredicatesRule.INSTANCE;
List<ExprRewriteRule> comboRules = Lists.newArrayList(normalizeRule,
@@ -546,7 +547,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestNormalizeCountStarRule() throws AnalysisException {
+ public void TestNormalizeCountStarRule() throws ImpalaException {
ExprRewriteRule rule = NormalizeCountStarRule.INSTANCE;
RewritesOk("count(1)", rule, "count(*)");
@@ -560,7 +561,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestSimplifyDistinctFromRule() throws AnalysisException {
+ public void TestSimplifyDistinctFromRule() throws ImpalaException {
ExprRewriteRule rule = SimplifyDistinctFromRule.INSTANCE;
// Can be simplified
@@ -583,7 +584,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
}
@Test
- public void TestRemoveRedundantStringCastRule() throws AnalysisException {
+ public void TestRemoveRedundantStringCastRule() throws ImpalaException {
ExprRewriteRule removeRule = RemoveRedundantStringCast.INSTANCE;
ExprRewriteRule foldConstantRule = FoldConstantsRule.INSTANCE;
List<ExprRewriteRule> comboRules = Lists.newArrayList(removeRule, foldConstantRule);
@@ -677,7 +678,7 @@ public class ExprRewriteRulesTest extends FrontendTestBase {
* it can be further simplified via SimplifyDistinctFromRule.
*/
@Test
- public void TestNullif() throws AnalysisException {
+ public void TestNullif() throws ImpalaException {
List<ExprRewriteRule> rules = Lists.newArrayList(
SimplifyConditionalsRule.INSTANCE,
SimplifyDistinctFromRule.INSTANCE);
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java b/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
index 310b7e4..bebc6ba 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ExprRewriterTest.java
@@ -17,13 +17,12 @@
package org.apache.impala.analysis;
-import org.apache.impala.authorization.AuthorizationConfig;
-import org.apache.impala.catalog.Catalog;
+import org.apache.impala.analysis.AnalysisContext.AnalysisResult;
import org.apache.impala.common.AnalysisException;
+import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.rewrite.ExprRewriteRule;
import org.apache.impala.rewrite.ExprRewriter;
-import org.apache.impala.testutil.TestUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -78,10 +77,10 @@ public class ExprRewriterTest extends AnalyzerTest {
* number of expressions and complicate validation.
*/
public void RewritesOk(String stmt, int expectedNumChanges,
- int expectedNumExprTrees) throws AnalysisException {
+ int expectedNumExprTrees) throws ImpalaException {
+ // Analyze without rewrites since that's what we want to test here.
StatementBase parsedStmt = (StatementBase) ParsesOk(stmt);
- parsedStmt.analyze(createAnalyzer(Catalog.DEFAULT_DB));
-
+ AnalyzesOkNoRewrite(parsedStmt);
exprToTrue_.reset();
parsedStmt.rewriteExprs(exprToTrue_);
Assert.assertEquals(expectedNumChanges, exprToTrue_.getNumChanges());
@@ -93,21 +92,17 @@ public class ExprRewriterTest extends AnalyzerTest {
// Make sure the stmt can be successfully re-analyzed.
parsedStmt.reset();
- parsedStmt.analyze(createAnalyzer(Catalog.DEFAULT_DB));
+ AnalyzesOkNoRewrite(parsedStmt);
}
/**
* Asserts that no rewrites are performed on the given stmt.
*/
- public void CheckNoRewrite(String stmt) throws AnalysisException {
+ public void CheckNoRewrite(String stmt) throws ImpalaException {
exprToTrue_.reset();
- Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
- AnalysisContext analysisCtx = new AnalysisContext(catalog_,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB,
- System.getProperty("user.name")),
- AuthorizationConfig.createAuthDisabledConfig(), exprToTrue_);
- analysisCtx.analyze(stmt, analyzer);
- Preconditions.checkNotNull(analysisCtx.getAnalysisResult().getStmt());
+ AnalysisContext analysisCtx = createAnalysisCtx();
+ AnalysisResult result = parseAndAnalyze(stmt, analysisCtx);
+ Preconditions.checkNotNull(result.getStmt());
Assert.assertEquals(0, exprToTrue_.getNumChanges());
}
@@ -122,7 +117,7 @@ public class ExprRewriterTest extends AnalyzerTest {
"order by a.int_col, 4 limit 10";
@Test
- public void TestQueryStmts() throws AnalysisException {
+ public void TestQueryStmts() throws ImpalaException {
RewritesOk(stmt_, 23, 11);
// Test rewriting in inline views. The view stmt is the same as the query above
// but with an order by + limit. Expanded star exprs are not rewritten.
@@ -146,7 +141,7 @@ public class ExprRewriterTest extends AnalyzerTest {
}
@Test
- public void TestDdlStmts() throws AnalysisException {
+ public void TestDdlStmts() throws ImpalaException {
RewritesOk("create table ctas_test as " + stmt_, 23, 11);
// Create/alter view stmts are not rewritten to preserve the original SQL.
CheckNoRewrite("create view view_test as " + stmt_);
@@ -154,7 +149,7 @@ public class ExprRewriterTest extends AnalyzerTest {
}
@Test
- public void TestDmlStmts() throws AnalysisException {
+ public void TestDmlStmts() throws ImpalaException {
// Insert.
RewritesOk("insert into functional.alltypes (id, int_col, float_col, bigint_col) " +
"partition(year=2009,month=10) " + stmt_, 23, 11);
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
new file mode 100644
index 0000000..f2c8faa
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.analysis;
+
+import java.util.Arrays;
+
+import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache;
+import org.apache.impala.authorization.AuthorizationConfig;
+import org.apache.impala.catalog.Catalog;
+import org.apache.impala.catalog.Table;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.common.InternalException;
+import org.apache.impala.service.Frontend;
+import org.apache.impala.testutil.ImpaladTestCatalog;
+import org.apache.impala.util.EventSequence;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StmtMetadataLoaderTest {
+
+ private void testLoadTables(String stmtStr,
+ int expectedNumLoadRequests, int expectedNumCatalogUpdates,
+ String[] expectedDbs, String[] expectedTables)
+ throws ImpalaException {
+ ImpaladTestCatalog catalog = new ImpaladTestCatalog();
+ Frontend fe = new Frontend(AuthorizationConfig.createAuthDisabledConfig(), catalog);
+ StatementBase stmt = fe.parse(stmtStr);
+ // Catalog is fresh and no tables are cached.
+ validateUncached(stmt, fe, expectedNumLoadRequests, expectedNumCatalogUpdates,
+ expectedDbs, expectedTables);
+ // All relevant tables should be cached now.
+ validateCached(stmt, fe, expectedDbs, expectedTables);
+ }
+
+ private void validateDbs(StmtTableCache stmtTableCache, String[] expectedDbs) {
+ String[] actualDbs = new String[stmtTableCache.dbs.size()];
+ actualDbs = stmtTableCache.dbs.toArray(actualDbs);
+ Arrays.sort(expectedDbs);
+ Arrays.sort(actualDbs);
+ Assert.assertArrayEquals(expectedDbs, actualDbs);
+ }
+
+ private void validateTables(StmtTableCache stmtTableCache, String[] expectedTables) {
+ String[] actualTables = new String[stmtTableCache.tables.size()];
+ int idx = 0;
+ for (Table t: stmtTableCache.tables.values()) {
+ Assert.assertTrue(t.isLoaded());
+ actualTables[idx++] = t.getFullName();
+ }
+ Arrays.sort(expectedTables);
+ Arrays.sort(actualTables);
+ Assert.assertArrayEquals(expectedTables, actualTables);
+ }
+
+ private void validateUncached(StatementBase stmt, Frontend fe,
+ int expectedNumLoadRequests, int expectedNumCatalogUpdates,
+ String[] expectedDbs, String[] expectedTables) throws InternalException {
+ EventSequence timeline = new EventSequence("Test Timeline");
+ StmtMetadataLoader mdLoader =
+ new StmtMetadataLoader(fe, Catalog.DEFAULT_DB, timeline);
+ StmtTableCache stmtTableCache = mdLoader.loadTables(stmt);
+ // Validate metrics.
+ Assert.assertEquals(expectedNumLoadRequests,
+ mdLoader.getNumLoadRequestsSent());
+ Assert.assertEquals(expectedNumCatalogUpdates,
+ mdLoader.getNumCatalogUpdatesReceived());
+ // Validate timeline.
+ Assert.assertEquals(2, mdLoader.getTimeline().getNumEvents());
+ // Validate dbs and tables.
+ validateDbs(stmtTableCache, expectedDbs);
+ validateTables(stmtTableCache, expectedTables);
+ }
+
+ private void validateCached(StatementBase stmt, Frontend fe,
+ String[] expectedDbs, String[] expectedTables) throws InternalException {
+ EventSequence timeline = new EventSequence("Test Timeline");
+ StmtMetadataLoader mdLoader =
+ new StmtMetadataLoader(fe, Catalog.DEFAULT_DB, timeline);
+ StmtTableCache stmtTableCache = mdLoader.loadTables(stmt);
+ // Validate metrics. Expect all tables to already be in the cache.
+ Assert.assertEquals(0, mdLoader.getNumLoadRequestsSent());
+ Assert.assertEquals(0, mdLoader.getNumCatalogUpdatesReceived());
+ // Validate timeline. Expect a single "everything is cached" event.
+ Assert.assertEquals(1, mdLoader.getTimeline().getNumEvents());
+ // Validate dbs and tables.
+ validateDbs(stmtTableCache, expectedDbs);
+ validateTables(stmtTableCache, expectedTables);
+ }
+
+ @Test
+ public void testSingleLoadRequest() throws ImpalaException {
+ // Single query block.
+ testLoadTables("select * from functional.alltypes", 1, 1,
+ new String[] {"default", "functional"},
+ new String[] {"functional.alltypes"});
+ // Single query block, multiple dbs and tables.
+ testLoadTables("select * from functional.alltypes, functional_parquet.alltypes, " +
+ "functional_avro.alltypes", 1, 1,
+ new String[] {"default", "functional", "functional_parquet", "functional_avro"},
+ new String[] {"functional.alltypes", "functional_parquet.alltypes",
+ "functional_avro.alltypes"});
+ // Single query block, test deduplication.
+ testLoadTables("select * from functional.alltypes, functional.alltypes, " +
+ "functional.alltypes", 1, 1,
+ new String[] {"default", "functional"},
+ new String[] {"functional.alltypes"});
+ // Multiple query blocks, multiple dbs and tables.
+ testLoadTables("with w as (select id from functional.alltypes) " +
+ "select * from w, (select id from functional.alltypessmall) v " +
+ "where v.id in (select id from functional.alltypestiny)", 1, 1,
+ new String[] {"default", "functional"},
+ new String[] {"functional.alltypes", "functional.alltypessmall",
+ "functional.alltypestiny"});
+ testLoadTables("select * from functional.alltypes union distinct " +
+ "select * from functional.alltypessmall union all " +
+ "select * from functional.alltypestiny", 1, 1,
+ new String[] {"default", "functional"},
+ new String[] {"functional.alltypes", "functional.alltypessmall",
+ "functional.alltypestiny"});
+ // Multiple query blocks, test deduplication.
+ testLoadTables("with w as (select id from functional.alltypes) " +
+ "select * from w, (select id from functional.alltypes) v " +
+ "where v.id in (select id from functional.alltypes)", 1, 1,
+ new String[] {"default", "functional"},
+ new String[] {"functional.alltypes"});
+ testLoadTables("select * from functional.alltypes union distinct " +
+ "select * from functional.alltypes union all " +
+ "select * from functional.alltypes", 1, 1,
+ new String[] {"default", "functional"},
+ new String[] {"functional.alltypes"});
+ }
+
+ @Test
+ public void testViewExpansion() throws ImpalaException {
+ // Test views:
+ // functional.alltypes_view references functional.alltypes
+ // functional.view_view references functional.alltypes_view
+ testLoadTables("select * from functional.alltypes_view", 2, 2,
+ new String[] {"default", "functional"},
+ new String[] {"functional.alltypes_view", "functional.alltypes"});
+ testLoadTables("select * from functional.view_view", 3, 3,
+ new String[] {"default", "functional"},
+ new String[] {"functional.view_view", "functional.alltypes_view",
+ "functional.alltypes"});
+ // Test deduplication.
+ testLoadTables("select * from functional.view_view, functional.view_view", 3, 3,
+ new String[] {"default", "functional"},
+ new String[] {"functional.view_view", "functional.alltypes_view",
+ "functional.alltypes"});
+ testLoadTables("select * from functional.alltypes, functional.view_view", 2, 2,
+ new String[] {"default", "functional"},
+ new String[] {"functional.view_view", "functional.alltypes_view",
+ "functional.alltypes"});
+ testLoadTables("select * from functional.alltypes_view, functional.view_view", 2, 2,
+ new String[] {"default", "functional"},
+ new String[] {"functional.view_view", "functional.alltypes_view",
+ "functional.alltypes"});
+ // All tables nested in views are also referenced at top level.
+ testLoadTables("select * from functional.alltypes, functional.alltypes_view, " +
+ "functional.view_view", 1, 1,
+ new String[] {"default", "functional"},
+ new String[] {"functional.view_view", "functional.alltypes_view",
+ "functional.alltypes"});
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
index 41124f5..a813cce 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
@@ -19,7 +19,6 @@ package org.apache.impala.analysis;
import static org.junit.Assert.fail;
-import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.testutil.TestUtils;
@@ -67,22 +66,6 @@ public class ToSqlTest extends FrontendTestBase {
}
}
- private static AnalysisContext.AnalysisResult analyze(String query, String defaultDb) {
- try {
- AnalysisContext analysisCtx = new AnalysisContext(catalog_,
- TestUtils.createQueryContext(defaultDb, System.getProperty("user.name")),
- AuthorizationConfig.createAuthDisabledConfig());
- analysisCtx.analyze(query);
- AnalysisContext.AnalysisResult analysisResult = analysisCtx.getAnalysisResult();
- Preconditions.checkNotNull(analysisResult.getStmt());
- return analysisResult;
- } catch (Exception e) {
- e.printStackTrace();
- fail("Failed to analyze query: " + query + "\n" + e.getMessage());
- }
- return null;
- }
-
private void testToSql(String query, String expected) {
testToSql(query, System.getProperty("user.name"), expected);
}
@@ -95,7 +78,7 @@ public class ToSqlTest extends FrontendTestBase {
boolean ignoreWhitespace) {
String actual = null;
try {
- ParseNode node = AnalyzesOk(query, createAnalyzer(defaultDb));
+ ParseNode node = AnalyzesOk(query, createAnalysisCtx(defaultDb));
actual = node.toSql();
if (ignoreWhitespace) {
// Transform whitespace to single space.
@@ -112,7 +95,7 @@ public class ToSqlTest extends FrontendTestBase {
fail("Failed to analyze query: " + query + "\n" + e.getMessage());
}
// Parse and analyze the resulting SQL to ensure its validity.
- AnalyzesOk(actual, createAnalyzer(defaultDb));
+ AnalyzesOk(actual, createAnalysisCtx(defaultDb));
}
private void runTestTemplate(String sql, String expectedSql, String[]... testDims) {
@@ -151,7 +134,7 @@ public class ToSqlTest extends FrontendTestBase {
Preconditions.checkState(query.contains("$TBL"));
String uqQuery = query.replace("$TBL", tbl.getTbl());
testToSql(uqQuery, tbl.getDb(), expectedSql);
- AnalyzesOk(uqQuery, createAnalyzer(tbl.getDb()));
+ AnalyzesOk(uqQuery, createAnalysisCtx(tbl.getDb()));
String fqQuery = query.replace("$TBL", tbl.toString());
testToSql(fqQuery, expectedSql);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
index c087f5a..9be3343 100644
--- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
+++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.impala.analysis.AnalysisContext;
+import org.apache.impala.analysis.AnalysisContext.AnalysisResult;
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.ColumnDef;
import org.apache.impala.analysis.CreateTableStmt;
@@ -36,6 +37,9 @@ import org.apache.impala.analysis.ParseNode;
import org.apache.impala.analysis.QueryStmt;
import org.apache.impala.analysis.SqlParser;
import org.apache.impala.analysis.SqlScanner;
+import org.apache.impala.analysis.StatementBase;
+import org.apache.impala.analysis.StmtMetadataLoader;
+import org.apache.impala.analysis.StmtMetadataLoader.StmtTableCache;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.catalog.AggregateFunction;
import org.apache.impala.catalog.Catalog;
@@ -44,7 +48,6 @@ import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.HdfsTable;
-import org.apache.impala.catalog.ImpaladCatalog;
import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.ScalarFunction;
import org.apache.impala.catalog.ScalarType;
@@ -58,6 +61,7 @@ import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TFunctionBinaryType;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TQueryOptions;
+import org.apache.impala.util.EventSequence;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -73,7 +77,7 @@ import com.google.common.collect.Lists;
* as well as helper functions for creating test-local tables/views and UDF/UDAs.
*/
public class FrontendTestBase {
- protected static ImpaladCatalog catalog_ = new ImpaladTestCatalog();
+ protected static ImpaladTestCatalog catalog_ = new ImpaladTestCatalog();
protected static Frontend frontend_ = new Frontend(
AuthorizationConfig.createAuthDisabledConfig(), catalog_);
@@ -91,26 +95,6 @@ public class FrontendTestBase {
RuntimeEnv.INSTANCE.setTestEnv(false);
}
- protected Analyzer createAnalyzer(String defaultDb) {
- TQueryCtx queryCtx =
- TestUtils.createQueryContext(defaultDb, System.getProperty("user.name"));
- return new Analyzer(catalog_, queryCtx,
- AuthorizationConfig.createAuthDisabledConfig());
- }
-
- protected Analyzer createAnalyzer(TQueryOptions queryOptions) {
- TQueryCtx queryCtx = TestUtils.createQueryContext();
- queryCtx.client_request.query_options = queryOptions;
- return new Analyzer(catalog_, queryCtx,
- AuthorizationConfig.createAuthDisabledConfig());
- }
-
- protected Analyzer createAnalyzerUsingHiveColLabels() {
- Analyzer analyzer = createAnalyzer(Catalog.DEFAULT_DB);
- analyzer.setUseHiveColLabels(true);
- return analyzer;
- }
-
// Adds a Udf: default.name(args) to the catalog.
// TODO: we could consider having this be the sql to run instead but that requires
// connecting to the BE.
@@ -292,7 +276,11 @@ public class FrontendTestBase {
* Analyze 'stmt', expecting it to pass. Asserts in case of analysis error.
*/
public ParseNode AnalyzesOk(String stmt) {
- return AnalyzesOk(stmt, createAnalyzer(Catalog.DEFAULT_DB), null);
+ return AnalyzesOk(stmt, createAnalysisCtx(), null);
+ }
+
+ public ParseNode AnalyzesOk(String stmt, AnalysisContext analysisCtx) {
+ return AnalyzesOk(stmt, analysisCtx, null);
}
/**
@@ -300,21 +288,56 @@ public class FrontendTestBase {
* If 'expectedWarning' is not null, asserts that a warning is produced.
*/
public ParseNode AnalyzesOk(String stmt, String expectedWarning) {
- return AnalyzesOk(stmt, createAnalyzer(Catalog.DEFAULT_DB), expectedWarning);
+ return AnalyzesOk(stmt, createAnalysisCtx(), expectedWarning);
+ }
+
+ protected AnalysisContext createAnalysisCtx() {
+ return createAnalysisCtx(Catalog.DEFAULT_DB);
+ }
+
+ protected AnalysisContext createAnalysisCtx(String defaultDb) {
+ TQueryCtx queryCtx = TestUtils.createQueryContext(
+ defaultDb, System.getProperty("user.name"));
+ EventSequence timeline = new EventSequence("Frontend Test Timeline");
+ AnalysisContext analysisCtx = new AnalysisContext(queryCtx,
+ AuthorizationConfig.createAuthDisabledConfig(), timeline);
+ return analysisCtx;
+ }
+
+ protected AnalysisContext createAnalysisCtx(TQueryOptions queryOptions) {
+ TQueryCtx queryCtx = TestUtils.createQueryContext();
+ queryCtx.client_request.query_options = queryOptions;
+ EventSequence timeline = new EventSequence("Frontend Test Timeline");
+ AnalysisContext analysisCtx = new AnalysisContext(queryCtx,
+ AuthorizationConfig.createAuthDisabledConfig(), timeline);
+ return analysisCtx;
+ }
+
+ protected AnalysisContext createAnalysisCtx(AuthorizationConfig authzConfig) {
+ return createAnalysisCtx(authzConfig, System.getProperty("user.name"));
+ }
+
+ protected AnalysisContext createAnalysisCtx(AuthorizationConfig authzConfig,
+ String user) {
+ TQueryCtx queryCtx = TestUtils.createQueryContext(Catalog.DEFAULT_DB, user);
+ EventSequence timeline = new EventSequence("Frontend Test Timeline");
+ AnalysisContext analysisCtx = new AnalysisContext(queryCtx, authzConfig, timeline);
+ return analysisCtx;
+ }
+
+ protected AnalysisContext createAnalysisCtxUsingHiveColLabels() {
+ AnalysisContext analysisCtx = createAnalysisCtx();
+ analysisCtx.setUseHiveColLabels(true);
+ return analysisCtx;
}
/**
* Analyze 'stmt', expecting it to pass. Asserts in case of analysis error.
* If 'expectedWarning' is not null, asserts that a warning is produced.
*/
- public ParseNode AnalyzesOk(String stmt, Analyzer analyzer, String expectedWarning) {
+ public ParseNode AnalyzesOk(String stmt, AnalysisContext ctx, String expectedWarning) {
try {
- AnalysisContext analysisCtx = new AnalysisContext(catalog_,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB,
- System.getProperty("user.name")),
- AuthorizationConfig.createAuthDisabledConfig());
- analysisCtx.analyze(stmt, analyzer);
- AnalysisContext.AnalysisResult analysisResult = analysisCtx.getAnalysisResult();
+ AnalysisResult analysisResult = parseAndAnalyze(stmt, ctx);
if (expectedWarning != null) {
List<String> actualWarnings = analysisResult.getAnalyzer().getWarnings();
boolean matchedWarning = false;
@@ -340,17 +363,23 @@ public class FrontendTestBase {
}
/**
- * Asserts if stmt passes analysis.
+ * Analyzes the given statement without performing rewrites or authorization.
*/
- public void AnalysisError(String stmt) {
- AnalysisError(stmt, null);
+ public StatementBase AnalyzesOkNoRewrite(StatementBase stmt) throws ImpalaException {
+ AnalysisContext ctx = createAnalysisCtx();
+ StmtMetadataLoader mdLoader =
+ new StmtMetadataLoader(frontend_, ctx.getQueryCtx().session.database, null);
+ StmtTableCache loadedTables = mdLoader.loadTables(stmt);
+ Analyzer analyzer = ctx.createAnalyzer(loadedTables);
+ stmt.analyze(analyzer);
+ return stmt;
}
/**
- * Analyze 'stmt', expecting it to pass. Asserts in case of analysis error.
+ * Asserts if stmt passes analysis.
*/
- public ParseNode AnalyzesOk(String stmt, Analyzer analyzer) {
- return AnalyzesOk(stmt, analyzer, null);
+ public void AnalysisError(String stmt) {
+ AnalysisError(stmt, null);
}
/**
@@ -358,22 +387,17 @@ public class FrontendTestBase {
* is non-null.
*/
public void AnalysisError(String stmt, String expectedErrorString) {
- AnalysisError(stmt, createAnalyzer(Catalog.DEFAULT_DB), expectedErrorString);
+ AnalysisError(stmt, createAnalysisCtx(), expectedErrorString);
}
/**
* Asserts if stmt passes analysis or the error string doesn't match and it
* is non-null.
*/
- public void AnalysisError(String stmt, Analyzer analyzer, String expectedErrorString) {
+ public void AnalysisError(String stmt, AnalysisContext ctx, String expectedErrorString) {
Preconditions.checkNotNull(expectedErrorString, "No expected error message given.");
try {
- AnalysisContext analysisCtx = new AnalysisContext(catalog_,
- TestUtils.createQueryContext(Catalog.DEFAULT_DB,
- System.getProperty("user.name")),
- AuthorizationConfig.createAuthDisabledConfig());
- analysisCtx.analyze(stmt, analyzer);
- AnalysisContext.AnalysisResult analysisResult = analysisCtx.getAnalysisResult();
+ AnalysisResult analysisResult = parseAndAnalyze(stmt, ctx);
Preconditions.checkNotNull(analysisResult.getStmt());
} catch (Exception e) {
String errorString = e.getMessage();
@@ -385,4 +409,18 @@ public class FrontendTestBase {
}
fail("Stmt didn't result in analysis error: " + stmt);
}
+
+ protected AnalysisResult parseAndAnalyze(String stmt, AnalysisContext ctx)
+ throws ImpalaException {
+ return parseAndAnalyze(stmt, ctx, frontend_);
+ }
+
+ protected AnalysisResult parseAndAnalyze(String stmt, AnalysisContext ctx, Frontend fe)
+ throws ImpalaException {
+ StatementBase parsedStmt = fe.parse(stmt);
+ StmtMetadataLoader mdLoader =
+ new StmtMetadataLoader(fe, ctx.getQueryCtx().session.database, null);
+ StmtTableCache stmtTableCache = mdLoader.loadTables(parsedStmt);
+ return ctx.analyzeAndAuthorize(parsedStmt, stmtTableCache, fe.getAuthzChecker());
+ }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/service/FrontendTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/service/FrontendTest.java b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
index c87882d..de8b7ac 100644
--- a/fe/src/test/java/org/apache/impala/service/FrontendTest.java
+++ b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
@@ -20,38 +20,28 @@ package org.apache.impala.service;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
-import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hive.service.rpc.thrift.TGetCatalogsReq;
import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
import org.apache.hive.service.rpc.thrift.TGetFunctionsReq;
import org.apache.hive.service.rpc.thrift.TGetInfoReq;
import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
import org.apache.hive.service.rpc.thrift.TGetTablesReq;
-import org.junit.Test;
-import org.apache.impala.analysis.AuthorizationTest;
-import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.catalog.Db;
-import org.apache.impala.catalog.Catalog;
import org.apache.impala.catalog.PrimitiveType;
import org.apache.impala.catalog.Table;
-import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.common.ImpalaException;
-import org.apache.impala.testutil.ImpaladTestCatalog;
-import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TMetadataOpRequest;
import org.apache.impala.thrift.TMetadataOpcode;
-import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TResultRow;
import org.apache.impala.thrift.TResultSet;
+import org.junit.Test;
-import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -205,15 +195,15 @@ public class FrontendTest extends FrontendTestBase {
}
}
- // Make sure tables that can't be loaded don't result in errors in the GetTables
- // request (see IMPALA-5579)
+ // IMPALA-5579: GetTables() should succeed and display the available information for
+ // tables that cannot be loaded.
req = new TMetadataOpRequest();
req.opcode = TMetadataOpcode.GET_TABLES;
req.get_tables_req = new TGetTablesReq();
req.get_tables_req.setSchemaName("functional");
req.get_tables_req.setTableName("hive_index_tbl");
resp = execMetadataOp(req);
- assertEquals(0, resp.rows.size());
+ assertEquals(1, resp.rows.size());
}
@Test
http://git-wip-us.apache.org/repos/asf/impala/blob/8ea1ce87/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java b/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
index 7e8ff46..32b5ee7 100644
--- a/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
+++ b/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
@@ -17,15 +17,19 @@
package org.apache.impala.testutil;
+import java.util.Set;
+
+import org.apache.impala.analysis.TableName;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.CatalogServiceCatalog;
import org.apache.impala.catalog.Db;
import org.apache.impala.catalog.HdfsCachePool;
+import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.ImpaladCatalog;
import org.apache.impala.catalog.Table;
-import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.util.PatternMatcher;
+
import com.google.common.base.Preconditions;
/**
@@ -68,27 +72,46 @@ public class ImpaladTestCatalog extends ImpaladCatalog {
public void reset() throws CatalogException { srcCatalog_.reset(); }
/**
- * Overrides ImpaladCatalog.getTable to load the table metadata if it is missing.
+ * Returns the Table for the given name, loading the table's metadata if necessary.
+ * Returns null if the database or table does not exist.
*/
- @Override
- public Table getTable(String dbName, String tableName)
- throws CatalogException {
- Table existingTbl = super.getTable(dbName, tableName);
- // Table doesn't exist or is already loaded. Just return it.
+ public Table getOrLoadTable(String dbName, String tblName) {
+ Db db = getDb(dbName);
+ if (db == null) return null;
+ Table existingTbl = db.getTable(tblName);
+ // Table doesn't exist or is already loaded.
if (existingTbl == null || existingTbl.isLoaded()) return existingTbl;
- // The table was not yet loaded. Load it in to the catalog and try getTable()
- // again.
- Table newTbl = srcCatalog_.getOrLoadTable(dbName, tableName);
+ // The table was not yet loaded. Load it in to the catalog now.
+ Table newTbl = null;
+ try {
+ newTbl = srcCatalog_.getOrLoadTable(dbName, tblName);
+ } catch (CatalogException e) {
+ throw new IllegalStateException("Unexpected table loading failure.", e);
+ }
Preconditions.checkNotNull(newTbl);
Preconditions.checkState(newTbl.isLoaded());
- Db db = getDb(dbName);
- Preconditions.checkNotNull(db);
- db.addTable(newTbl);
- Table resultTable = super.getTable(dbName, tableName);
- if (resultTable instanceof HdfsTable) {
- ((HdfsTable) resultTable).computeHdfsStatsForTesting();
+ if (newTbl instanceof HdfsTable) {
+ ((HdfsTable) newTbl).computeHdfsStatsForTesting();
}
- return resultTable;
+ db.addTable(newTbl);
+ return newTbl;
+ }
+
+ /**
+ * Fast loading path for FE unit testing. Immediately loads the given tables into
+ * this catalog from this thread without involving the catalogd/statestored.
+ */
+ @Override
+ public void prioritizeLoad(Set<TableName> tableNames) {
+ for (TableName tbl: tableNames) getOrLoadTable(tbl.getDb(), tbl.getTbl());
+ }
+
+ /**
+ * No-op. Metadata loading does not go through the catalogd/statestored in a
+ * FE test environment.
+ */
+ @Override
+ public void waitForCatalogUpdate(long timeoutMs) {
}
}