You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/02/14 01:09:06 UTC
[3/6] impala git commit: IMPALA-5440: Add planner tests with extreme
statistics values
IMPALA-5440: Add planner tests with extreme statistics values
This commit address some of the issues in JIRA: tests against the cardinality
overflowing from JOIN, UNION, CROSS JOIN, FULL OUTER JOIN,
0 row number and negative row number, as well as cardinality on Subplan node.
Change-Id: I86dec47cf1438882cafaec53e97864ccfcdff3cb
Reviewed-on: http://gerrit.cloudera.org:8080/9065
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/4e29a7f9
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/4e29a7f9
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/4e29a7f9
Branch: refs/heads/2.x
Commit: 4e29a7f91a35aec7371b2431080ec20082296471
Parents: 3660122
Author: Xinran Yu Tinney <xy...@cloudera.com>
Authored: Thu Jan 18 15:27:31 2018 -0600
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Feb 13 07:44:19 2018 +0000
----------------------------------------------------------------------
.../org/apache/impala/planner/PlannerTest.java | 77 ++++++++++++++++++++
.../apache/impala/planner/PlannerTestBase.java | 40 ++++++++++
2 files changed, 117 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/4e29a7f9/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
index 760334d..5dbba75 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
@@ -504,4 +504,81 @@ public class PlannerTest extends PlannerTestBase {
options.setExplain_level(TExplainLevel.EXTENDED);
runPlannerTestFile("min-max-runtime-filters", options);
}
+
+ @Test
+ public void testCardinalityOverflow() throws ImpalaException {
+ String tblName = "tpch.cardinality_overflow";
+ String colDefs = "("
+ + "l_orderkey BIGINT, "
+ + "l_partkey BIGINT, "
+ + "l_suppkey BIGINT, "
+ + "l_linenumber INT, "
+ + "l_shipmode STRING, "
+ + "l_comment STRING"
+ + ")";
+ String tblLocation = "LOCATION "
+ + "'hdfs://localhost:20500/test-warehouse/tpch.lineitem'";
+ String tblPropsTemplate = "TBLPROPERTIES('numRows'='%s')";
+ String tblProps = String.format(tblPropsTemplate, Long.toString(Long.MAX_VALUE));
+
+ addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s;",
+ tblName, colDefs, tblLocation, tblProps));
+
+ // CROSS JOIN query: tests that multiplying the input cardinalities does not overflow
+ // the cross-join's estimated cardinality
+ String query = "select * from tpch.cardinality_overflow a,"
+ + "tpch.cardinality_overflow b, tpch.cardinality_overflow c";
+ checkCardinality(query, 0, Long.MAX_VALUE);
+
+ // FULL OUTER JOIN query: tests that adding the input cardinalities does not overflow
+ // the full outer join's estimated cardinality
+ query = "select a.l_comment from tpch.cardinality_overflow a full outer join "
+ + "tpch.cardinality_overflow b on a.l_orderkey = b.l_partkey";
+ checkCardinality(query, 0, Long.MAX_VALUE);
+
+ // UNION query: tests that adding the input cardinalities does not overflow
+ // the union's estimated cardinality
+ query = "select l_shipmode from tpch.cardinality_overflow "
+ + "union select l_comment from tpch.cardinality_overflow";
+ checkCardinality(query, 0, Long.MAX_VALUE);
+
+ // JOIN query: tests that multiplying the input cardinalities does not overflow
+ // the join's estimated cardinality
+ query = "select a.l_comment from tpch.cardinality_overflow a inner join "
+ + "tpch.cardinality_overflow b on a.l_linenumber < b.l_orderkey";
+ checkCardinality(query, 0, Long.MAX_VALUE);
+
+ // creates an empty table and tests that the cardinality is 0
+ tblName = "tpch.ex_customer_cardinality_zero";
+ tblProps = String.format(tblPropsTemplate, 0);
+ addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s;",
+ tblName, colDefs, tblLocation, tblProps));
+ query = "select * from tpch.ex_customer_cardinality_zero";
+ checkCardinality(query, 0, 0);
+
+ // creates a table with negative row count and
+ // tests that the cardinality is not negative
+ tblName = "tpch.ex_customer_cardinality_neg";
+ tblProps = String.format(tblPropsTemplate, -1);
+ addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s;",
+ tblName, colDefs, tblLocation, tblProps));
+ query = "select * from tpch.ex_customer_cardinality_neg";
+ checkCardinality(query, -1, Long.MAX_VALUE);
+
+ // SUBPLAN query: tests that adding the input cardinalities does not overflow
+ // the SUBPLAN's estimated cardinality
+ tblName = "functional_parquet.cardinality_overflow";
+ colDefs = "("
+ + "id BIGINT, "
+ + "int_array ARRAY<INT>"
+ + ")";
+ String storedAs = "STORED AS PARQUET";
+ tblLocation = "LOCATION "
+ + "'hdfs://localhost:20500/test-warehouse/complextypestbl_parquet'";
+ tblProps = String.format(tblPropsTemplate, Long.toString(Long.MAX_VALUE));
+ addTestTable(String.format("CREATE EXTERNAL TABLE %s %s %s %s %s;",
+ tblName, colDefs, storedAs, tblLocation, tblProps));
+ query = "select id from functional_parquet.cardinality_overflow t, t.int_array";
+ checkCardinality(query, 0, Long.MAX_VALUE);
+ }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/4e29a7f9/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
index bc93160..7c47d74 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
@@ -36,6 +36,7 @@ import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.fs.Path;
import org.apache.impala.analysis.ColumnLineageGraph;
import org.apache.impala.analysis.DescriptorTable;
+import org.apache.impala.catalog.Catalog;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.common.ImpalaException;
@@ -665,6 +666,45 @@ public class PlannerTestBase extends FrontendTestBase {
}
}
+ /**
+ * This function plans the given query and fails if the estimated cardinalities are
+ * not within the specified bounds [min, max].
+ */
+ protected void checkCardinality(String query, long min, long max)
+ throws ImpalaException {
+ TQueryCtx queryCtx = TestUtils.createQueryContext(Catalog.DEFAULT_DB,
+ System.getProperty("user.name"));
+ queryCtx.client_request.setStmt(query);
+ StringBuilder explainBuilder = new StringBuilder();
+ TExecRequest execRequest = frontend_.createExecRequest(queryCtx, explainBuilder);
+
+ if (!execRequest.isSetQuery_exec_request()
+ || execRequest.query_exec_request == null
+ || execRequest.query_exec_request.plan_exec_info == null) {
+ return;
+ }
+ for (TPlanExecInfo execInfo : execRequest.query_exec_request.plan_exec_info) {
+ for (TPlanFragment planFragment : execInfo.fragments) {
+ if (!planFragment.isSetPlan() || planFragment.plan == null) continue;
+ for (TPlanNode node : planFragment.plan.nodes) {
+ if (node.estimated_stats == null) {
+ fail("Query: " + query + " has no estimated statistics");
+ }
+ long cardinality = node.estimated_stats.cardinality;
+ if (cardinality < min || cardinality > max) {
+ StringBuilder errorLog = new StringBuilder();
+ errorLog.append("Query: " + query + "\n");
+ errorLog.append(
+ "Expected cardinality estimate between " + min + " and " + max + "\n");
+ errorLog.append("Actual cardinality estimate: " + cardinality + "\n");
+ errorLog.append("In node id " + node.node_id + "\n");
+ fail(errorLog.toString());
+ }
+ }
+ }
+ }
+ }
+
private void checkColumnLineage(TestCase testCase, TExecRequest execRequest,
StringBuilder errorLog, StringBuilder actualOutput) {
String query = testCase.getQuery();