You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/12/15 23:00:58 UTC
[32/50] [abbrv] incubator-impala git commit: IMPALA-4014: Introduce
query-wide execution state.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index d2eff19..77bc4cc 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -309,7 +309,7 @@ public class HdfsScanNode extends ScanNode {
* Returns the set of file formats being scanned.
*/
private Set<HdfsFileFormat> computeScanRangeLocations(Analyzer analyzer) {
- long maxScanRangeLength = analyzer.getQueryCtx().getRequest().getQuery_options()
+ long maxScanRangeLength = analyzer.getQueryCtx().client_request.getQuery_options()
.getMax_scan_range_length();
scanRanges_ = Lists.newArrayList();
Set<HdfsFileFormat> fileFormats = Sets.newHashSet();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 3687a17..87ac427 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -601,7 +601,7 @@ public class SingleNodePlanner {
// of table scans. This is only feasible if all materialized aggregate expressions
// have distinct semantics. Please see createHdfsScanPlan() for details.
boolean fastPartitionKeyScans =
- analyzer.getQueryCtx().getRequest().query_options.optimize_partition_key_scans &&
+ analyzer.getQueryCtx().client_request.query_options.optimize_partition_key_scans &&
aggInfo != null && aggInfo.hasAllDistinctAgg();
// Separate table refs into parent refs (uncorrelated or absolute) and
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/main/java/org/apache/impala/planner/UnionNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/UnionNode.java b/fe/src/main/java/org/apache/impala/planner/UnionNode.java
index a8d2291..e88b0e7 100644
--- a/fe/src/main/java/org/apache/impala/planner/UnionNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/UnionNode.java
@@ -124,7 +124,7 @@ public class UnionNode extends PlanNode {
List<Pair<Long, Integer>> memByChildIdx = Lists.newArrayList();
for (int i = 0; i < children_.size(); ++i) {
PlanNode child = children_.get(i);
- child.computeCosts(analyzer.getQueryCtx().request.getQuery_options());
+ child.computeCosts(analyzer.getQueryCtx().client_request.getQuery_options());
memByChildIdx.add(new Pair<Long, Integer>(child.getPerHostMemCost(), i));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 9e7a16a..d236cb1 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -887,7 +887,7 @@ public class Frontend {
AnalysisContext analysisCtx = new AnalysisContext(impaladCatalog_, queryCtx,
authzConfig_);
- if (LOG.isTraceEnabled()) LOG.trace("analyze query " + queryCtx.request.stmt);
+ if (LOG.isTraceEnabled()) LOG.trace("analyze query " + queryCtx.client_request.stmt);
// Run analysis in a loop until it any of the following events occur:
// 1) Analysis completes successfully.
@@ -896,7 +896,7 @@ public class Frontend {
try {
while (true) {
try {
- analysisCtx.analyze(queryCtx.request.stmt);
+ analysisCtx.analyze(queryCtx.client_request.stmt);
Preconditions.checkState(analysisCtx.getAnalyzer().getMissingTbls().isEmpty());
return analysisCtx.getAnalysisResult();
} catch (AnalysisException e) {
@@ -965,7 +965,7 @@ public class Frontend {
} catch (Exception e) {
// Turn exceptions into a warning to allow the query to execute.
LOG.error("Failed to compute resource requirements for query\n" +
- queryCtx.request.getStmt(), e);
+ queryCtx.client_request.getStmt(), e);
}
// The fragment at this point has all state set, serialize it to thrift.
@@ -984,9 +984,9 @@ public class Frontend {
Planner planner, StringBuilder explainString) throws ImpalaException {
TQueryCtx queryCtx = planner.getQueryCtx();
AnalysisContext.AnalysisResult analysisResult = planner.getAnalysisResult();
- boolean isMtExec = analysisResult.isQueryStmt() &&
- queryCtx.request.query_options.isSetMt_dop() &&
- queryCtx.request.query_options.mt_dop > 0;
+ boolean isMtExec = analysisResult.isQueryStmt()
+ && queryCtx.client_request.query_options.isSetMt_dop()
+ && queryCtx.client_request.query_options.mt_dop > 0;
List<PlanFragment> planRoots = Lists.newArrayList();
TQueryExecRequest result = new TQueryExecRequest();
@@ -1009,7 +1009,7 @@ public class Frontend {
// Optionally disable spilling in the backend. Allow spilling if there are plan hints
// or if all tables have stats.
boolean disableSpilling =
- queryCtx.request.query_options.isDisable_unsafe_spills()
+ queryCtx.client_request.query_options.isDisable_unsafe_spills()
&& !queryCtx.tables_missing_stats.isEmpty()
&& !analysisResult.getAnalyzer().hasPlanHints();
// for now, always disable spilling for multi-threaded execution
@@ -1041,11 +1041,11 @@ public class Frontend {
timeline.markEvent("Analysis finished");
Preconditions.checkNotNull(analysisResult.getStmt());
TExecRequest result = new TExecRequest();
- result.setQuery_options(queryCtx.request.getQuery_options());
+ result.setQuery_options(queryCtx.client_request.getQuery_options());
result.setAccess_events(analysisResult.getAccessEvents());
result.analysis_warnings = analysisResult.getAnalyzer().getWarnings();
- TQueryOptions queryOptions = queryCtx.request.query_options;
+ TQueryOptions queryOptions = queryCtx.client_request.query_options;
if (analysisResult.isCatalogOp()) {
result.stmt_type = TStmtType.DDL;
createCatalogOpRequest(analysisResult, result);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
index 3244b20..9dc08df 100644
--- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
+++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
@@ -84,7 +84,7 @@ public class FrontendTestBase {
protected Analyzer createAnalyzer(TQueryOptions queryOptions) {
TQueryCtx queryCtx = TestUtils.createQueryContext();
- queryCtx.request.query_options = queryOptions;
+ queryCtx.client_request.query_options = queryOptions;
return new Analyzer(catalog_, queryCtx,
AuthorizationConfig.createAuthDisabledConfig());
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
index 8c48ee4..8eabe8a 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTest.java
@@ -344,9 +344,9 @@ public class PlannerTest extends PlannerTestBase {
private void testEffectiveMtDop(String stmt, int userMtDop, int expectedMtDop) {
TQueryCtx queryCtx = TestUtils.createQueryContext(
Catalog.DEFAULT_DB, System.getProperty("user.name"));
- queryCtx.request.setStmt(stmt);
- queryCtx.request.query_options = defaultQueryOptions();
- if (userMtDop != -1) queryCtx.request.query_options.setMt_dop(userMtDop);
+ queryCtx.client_request.setStmt(stmt);
+ queryCtx.client_request.query_options = defaultQueryOptions();
+ if (userMtDop != -1) queryCtx.client_request.query_options.setMt_dop(userMtDop);
StringBuilder explainBuilder = new StringBuilder();
TExecRequest request = null;
try {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
index 5e6dbc7..d354897 100644
--- a/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
+++ b/fe/src/test/java/org/apache/impala/planner/PlannerTestBase.java
@@ -406,7 +406,7 @@ public class PlannerTestBase extends FrontendTestBase {
}
TQueryCtx queryCtx = TestUtils.createQueryContext(
dbName, System.getProperty("user.name"));
- queryCtx.request.query_options = options;
+ queryCtx.client_request.query_options = options;
// Test single node plan, scan range locations, and column lineage.
TExecRequest singleNodeExecRequest =
testPlan(testCase, Section.PLAN, queryCtx, errorLog, actualOutput);
@@ -475,16 +475,16 @@ public class PlannerTestBase extends FrontendTestBase {
private TExecRequest testPlan(TestCase testCase, Section section,
TQueryCtx queryCtx, StringBuilder errorLog, StringBuilder actualOutput) {
String query = testCase.getQuery();
- queryCtx.request.setStmt(query);
+ queryCtx.client_request.setStmt(query);
if (section == Section.PLAN) {
- queryCtx.request.getQuery_options().setNum_nodes(1);
+ queryCtx.client_request.getQuery_options().setNum_nodes(1);
} else {
// for distributed and parallel execution we want to run on all available nodes
- queryCtx.request.getQuery_options().setNum_nodes(
+ queryCtx.client_request.getQuery_options().setNum_nodes(
ImpalaInternalServiceConstants.NUM_NODES_ALL);
}
if (section == Section.PARALLELPLANS) {
- queryCtx.request.query_options.setMt_dop(2);
+ queryCtx.client_request.query_options.setMt_dop(2);
}
ArrayList<String> expectedPlan = testCase.getSectionContents(section);
boolean sectionExists = expectedPlan != null && !expectedPlan.isEmpty();
@@ -537,14 +537,14 @@ public class PlannerTestBase extends FrontendTestBase {
StringBuilder explainBuilder = new StringBuilder();
TExecRequest execRequest = null;
TExplainLevel origExplainLevel =
- queryCtx.request.getQuery_options().getExplain_level();
+ queryCtx.client_request.getQuery_options().getExplain_level();
try {
- queryCtx.request.getQuery_options().setExplain_level(TExplainLevel.VERBOSE);
+ queryCtx.client_request.getQuery_options().setExplain_level(TExplainLevel.VERBOSE);
execRequest = frontend_.createExecRequest(queryCtx, explainBuilder);
} catch (ImpalaException e) {
return ExceptionUtils.getStackTrace(e);
} finally {
- queryCtx.request.getQuery_options().setExplain_level(origExplainLevel);
+ queryCtx.client_request.getQuery_options().setExplain_level(origExplainLevel);
}
Preconditions.checkNotNull(execRequest);
String explainStr = removeExplainHeader(explainBuilder.toString());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/test/java/org/apache/impala/service/FrontendTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/service/FrontendTest.java b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
index dd6a6c8..24056dc 100644
--- a/fe/src/test/java/org/apache/impala/service/FrontendTest.java
+++ b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
@@ -101,7 +101,7 @@ public class FrontendTest {
System.out.println(stmt);
TQueryCtx queryCtx = TestUtils.createQueryContext(
Catalog.DEFAULT_DB, AuthorizationTest.USER.getName());
- queryCtx.request.setStmt(stmt);
+ queryCtx.client_request.setStmt(stmt);
try {
fe.createExecRequest(queryCtx, new StringBuilder());
} catch (Exception e) {
@@ -116,7 +116,7 @@ public class FrontendTest {
private void testCatalogIsNotReady(String stmt, Frontend fe) {
TQueryCtx queryCtx = TestUtils.createQueryContext(
Catalog.DEFAULT_DB, AuthorizationTest.USER.getName());
- queryCtx.request.setStmt(stmt);
+ queryCtx.client_request.setStmt(stmt);
try {
fe.createExecRequest(queryCtx, new StringBuilder());
fail("Expected failure to due uninitialized catalog.");
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4b2d76db/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index c958940..e3e5798 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -252,7 +252,7 @@ public class TestUtils {
*/
public static TQueryCtx createQueryContext(String defaultDb, String user) {
TQueryCtx queryCtx = new TQueryCtx();
- queryCtx.setRequest(new TClientRequest("FeTests", new TQueryOptions()));
+ queryCtx.setClient_request(new TClientRequest("FeTests", new TQueryOptions()));
queryCtx.setQuery_id(new TUniqueId());
queryCtx.setSession(new TSessionState(new TUniqueId(), TSessionType.BEESWAX,
defaultDb, user, new TNetworkAddress("localhost", 0)));
@@ -262,7 +262,7 @@ public class TestUtils {
queryCtx.setPid(1000);
// Disable rewrites by default because some analyzer tests have non-executable
// constant exprs (e.g. dummy UDFs) that do not work with constant folding.
- queryCtx.request.query_options.setEnable_expr_rewrites(false);
+ queryCtx.client_request.query_options.setEnable_expr_rewrites(false);
return queryCtx;
}