You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/01/21 14:18:33 UTC
incubator-lens git commit: LENS-203 Pass ExplainQueryContext in
LensDriver.explain (Amareshwari via jdhok)
Repository: incubator-lens
Updated Branches:
refs/heads/master 930343b91 -> ef2355f34
LENS-203 Pass ExplainQueryContext in LensDriver.explain (Amareshwari via
jdhok)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ef2355f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ef2355f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ef2355f3
Branch: refs/heads/master
Commit: ef2355f34c5c880ea470329720ebc5ed9587ea38
Parents: 930343b
Author: jdhok <ja...@inmobi.com>
Authored: Wed Jan 21 18:46:36 2015 +0530
Committer: jdhok <ja...@inmobi.com>
Committed: Wed Jan 21 18:46:36 2015 +0530
----------------------------------------------------------------------
.../lens/driver/cube/TestMinCostSelector.java | 2 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 24 ++++----
.../apache/lens/driver/hive/HiveQueryPlan.java | 16 ++++-
.../apache/lens/driver/hive/TestHiveDriver.java | 24 +++++---
.../lens/driver/hive/TestRemoteHiveDriver.java | 2 +-
.../apache/lens/driver/impala/ImpalaDriver.java | 3 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 21 +++----
.../apache/lens/driver/jdbc/TestJDBCFinal.java | 5 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 18 ++++--
.../lens/server/api/driver/LensDriver.java | 9 +--
.../server/api/query/AbstractQueryContext.java | 63 +++++++++++++++++---
.../api/query/DriverSelectorQueryContext.java | 10 ++--
.../server/api/query/ExplainQueryContext.java | 7 ++-
.../server/api/query/PreparedQueryContext.java | 4 +-
.../lens/server/api/query/QueryContext.java | 22 ++-----
.../lens/server/api/driver/MockDriver.java | 5 +-
.../lens/server/api/driver/MockFailDriver.java | 6 +-
.../server/api/driver/MockQueryContext.java | 2 +-
.../server/query/QueryExecutionServiceImpl.java | 19 +++---
.../lens/server/query/TestQueryService.java | 20 +++----
.../lens/server/query/TestResultFormatting.java | 9 +++
21 files changed, 184 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
index 1981757..030e170 100644
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
+++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
@@ -39,7 +39,7 @@ public class TestMinCostSelector {
private MockQueryContext createMockContext(String query, Configuration conf, LensConf lensConf,
Map<LensDriver, String> driverQueries) throws LensException {
MockQueryContext ctx = new MockQueryContext(query, lensConf, conf, driverQueries.keySet());
- ctx.getDriverContext().setDriverQueriesAndPlans(driverQueries);
+ ctx.setDriverQueriesAndPlans(driverQueries);
return ctx;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index c17004f..64bc281 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -53,6 +53,7 @@ import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.priority.QueryPriorityDecider;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.log4j.Logger;
@@ -307,15 +308,16 @@ public class HiveDriver implements LensDriver {
* @see org.apache.lens.server.api.driver.LensDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
*/
@Override
- public HiveQueryPlan explain(final String query, final Configuration conf) throws LensException {
- LOG.info("Explain: " + query);
- HiveConf explainConf = new HiveConf(conf, this.getClass());
- explainConf.setClassLoader(conf.getClassLoader());
+ public HiveQueryPlan explain(AbstractQueryContext explainCtx) throws LensException {
+ LOG.info("Explain: " + explainCtx.getDriverQuery(this));
+ HiveConf explainConf = new HiveConf(explainCtx.getDriverConf(this), this.getClass());
+ explainConf.setClassLoader(explainCtx.getConf().getClassLoader());
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- final String explainQuery = "EXPLAIN EXTENDED " + query;
- QueryContext explainQueryCtx = new QueryContext(explainQuery, SessionState.get().getUserName(), new LensConf(),
+ final String explainQuery = "EXPLAIN EXTENDED " + explainCtx.getDriverQuery(this);
+ QueryContext explainQueryCtx = new QueryContext(explainQuery, explainCtx.getSubmittedUser(), new LensConf(),
explainConf,
Lists.newArrayList((LensDriver) this), (LensDriver) this, new Date().getTime());
+ explainQueryCtx.setLensSessionIdentifier(explainCtx.getLensSessionIdentifier());
// Get result set of explain
HiveInMemoryResultSet inMemoryResultSet = (HiveInMemoryResultSet) execute(explainQueryCtx);
List<String> explainOutput = new ArrayList<String>();
@@ -344,7 +346,7 @@ public class HiveDriver implements LensDriver {
*/
@Override
public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException {
- DriverQueryPlan plan = explain(pContext.getSelectedDriverQuery(), pContext.getSelectedDriverConf());
+ DriverQueryPlan plan = explain(pContext);
plan.setPrepareHandle(pContext.getPrepareHandle());
return plan;
}
@@ -381,7 +383,7 @@ public class HiveDriver implements LensDriver {
addPersistentPath(ctx);
ctx.getConf().set("mapred.job.name", ctx.getQueryHandle().toString());
OperationHandle op = getClient().executeStatement(getSession(ctx), ctx.getSelectedDriverQuery(),
- ctx.getConf().getValByRegex(".*"));
+ ctx.getSelectedDriverConf().getValByRegex(".*"));
LOG.info("The hive operation handle: " + op);
ctx.setDriverOpHandle(op.toString());
hiveHandles.put(ctx.getQueryHandle(), op);
@@ -423,7 +425,7 @@ public class HiveDriver implements LensDriver {
try {
// Inside try since non-data fetching queries can also be executed by async method.
String priority = queryPriorityDecider.decidePriority(ctx).toString();
- ctx.getConf().set("mapred.job.priority", priority);
+ ctx.getSelectedDriverConf().set("mapred.job.priority", priority);
LOG.info("set priority to " + priority);
} catch (LensException e) {
LOG.error("could not set priority for lens session id:" + ctx.getLensSessionIdentifier()
@@ -752,7 +754,9 @@ public class HiveDriver implements LensDriver {
String hiveQuery;
if (context.isDriverPersistent()
&& context.getConf().getBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE,
- LensConfConstants.DEFAULT_ADD_INSERT_OVEWRITE)) {
+ LensConfConstants.DEFAULT_ADD_INSERT_OVEWRITE)
+ && (context.getSelectedDriverQuery().startsWith("SELECT")
+ || context.getSelectedDriverQuery().startsWith("select"))) {
// store persistent data into user specified location
// If absent, take default home directory
Path resultSetPath = context.getHDFSResultDir();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
index 78f776a..4bda55d 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
@@ -84,6 +84,9 @@ public class HiveQueryPlan extends DriverQueryPlan {
/** The partition. */
PARTITION,
+
+ /** CREATE TABLE if destination is a table */
+ CREATE
};
/**
@@ -187,8 +190,13 @@ public class HiveQueryPlan extends DriverQueryPlan {
String table = explainOutput.get(i).trim().substring("name:".length()).trim();
// update tables queried and weights
if (!tablesQueried.contains(table)) {
+ Table tbl = metastore.getTable(table, false);
+ if (tbl == null) {
+ // table not found, possible case if query is create table
+ HiveDriver.LOG.info("Table " + table + " not found while extracting plan details");
+ continue;
+ }
tablesQueried.add(table);
- Table tbl = metastore.getTable(table);
String costStr = tbl.getParameters().get(LensConfConstants.STORAGE_COST);
Double weight = 1d;
@@ -208,6 +216,10 @@ public class HiveQueryPlan extends DriverQueryPlan {
}
break;
}
+ if (explainOutput.get(i).trim().startsWith("Stage: ")) {
+ // stage got changed
+ break;
+ }
}
break;
}
@@ -246,6 +258,8 @@ public class HiveQueryPlan extends DriverQueryPlan {
return ParserState.PARTITION_LIST;
} else if (tr.equals("Partition") && state == ParserState.PARTITION_LIST) {
return ParserState.PARTITION;
+ } else if (tr.equals("Create Table Operator")) {
+ return ParserState.CREATE;
}
return state;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index cd1a0c7..c63a256 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -46,6 +46,7 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.ExplainQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.testng.annotations.AfterTest;
@@ -129,9 +130,10 @@ public class TestHiveDriver {
protected QueryContext createContext(final String query, Configuration conf) throws LensException {
QueryContext context = new QueryContext(query, "testuser", conf, drivers);
- context.getDriverContext().setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {{ put(driver, query); }} );
- context.setSelectedDriver(driver);
+ // session id has to be set before calling setDriverQueriesAndPlans
context.setLensSessionIdentifier(sessionid);
+ context.setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {{ put(driver, query); }} );
+ context.setSelectedDriver(driver);
return context;
}
@@ -141,6 +143,12 @@ public class TestHiveDriver {
return context;
}
+ protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
+ ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers);
+ ectx.setLensSessionIdentifier(sessionid);
+ return ectx;
+ }
+
/**
* After test.
*
@@ -387,8 +395,8 @@ public class TestHiveDriver {
// Now run a command that would fail
String expectFail = "SELECT ID FROM test_execute_sync";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
- conf.set("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
QueryContext context = createContext(expectFail, conf);
+ context.getConf().set("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
driver.executeAsync(context);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.FAILED, true, false);
@@ -654,7 +662,7 @@ public class TestHiveDriver {
public void testExplain() throws Exception {
createTestTable("test_explain");
SessionState.setCurrentSessionState(ss);
- DriverQueryPlan plan = driver.explain("SELECT ID FROM test_explain", conf);
+ DriverQueryPlan plan = driver.explain(createExplainContext("SELECT ID FROM test_explain", conf));
assertTrue(plan instanceof HiveQueryPlan);
assertEquals(plan.getTableWeight(DATA_BASE + ".test_explain"), 500.0);
Assert.assertEquals(0, driver.getHiveHandleSize());
@@ -710,7 +718,7 @@ public class TestHiveDriver {
createPartitionedTable("test_part_table");
// acquire
SessionState.setCurrentSessionState(ss);
- DriverQueryPlan plan = driver.explain("SELECT ID FROM test_part_table", conf);
+ DriverQueryPlan plan = driver.explain(createExplainContext("SELECT ID FROM test_part_table", conf));
Assert.assertEquals(0, driver.getHiveHandleSize());
assertTrue(plan instanceof HiveQueryPlan);
assertNotNull(plan.getTablesQueried());
@@ -735,9 +743,9 @@ public class TestHiveDriver {
createTestTable("explain_test_2");
SessionState.setCurrentSessionState(ss);
- DriverQueryPlan plan = driver.explain("SELECT explain_test_1.ID, count(1) FROM "
+ DriverQueryPlan plan = driver.explain(createExplainContext("SELECT explain_test_1.ID, count(1) FROM "
+ " explain_test_1 join explain_test_2 on explain_test_1.ID = explain_test_2.ID"
- + " WHERE explain_test_1.ID = 'foo' or explain_test_2.ID = 'bar'" + " GROUP BY explain_test_1.ID", conf);
+ + " WHERE explain_test_1.ID = 'foo' or explain_test_2.ID = 'bar'" + " GROUP BY explain_test_1.ID", conf));
Assert.assertEquals(0, driver.getHiveHandleSize());
assertTrue(plan instanceof HiveQueryPlan);
@@ -811,7 +819,7 @@ public class TestHiveDriver {
};
AbstractQueryContext ctx = new MockQueryContext("driverQuery1", new LensConf(), conf,
driverQuery1.keySet());
- ctx.getDriverContext().setDriverQueriesAndPlans(driverQuery1);
+ ctx.setDriverQueriesAndPlans(driverQuery1);
ctx.setSelectedDriver(mockDriver);
((MockDriver.MockQueryPlan)ctx.getDriverContext().getDriverQueryPlan(mockDriver)).setPartitions
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index b2f980e..26202d7 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -414,7 +414,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
+ "AND table_1.ET='1'";
SessionState.setCurrentSessionState(ss);
- DriverQueryPlan plan = driver.explain(explainQuery, conf);
+ DriverQueryPlan plan = driver.explain(createExplainContext(explainQuery, conf));
Assert.assertEquals(0, driver.getHiveHandleSize());
System.out.println("@@ partitions" + plan.getPartitions());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-impala/src/main/java/org/apache/lens/driver/impala/ImpalaDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-impala/src/main/java/org/apache/lens/driver/impala/ImpalaDriver.java b/lens-driver-impala/src/main/java/org/apache/lens/driver/impala/ImpalaDriver.java
index e9df199..8ed0fee 100644
--- a/lens-driver-impala/src/main/java/org/apache/lens/driver/impala/ImpalaDriver.java
+++ b/lens-driver-impala/src/main/java/org/apache/lens/driver/impala/ImpalaDriver.java
@@ -28,6 +28,7 @@ import org.apache.lens.api.LensException;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.events.LensEventListener;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.log4j.Logger;
@@ -71,7 +72,7 @@ public class ImpalaDriver implements LensDriver {
* @see org.apache.lens.server.api.driver.LensDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
*/
@Override
- public DriverQueryPlan explain(String query, Configuration conf) {
+ public DriverQueryPlan explain(AbstractQueryContext explainCtx) {
/*
* QueryCost q = new QueryCost(); q.setExecMode(ExecMode.INTERACTIVE); q.setScanMode(ScanMode.FULL_SCAN);
* q.setScanSize(-1);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index a5c89cb..1b4982a 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -31,6 +31,7 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.events.LensEventListener;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryRewriter;
@@ -333,7 +334,8 @@ public class JDBCDriver implements LensDriver {
public Statement createStatement(Connection conn) throws SQLException {
Statement stmt;
- boolean enabledRowRetrieval = queryContext.getLensContext().getConf().getBoolean(JDBCDriverConfConstants.JDBC_ENABLE_RESULTSET_STREAMING_RETRIEVAL,
+ boolean enabledRowRetrieval = queryContext.getLensContext().getSelectedDriverConf().getBoolean(
+ JDBCDriverConfConstants.JDBC_ENABLE_RESULTSET_STREAMING_RETRIEVAL,
JDBCDriverConfConstants.DEFAULT_JDBC_ENABLE_RESULTSET_STREAMING_RETRIEVAL);
if (enabledRowRetrieval) {
@@ -350,7 +352,7 @@ public class JDBCDriver implements LensDriver {
: conn.createStatement();
// Get default fetch size from conf if not overridden in query conf
- int fetchSize = queryContext.getLensContext().getConf().getInt(
+ int fetchSize = queryContext.getLensContext().getSelectedDriverConf().getInt(
JDBCDriverConfConstants.JDBC_FETCH_SIZE, JDBCDriverConfConstants.DEFAULT_JDBC_FETCH_SIZE);
stmt.setFetchSize(fetchSize);
}
@@ -557,21 +559,17 @@ public class JDBCDriver implements LensDriver {
/**
* Explain the given query.
*
- * @param query
- * The query should be in HiveQL(SQL like)
- * @param conf
- * The query configuration
+ * @param explainCtx The explain context
* @return The query plan object;
* @throws LensException
* the lens exception
*/
@Override
- public DriverQueryPlan explain(String query, Configuration conf)
- throws LensException {
+ public DriverQueryPlan explain(AbstractQueryContext explainCtx) throws LensException {
checkConfigured();
String explainQuery;
- String rewrittenQuery = rewriteQuery(query, conf);
- Configuration explainConf = new Configuration(conf);
+ String rewrittenQuery = rewriteQuery(explainCtx.getDriverQuery(this), conf);
+ Configuration explainConf = new Configuration(explainCtx.getDriverConf(this));
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER,
false);
String explainKeyword = explainConf.get(JDBC_EXPLAIN_KEYWORD_PARAM,
@@ -588,7 +586,7 @@ public class JDBCDriver implements LensDriver {
final LensDriver driver = (LensDriver) this;
QueryContext explainQueryCtx = new QueryContext(explainQuery, null,
explainConf, new ArrayList<LensDriver>() {{ add(driver); }});
-
+ explainQueryCtx.setLensSessionIdentifier(explainCtx.getLensSessionIdentifier());
QueryResult result = null;
try {
result = executeInternal(explainQueryCtx, explainQuery);
@@ -656,7 +654,6 @@ public class JDBCDriver implements LensDriver {
@Override
public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException {
checkConfigured();
- String rewritten = rewriteQuery(pContext.getSelectedDriverQuery(), conf);
prepare(pContext);
return new JDBCQueryPlan();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
index 9c17749..7534d89 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
@@ -77,6 +77,7 @@ public class TestJDBCFinal {
baseConf.set(JDBCDriverConfConstants.JDBC_USER, "sa");
baseConf.set(JDBCDriverConfConstants.JDBC_PASSWORD, "");
baseConf.set(JDBCDriverConfConstants.JDBC_QUERY_REWRITER_CLASS, ColumnarSQLRewriter.class.getName());
+ baseConf.set(JDBCDriverConfConstants.JDBC_EXPLAIN_KEYWORD_PARAM, "explain plan for ");
driver = new JDBCDriver();
driver.configure(baseConf);
@@ -189,7 +190,7 @@ public class TestJDBCFinal {
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold desc";
QueryContext context = new QueryContext(query, "SA", baseConf, drivers);
- context.getDriverContext().setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {{ put(driver, query); }} );
+ context.setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {{ put(driver, query); }} );
context.setSelectedDriver(driver);
LensResultSet resultSet = driver.execute(context);
@@ -248,7 +249,7 @@ public class TestJDBCFinal {
+ "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold desc ";
QueryContext context = new QueryContext(query, "SA", baseConf, drivers);
- context.getDriverContext().setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {{ put(driver, query); }} );
+ context.setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {{ put(driver, query); }} );
context.setSelectedDriver(driver);
LensResultSet resultSet = driver.execute(context);
assertNotNull(resultSet);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index f86e5b9..e1988d4 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -45,6 +45,7 @@ import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.driver.InMemoryResultSet;
import org.apache.lens.server.api.driver.QueryCompletionListener;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
+import org.apache.lens.server.api.query.ExplainQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.testng.Assert;
@@ -112,7 +113,7 @@ public class TestJdbcDriver {
private QueryContext createQueryContext(final String query) throws LensException {
QueryContext context = new QueryContext(query, "SA", baseConf, drivers);
- context.getDriverContext().setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {
+ context.setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {
{
put(driver, query);
}
@@ -121,6 +122,11 @@ public class TestJdbcDriver {
return context;
}
+ protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
+ ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers);
+ return ectx;
+ }
+
/**
* Creates the table.
*
@@ -241,10 +247,10 @@ public class TestJdbcDriver {
insertData("explain_test"); // Insert some data into table
String query1 = "SELECT * FROM explain_test"; // Select query against existing table
String query2 = "SELECT * FROM explain_test1"; // Select query against non existing table
- driver.explain(query1, baseConf);
+ driver.explain(createExplainContext(query1, baseConf));
try {
- driver.explain(query2, baseConf);
+ driver.explain(createExplainContext(query2, baseConf));
Assert.fail("Running explain on a non existing table.");
} catch (LensException ex) {
System.out.println("Error : " + ex);
@@ -303,7 +309,7 @@ public class TestJdbcDriver {
final String query = "SELECT * from prepare_test";
PreparedQueryContext pContext = new PreparedQueryContext(query, "SA", baseConf, drivers);
- pContext.getDriverContext().setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {
+ pContext.setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {
{
put(driver, query);
}
@@ -415,7 +421,7 @@ public class TestJdbcDriver {
insertData("invalid_conn_close");
final String query = "SELECT * from invalid_conn_close2";
- QueryContext ctx = createQueryContext(query);
+ QueryContext ctx = new QueryContext(query, "SA", baseConf, drivers);
for (int i = 0; i < JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE_DEFAULT; i++) {
driver.executeAsync(ctx);
@@ -519,7 +525,7 @@ public class TestJdbcDriver {
@Test
public void testInvalidQuery() throws Exception {
final String query = "SELECT * FROM invalid_table";
- QueryContext ctx = createQueryContext(query);
+ QueryContext ctx = new QueryContext(query, "SA", baseConf, drivers);
try {
LensResultSet rs = driver.execute(ctx);
fail("Should have thrown exception");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 189600e..c732e09 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -24,6 +24,7 @@ import org.apache.lens.api.LensException;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.server.api.events.LensEventListener;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
@@ -50,12 +51,12 @@ public interface LensDriver extends Externalizable {
/**
* Explain the given query.
*
- * @param query The query should be in HiveQL(SQL like)
- * @param conf The query configuration
- * @return The query plan object;
+ * @param explainCtx The explain context
+ *
+ * @return The query plan object
* @throws LensException the lens exception
*/
- DriverQueryPlan explain(String query, Configuration conf) throws LensException;
+ DriverQueryPlan explain(AbstractQueryContext explainCtx) throws LensException;
/**
* Prepare the given query.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 6c78bea..31ad625 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -20,6 +20,7 @@ package org.apache.lens.server.api.query;
import java.io.Serializable;
import java.util.Collection;
+import java.util.Map;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensException;
@@ -34,6 +35,8 @@ import lombok.Getter;
import lombok.Setter;
public abstract class AbstractQueryContext implements Serializable {
+ private static final long serialVersionUID = 1L;
+
/**
* The Constant LOG
*/
@@ -68,16 +71,51 @@ public abstract class AbstractQueryContext implements Serializable {
/**
* The selected Driver query.
*/
+ protected String selectedDriverQuery;
+
+ /**
+ * The submitted user.
+ */
@Getter
- protected String driverQuery;
+ private final String submittedUser; // Logged in user.
+
+ /**
+ * The lens session identifier.
+ */
+ @Getter
+ @Setter
+ private String lensSessionIdentifier;
+
+ /**
+ * Will be set to true when the selected driver query is set other than user query
+ * This will help avoiding rewrites in case of system restarts.
+ */
+ @Getter private boolean isSelectedDriverQueryExplicitlySet = false;
- protected AbstractQueryContext(final String query, final LensConf qconf, final Configuration conf, final
- Collection<LensDriver> drivers) {
+ protected AbstractQueryContext(final String query, final String user, final LensConf qconf, final Configuration conf,
+ final Collection<LensDriver> drivers) {
driverContext = new DriverSelectorQueryContext(query, conf, drivers);
userQuery = query;
this.lensConf = qconf;
this.conf = conf;
- this.driverQuery = query;
+ this.submittedUser = user;
+ // we are setting selectedDriverQuery as user query only when the drivers size is 1
+ // if drivers size is more than the driver query will be set after selection over drivers
+ if (drivers != null && drivers.size() == 1) {
+ this.selectedDriverQuery = query;
+ setSelectedDriver(drivers.iterator().next());
+ }
+ }
+
+ /**
+ * Set driver queries, and updates for plan from each driver in the context
+ *
+ * @param driverQueries Map of LensDriver to driver's query
+ * @throws LensException
+ */
+ public void setDriverQueriesAndPlans(Map<LensDriver, String> driverQueries) throws LensException {
+ driverContext.setDriverQueriesAndPlans(driverQueries, this);
+ isSelectedDriverQueryExplicitlySet = true;
}
/**
@@ -86,14 +124,22 @@ public abstract class AbstractQueryContext implements Serializable {
* @return the selected driver's query
*/
public String getSelectedDriverQuery() {
- if (driverQuery != null) {
- return driverQuery;
+ if (selectedDriverQuery != null) {
+ return selectedDriverQuery;
} else if (driverContext != null) {
return driverContext.getSelectedDriverQuery();
}
return null;
}
+ public String getDriverQuery(LensDriver driver) {
+ return driverContext.getDriverQuery(driver);
+ }
+
+ public Configuration getDriverConf(LensDriver driver) {
+ return driverContext.getDriverConf(driver);
+ }
+
/**
* Wrapper method for convenience on driver context
*
@@ -112,9 +158,10 @@ public abstract class AbstractQueryContext implements Serializable {
* @param driverQuery
*/
public void setSelectedDriverQuery(String driverQuery) {
- this.driverQuery = driverQuery;
+ this.selectedDriverQuery = driverQuery;
if (driverContext != null) {
driverContext.setSelectedDriverQuery(driverQuery);
+ isSelectedDriverQueryExplicitlySet = true;
}
}
@@ -127,7 +174,7 @@ public abstract class AbstractQueryContext implements Serializable {
public void setSelectedDriver(LensDriver driver) {
if (driverContext != null) {
driverContext.setSelectedDriver(driver);
- driverQuery = driverContext.getSelectedDriverQuery();
+ selectedDriverQuery = driverContext.getSelectedDriverQuery();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index bb47cbd..af989b5 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -122,13 +122,13 @@ public class DriverSelectorQueryContext {
* @param driverQueries
* @throws LensException
*/
- public void setDriverQueriesAndPlans(Map<LensDriver, String> driverQueries) throws LensException {
+ void setDriverQueriesAndPlans(Map<LensDriver, String> driverQueries, AbstractQueryContext qctx)
+ throws LensException {
for (LensDriver driver : driverQueries.keySet()) {
final DriverQueryContext driverQueryContext = driverQueryContextMap.get(driver);
driverQueryContext.setQuery(driverQueries.get(driver));
try {
- driverQueryContext.setDriverQueryPlan(driver.explain(driverQueries.get(driver),
- driverQueryContext.getDriverSpecificConf()));
+ driverQueryContext.setDriverQueryPlan(driver.explain(qctx));
} catch (Exception e) {
LOG.error("Setting driver plan failed for driver " + driver, e);
driverQueryContext.setDriverQueryPlanGenerationError(e);
@@ -163,11 +163,11 @@ public class DriverSelectorQueryContext {
}
public Configuration getSelectedDriverConf() {
- return driverQueryContextMap.get(getSelectedDriver()).getDriverSpecificConf();
+ return getSelectedDriver() == null ? null : driverQueryContextMap.get(getSelectedDriver()).getDriverSpecificConf();
}
public String getSelectedDriverQuery() {
- return driverQueryContextMap.get(getSelectedDriver()).getQuery();
+ return getSelectedDriver() == null ? null : driverQueryContextMap.get(getSelectedDriver()).getQuery();
}
public void setDriverConf(LensDriver driver, Configuration conf) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
index cb42453..0a0cfbd 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
@@ -29,13 +29,16 @@ import org.apache.hadoop.conf.Configuration;
* The class ExplainQueryContext
*/
public class ExplainQueryContext extends AbstractQueryContext {
+ private static final long serialVersionUID = 1L;
+
/**
* Constructor. Only needs user query and conf.
*
* @param query
* @param qconf
*/
- public ExplainQueryContext(String query, LensConf conf, Configuration qconf, Collection<LensDriver> drivers) {
- super(query, conf, qconf, drivers);
+ public ExplainQueryContext(String query, final String user, LensConf conf, Configuration qconf,
+ Collection<LensDriver> drivers) {
+ super(query, user, conf, qconf, drivers);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
index c2a17ba..49d01d2 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
@@ -40,6 +40,8 @@ import lombok.Setter;
*/
public class PreparedQueryContext extends AbstractQueryContext implements Delayed {
+ private static final long serialVersionUID = 1L;
+
/**
* The prepare handle.
*/
@@ -91,7 +93,7 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye
*/
public PreparedQueryContext(String query, String user, Configuration conf, LensConf qconf, Collection<LensDriver>
drivers) {
- super(query, qconf, conf, drivers);
+ super(query, user, qconf, conf, drivers);
this.preparedTime = new Date();
this.preparedUser = user;
this.prepareHandle = new QueryPrepareHandle(UUID.randomUUID());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 93103d0..abaa5f0 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -58,12 +58,6 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
private QueryHandle queryHandle;
/**
- * The submitted user.
- */
- @Getter
- private final String submittedUser; // Logged in user.
-
- /**
* The priority.
*/
@Getter
@@ -129,13 +123,6 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
private long closedTime;
/**
- * The lens session identifier.
- */
- @Getter
- @Setter
- private String lensSessionIdentifier;
-
- /**
* The driver op handle.
*/
@Getter
@@ -245,7 +232,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
prepared.getDriverContext()
.getSelectedDriver(), new Date().getTime());
setDriverContext(prepared.getDriverContext());
- setSelectedDriverQuery(prepared.getDriverQuery());
+ setSelectedDriverQuery(prepared.getSelectedDriverQuery());
}
/**
@@ -260,7 +247,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
*/
public QueryContext(String userQuery, String user, LensConf qconf, Configuration conf,
Collection<LensDriver> drivers, LensDriver selectedDriver, long submissionTime) {
- super(userQuery, qconf, conf, drivers);
+ super(userQuery, user, qconf, conf, drivers);
this.submissionTime = submissionTime;
this.queryHandle = new QueryHandle(UUID.randomUUID());
this.status = new QueryStatus(0.0f, Status.NEW, "Query just got created", false, null, null);
@@ -272,7 +259,6 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
this.isDriverPersistent = conf.getBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER,
LensConfConstants.DEFAULT_DRIVER_PERSISTENT_RESULT_SET);
this.userQuery = userQuery;
- this.submittedUser = user;
if (selectedDriver != null) {
this.setSelectedDriver(selectedDriver);
}
@@ -341,7 +327,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
* @return the lens query
*/
public LensQuery toLensQuery() {
- return new LensQuery(queryHandle, userQuery, submittedUser, priority, isPersistent,
+ return new LensQuery(queryHandle, userQuery, super.getSubmittedUser(), priority, isPersistent,
getSelectedDriver() != null ? getSelectedDriver().getClass()
.getCanonicalName() : null,
getSelectedDriverQuery(),
@@ -406,6 +392,6 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
}
public String getClusterUser() {
- return conf.get(LensConfConstants.SESSION_CLUSTER_USER, submittedUser);
+ return conf.get(LensConfConstants.SESSION_CLUSTER_USER, getSubmittedUser());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index fad480d..02bf324 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -32,6 +32,7 @@ import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.api.query.ResultRow;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.events.LensEventListener;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
@@ -128,8 +129,8 @@ public class MockDriver implements LensDriver {
* org.apache.hadoop.conf.Configuration)
*/
@Override
- public DriverQueryPlan explain(String query, Configuration conf) throws LensException {
- return new MockQueryPlan(query);
+ public DriverQueryPlan explain(AbstractQueryContext explainCtx) throws LensException {
+ return new MockQueryPlan(explainCtx.getUserQuery());
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
index 449acc5..433914a 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
@@ -19,9 +19,7 @@
package org.apache.lens.server.api.driver;
import org.apache.lens.api.LensException;
-
-import org.apache.hadoop.conf.Configuration;
-
+import org.apache.lens.server.api.query.AbstractQueryContext;
public class MockFailDriver extends MockDriver {
@@ -30,7 +28,7 @@ public class MockFailDriver extends MockDriver {
*
* @see org.apache.lens.server.api.driver.MockDriver#explain(java.lang.String, org.apache.hadoop.conf.Configuration)
*/
- public DriverQueryPlan explain(String query, Configuration conf) throws LensException {
+ public DriverQueryPlan explain(AbstractQueryContext explainCtx) throws LensException {
throw new LensException("failing!");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockQueryContext.java
index 9b6f7b1..175eab3 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockQueryContext.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockQueryContext.java
@@ -29,6 +29,6 @@ public class MockQueryContext extends AbstractQueryContext {
public MockQueryContext(final String query, final LensConf qconf,
final Configuration conf, final Collection<LensDriver> drivers) {
- super(query, qconf, conf, drivers);
+ super(query, "testuser", qconf, conf, drivers);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 9521136..c95a948 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -468,11 +468,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
QueryContext ctx = acceptedQueries.take();
synchronized (ctx) {
if (ctx.getStatus().getStatus().equals(Status.QUEUED)) {
- LOG.info("Launching query:" + ctx.getSelectedDriverQuery());
+ LOG.info("Launching query:" + ctx.getUserQuery());
try {
// acquire session before any query operation.
acquire(ctx.getLensSessionIdentifier());
- if (ctx.getSelectedDriver() == null) {
+ // the check to see if the query was already rewritten and selected driver's rewritten query is set
+ if (!ctx.isSelectedDriverQueryExplicitlySet()) {
rewriteAndSelect(ctx);
} else {
LOG.info("Submitting to already selected driver");
@@ -964,7 +965,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* @throws LensException the lens exception
*/
private void rewriteAndSelect(AbstractQueryContext ctx) throws LensException {
- ctx.getDriverContext().setDriverQueriesAndPlans(RewriteUtil.rewriteQuery(ctx));
+ ctx.setDriverQueriesAndPlans(RewriteUtil.rewriteQuery(ctx));
// 2. select driver to run the query
LensDriver driver = driverSelector.select(ctx, conf);
@@ -1782,10 +1783,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LOG.info("Explain: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
- ExplainQueryContext explainQueryContext = new ExplainQueryContext(query, lensConf, qconf, drivers.values());
+ ExplainQueryContext explainQueryContext = new ExplainQueryContext(query,
+ getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values());
accept(query, qconf, SubmitOp.EXPLAIN);
- explainQueryContext.getDriverContext().setDriverQueriesAndPlans(RewriteUtil.rewriteQuery(explainQueryContext));
+ explainQueryContext.setDriverQueriesAndPlans(RewriteUtil.rewriteQuery(explainQueryContext));
// select driver to run the query
explainQueryContext.setSelectedDriver(driverSelector.select(explainQueryContext, qconf));
return explainQueryContext.getSelectedDriverQueryPlan().toQueryPlan();
@@ -1907,11 +1909,14 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
for (int i = 0; i < numQueries; i++) {
QueryContext ctx = (QueryContext) in.readObject();
- //Create Driver Selector Context with driver conf for now and reset it in start()
- DriverSelectorQueryContext driverCtx = new DriverSelectorQueryContext(ctx.getDriverQuery(), new Configuration(),
+ //Create DriverSelectorQueryContext by passing all the drivers and the user query
+ //Driver conf gets reset in start
+ DriverSelectorQueryContext driverCtx = new DriverSelectorQueryContext(ctx.getUserQuery(), new Configuration(),
drivers.values());
ctx.setDriverContext(driverCtx);
boolean driverAvailable = in.readBoolean();
+ // set the selected driver if available, if not available for the cases of queued queries,
+ // query service will do the selection from existing drivers and update
if (driverAvailable) {
String clsName = in.readUTF();
ctx.getDriverContext().setSelectedDriver(drivers.get(clsName));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 7edc3bc..07b106b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -40,8 +40,6 @@ import javax.ws.rs.core.MediaType;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.HiveDriverRunHook;
-import org.apache.hadoop.hive.ql.HiveDriverRunHookContext;
import org.apache.hadoop.io.IOUtils;
import org.apache.lens.driver.hive.TestHiveDriver;
import org.apache.lens.server.api.query.QueryContext;
@@ -345,9 +343,6 @@ public class TestQueryService extends LensJerseyTest {
Thread.sleep(1000);
}
assertTrue(ctx.getSubmissionTime() > 0);
- assertTrue(ctx.getLaunchTime() > 0);
- assertTrue(ctx.getDriverStartTime() > 0);
- assertTrue(ctx.getDriverFinishTime() > 0);
assertTrue(ctx.getFinishTime() > 0);
Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
@@ -375,7 +370,6 @@ public class TestQueryService extends LensJerseyTest {
// test post execute op
final WebTarget target = target().path("queryapi/queries");
LensConf conf = new LensConf();
- conf.addProperty("hive.exec.driver.run.hooks", TestHiveDriver.FailHook.class.getCanonicalName());
final FormDataMultiPart mp = new FormDataMultiPart();
/**
@@ -1366,17 +1360,17 @@ public class TestQueryService extends LensJerseyTest {
Assert.assertFalse(Boolean.parseBoolean(queryService.getHiveConf().get("hive.server2.log.redirection.enabled")));
Assert.assertEquals(queryService.getHiveConf().get("hive.server2.query.log.dir"), "target/query-logs");
- final String query = "test query";
+ final String query = "select ID from " + testTable;
QueryContext ctx = new QueryContext(query, null, queryConf, conf, queryService.getDrivers());
- Map<LensDriver, String> driverQueries = new HashMap<LensDriver, String>() {{ put(queryService.getDrivers
- ().iterator().next(), query); }};
- ctx.getDriverContext().setDriverQueriesAndPlans(driverQueries);
+ Map<LensDriver, String> driverQueries = new HashMap<LensDriver, String>() {{ put(queryService.getDrivers()
+ .iterator().next(), query); }};
+ ctx.setDriverQueriesAndPlans(driverQueries);
Assert.assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader() , ctx.getConf()
.getClassLoader());
- Assert.assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader(), ctx.getDriverContext().getDriverConf(queryService.getDrivers
- ().iterator().next()).getClassLoader());
-
+ Assert.assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader(),
+ ctx.getDriverContext().getDriverConf(queryService.getDrivers().iterator().next()).getClassLoader());
+ Assert.assertTrue(ctx.isSelectedDriverQueryExplicitlySet());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ef2355f3/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
index 5074548..f09a876 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.server.query;
+import static org.testng.Assert.assertTrue;
+
import java.io.IOException;
import java.util.HashMap;
@@ -259,6 +261,13 @@ public class TestResultFormatting extends LensJerseyTest {
if (!isDir) {
TestQueryService.validateHttpEndPoint(target(), lensSessionId, handle, reDirectUrl);
}
+ } else {
+ assertTrue(ctx.getSubmissionTime() > 0);
+ assertTrue(ctx.getLaunchTime() > 0);
+ assertTrue(ctx.getDriverStartTime() > 0);
+ assertTrue(ctx.getDriverFinishTime() > 0);
+ assertTrue(ctx.getFinishTime() > 0);
+ Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
}
}