You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:49:57 UTC
[23/50] [abbrv] incubator-lens git commit: LENS-480 : Priority is not
getting set for the HIVE MR jobs
LENS-480 : Priority is not getting set for the HIVE MR jobs
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/278e0e85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/278e0e85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/278e0e85
Branch: refs/heads/current-release-line
Commit: 278e0e857ccc02f7b673147e5fe0fafa28ea2245
Parents: 019eb94
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Apr 3 20:54:42 2015 +0530
Committer: jdhok <jd...@apache.org>
Committed: Fri Apr 3 20:54:42 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 17 ++++++++++-------
.../apache/lens/driver/hive/TestHiveDriver.java | 19 ++++++++++++++-----
.../lens/driver/hive/TestRemoteHiveDriver.java | 16 ++++++++--------
3 files changed, 32 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/278e0e85/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index a230515..cc944b7 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -457,9 +457,10 @@ public class HiveDriver implements LensDriver {
public LensResultSet execute(QueryContext ctx) throws LensException {
try {
addPersistentPath(ctx);
- ctx.getConf().set("mapred.job.name", ctx.getQueryHandle().toString());
+ Configuration qdconf = ctx.getDriverConf(this);
+ qdconf.set("mapred.job.name", ctx.getQueryHandle().toString());
OperationHandle op = getClient().executeStatement(getSession(ctx), ctx.getSelectedDriverQuery(),
- ctx.getSelectedDriverConf().getValByRegex(".*"));
+ qdconf.getValByRegex(".*"));
LOG.info("The hive operation handle: " + op);
ctx.setDriverOpHandle(op.toString());
hiveHandles.put(ctx.getQueryHandle(), op);
@@ -494,14 +495,15 @@ public class HiveDriver implements LensDriver {
public void executeAsync(QueryContext ctx) throws LensException {
try {
addPersistentPath(ctx);
- ctx.getConf().set("mapred.job.name", ctx.getQueryHandle().toString());
+ Configuration qdconf = ctx.getDriverConf(this);
+ qdconf.set("mapred.job.name", ctx.getQueryHandle().toString());
//Query is already explained.
LOG.info("whetherCalculatePriority: " + whetherCalculatePriority);
if (whetherCalculatePriority) {
try {
// Inside try since non-data fetching queries can also be executed by async method.
String priority = queryPriorityDecider.decidePriority(ctx).toString();
- ctx.getSelectedDriverConf().set("mapred.job.priority", priority);
+ qdconf.set("mapred.job.priority", priority);
LOG.info("set priority to " + priority);
} catch (Exception e) {
// not failing query launch when setting priority fails
@@ -511,7 +513,7 @@ public class HiveDriver implements LensDriver {
}
}
OperationHandle op = getClient().executeStatementAsync(getSession(ctx), ctx.getSelectedDriverQuery(),
- ctx.getConf().getValByRegex(".*"));
+ qdconf.getValByRegex(".*"));
ctx.setDriverOpHandle(op.toString());
LOG.info("QueryHandle: " + ctx.getQueryHandle() + " HiveHandle:" + op);
hiveHandles.put(ctx.getQueryHandle(), op);
@@ -823,7 +825,8 @@ public class HiveDriver implements LensDriver {
*/
void addPersistentPath(QueryContext context) throws IOException {
String hiveQuery;
- boolean addInsertOverwrite = context.getConf().getBoolean(
+ Configuration qdconf = context.getDriverConf(this);
+ boolean addInsertOverwrite = qdconf.getBoolean(
LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, LensConfConstants.DEFAULT_ADD_INSERT_OVEWRITE);
if (context.isDriverPersistent() && addInsertOverwrite
&& (context.getSelectedDriverQuery().startsWith("SELECT")
@@ -835,7 +838,7 @@ public class HiveDriver implements LensDriver {
StringBuilder builder = new StringBuilder("INSERT OVERWRITE DIRECTORY ");
context.setHdfsoutPath(resultSetPath.makeQualified(resultSetPath.getFileSystem(context.getConf())).toString());
builder.append('"').append(resultSetPath).append("\" ");
- String outputDirFormat = context.getConf().get(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT);
+ String outputDirFormat = qdconf.get(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT);
if (outputDirFormat != null) {
builder.append(outputDirFormat);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/278e0e85/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 36594e9..dfc178f 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -120,7 +120,7 @@ public class TestHiveDriver {
conf.addResource("hivedriver-site.xml");
conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
- conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
+ conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
driver.configure(conf);
drivers = new ArrayList<LensDriver>() {
@@ -143,6 +143,13 @@ public class TestHiveDriver {
return context;
}
+ protected QueryContext createContext(final String query, Configuration conf, LensDriver driver) throws LensException {
+ QueryContext context = new QueryContext(query, "testuser", new LensConf(), conf, Arrays.asList(driver));
+ // session id has to be set before calling setDriverQueriesAndPlans
+ context.setLensSessionIdentifier(sessionid);
+ return context;
+ }
+
protected QueryContext createContext(PreparedQueryContext query, Configuration conf) {
QueryContext context = new QueryContext(query, "testuser", new LensConf(), conf);
context.setLensSessionIdentifier(sessionid);
@@ -273,6 +280,7 @@ public class TestHiveDriver {
String select = "SELECT ID FROM test_execute";
QueryContext context = createContext(select, conf);
resultSet = driver.execute(context);
+ Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.name"));
validateInMemoryResult(resultSet);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
context = createContext(select, conf);
@@ -387,21 +395,22 @@ public class TestHiveDriver {
// Now run a command that would fail
String expectFail = "SELECT ID FROM test_execute_sync";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
- QueryContext context = createContext(expectFail, conf);
- context.getConf().set("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
+ Configuration failConf = new Configuration(conf);
+ failConf.set("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
+ QueryContext context = createContext(expectFail, failConf);
driver.executeAsync(context);
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.FAILED, true, false);
Assert.assertEquals(1, driver.getHiveHandleSize());
driver.closeQuery(context.getQueryHandle());
Assert.assertEquals(0, driver.getHiveHandleSize());
-
- conf.set("hive.exec.driver.run.hooks", "");
// Async select query
String select = "SELECT ID FROM test_execute_sync";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
context = createContext(select, conf);
driver.executeAsync(context);
+ Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.name"));
+ Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.priority"));
Assert.assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, false, false);
driver.closeQuery(context.getQueryHandle());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/278e0e85/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 5c70a89..b9fb247 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -134,7 +134,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
conf = new HiveConf(remoteConf);
conf.addResource("hivedriver-site.xml");
driver = new HiveDriver();
- conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
+ conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver.configure(conf);
drivers = new ArrayList<LensDriver>() {
{
@@ -158,7 +158,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
thConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
final HiveDriver thrDriver = new HiveDriver();
thrDriver.configure(thConf);
- QueryContext ctx = createContext("USE " + dataBase, conf);
+ QueryContext ctx = createContext("USE " + dataBase, conf, thrDriver);
thrDriver.execute(ctx);
// Launch a select query
@@ -171,7 +171,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
for (int q = 0; q < QUERIES; q++) {
final QueryContext qctx;
try {
- qctx = createContext("SELECT * FROM test_multithreads", conf);
+ qctx = createContext("SELECT * FROM test_multithreads", conf, thrDriver);
thrDriver.executeAsync(qctx);
} catch (LensException e) {
errCount.incrementAndGet();
@@ -247,7 +247,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- QueryContext ctx = createContext("USE " + dataBase, driverConf);
+ QueryContext ctx = createContext("USE " + dataBase, driverConf, oldDriver);
oldDriver.execute(ctx);
Assert.assertEquals(0, oldDriver.getHiveHandleSize());
@@ -255,20 +255,20 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
// Create some ops with a driver
String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)";
- ctx = createContext(createTable, driverConf);
+ ctx = createContext(createTable, driverConf, oldDriver);
oldDriver.execute(ctx);
// Load some data into the table
String dataLoad = "LOAD DATA LOCAL INPATH '" + TEST_DATA_FILE + "' OVERWRITE INTO TABLE " + tableName;
- ctx = createContext(dataLoad, driverConf);
+ ctx = createContext(dataLoad, driverConf, oldDriver);
oldDriver.execute(ctx);
driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
// Fire two queries
- QueryContext ctx1 = createContext("SELECT * FROM " + tableName, driverConf);
+ QueryContext ctx1 = createContext("SELECT * FROM " + tableName, driverConf, oldDriver);
oldDriver.executeAsync(ctx1);
- QueryContext ctx2 = createContext("SELECT ID FROM " + tableName, driverConf);
+ QueryContext ctx2 = createContext("SELECT ID FROM " + tableName, driverConf, oldDriver);
oldDriver.executeAsync(ctx2);
Assert.assertEquals(2, oldDriver.getHiveHandleSize());