You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pu...@apache.org on 2016/04/26 13:11:15 UTC
lens git commit: LENS-1016 : Fix setting priority in context.
Repository: lens
Updated Branches:
refs/heads/master 06d968151 -> f21130992
LENS-1016 : Fix setting priority in context.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/f2113099
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/f2113099
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/f2113099
Branch: refs/heads/master
Commit: f211309921d1ba32626df7d34e8211c24ab50e24
Parents: 06d9681
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Tue Apr 26 16:40:01 2016 +0530
Committer: Puneet <pu...@inmobi.com>
Committed: Tue Apr 26 16:40:01 2016 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 17 +++++++++++++++--
.../apache/lens/driver/hive/TestHiveDriver.java | 4 ++--
.../server/api/query/AbstractQueryContext.java | 19 +++++++++----------
.../lens/server/api/query/QueryContext.java | 12 ------------
.../api/query/TestAbstractQueryContext.java | 11 -----------
.../server/query/QueryExecutionServiceImpl.java | 6 ++++--
.../lens/server/query/TestQueryService.java | 15 +++++++++++++++
7 files changed, 45 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index aa37dcc..6eec3f7 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -801,12 +801,25 @@ public class HiveDriver extends AbstractLensDriver {
@Override
public Priority decidePriority(AbstractQueryContext ctx) {
- if (whetherCalculatePriority && ctx.getDriverConf(this).get("mapred.job.priority") == null) {
+ return decidePriority(ctx, queryPriorityDecider);
+ }
+
+ Priority decidePriority(AbstractQueryContext ctx, QueryPriorityDecider queryPriorityDecider) {
+ if (whetherCalculatePriority && ctx.getPriority() == null) {
try {
+ // On-demand re-computation of cost, in case it's not alredy set by a previous estimate call.
+ // In driver test cases, estimate doesn't happen. Hence this code path ensures cost is computed and
+ // priority is set based on correct cost.
+ if (ctx.getDriverQueryCost(this) == null) {
+ ctx.setDriverCost(this, this.estimate(ctx));
+ }
// Inside try since non-data fetching queries can also be executed by async method.
- Priority priority = ctx.decidePriority(this, queryPriorityDecider);
+ Priority priority = queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(this));
String priorityStr = priority.toString();
ctx.getDriverConf(this).set("mapred.job.priority", priorityStr);
+ Map<String, String> confUpdate = new HashMap<>();
+ confUpdate.put("mapred.job.priority", priorityStr);
+ ctx.updateConf(confUpdate);
log.info("set priority to {}", priority);
return priority;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index ba3f3d5..3f77b0c 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -887,8 +887,8 @@ public class TestHiveDriver {
ctx.setOlapQuery(true);
Priority priority = driver.decidePriority(ctx);
assertEquals(priority, expected, "cost: " + ctx.getDriverQueryCost(driver) + "priority: " + priority);
- assertEquals(ctx.decidePriority(driver,
- alwaysNormalPriorityDecider), Priority.NORMAL);
+ assertEquals(ctx.getConf().get("mapred.job.priority"), priority.toString());
+ assertEquals(driver.decidePriority(ctx, alwaysNormalPriorityDecider), Priority.NORMAL);
}
@Test
http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index b568ffb..e160f58 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -36,7 +36,6 @@ import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.query.DriverSelectorQueryContext.DriverQueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
-import org.apache.lens.server.api.query.priority.QueryPriorityDecider;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -479,15 +478,15 @@ public abstract class AbstractQueryContext implements Serializable {
hiveConf = null;
}
- public Priority decidePriority(LensDriver driver, QueryPriorityDecider queryPriorityDecider) throws LensException {
- // On-demand re-computation of cost, in case it's not alredy set by a previous estimate call.
- // In driver test cases, estimate doesn't happen. Hence this code path ensures cost is computed and
- // priority is set based on correct cost.
- if (getDriverQueryCost(driver) == null) {
- setDriverCost(driver, driver.estimate(this));
+ /**
+ * Update conf.
+ *
+ * @param confoverlay the conf to set
+ */
+ public void updateConf(Map<String, String> confoverlay) {
+ lensConf.getProperties().putAll(confoverlay);
+ for (Map.Entry<String, String> prop : confoverlay.entrySet()) {
+ this.conf.set(prop.getKey(), prop.getValue());
}
- priority = queryPriorityDecider.decidePriority(getDriverQueryCost(driver));
- return priority;
}
-
}
http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 94b79d0..2a11f97 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -307,18 +307,6 @@ public class QueryContext extends AbstractQueryContext {
return conf;
}
- /**
- * Update conf.
- *
- * @param confoverlay the conf to set
- */
- public void updateConf(Map<String, String> confoverlay) {
- lensConf.getProperties().putAll(confoverlay);
- for (Map.Entry<String, String> prop : confoverlay.entrySet()) {
- this.conf.set(prop.getKey(), prop.getValue());
- }
- }
-
public String getResultSetParentDir() {
return conf.get(LensConfConstants.RESULT_SET_PARENT_DIR, LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
index 7e9fda9..36cc8fa 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
@@ -18,7 +18,6 @@
*/
package org.apache.lens.server.api.query;
-import static org.apache.lens.api.Priority.HIGH;
import static org.apache.lens.server.api.LensConfConstants.*;
import static org.apache.lens.server.api.LensServerAPITestUtil.getConfiguration;
@@ -28,12 +27,10 @@ import java.io.*;
import java.util.Arrays;
import java.util.List;
-import org.apache.lens.api.Priority;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.driver.MockDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.metrics.LensMetricsRegistry;
-import org.apache.lens.server.api.query.priority.MockQueryPriorityDecider;
import org.apache.hadoop.conf.Configuration;
@@ -99,14 +96,6 @@ public class TestAbstractQueryContext {
}
@Test
- public void testPrioritySetting() throws LensException {
- MockQueryContext ctx = new MockQueryContext();
- Priority p = ctx.decidePriority(ctx.getSelectedDriver(), new MockQueryPriorityDecider());
- assertEquals(p, HIGH);
- assertEquals(ctx.getPriority(), HIGH);
- }
-
- @Test
public void testReadAndWriteExternal() throws Exception {
Configuration conf = new Configuration();
List<LensDriver> drivers = MockQueryContext.getDrivers(conf);
http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index d1d8e65..c72a890 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -39,6 +39,7 @@ import javax.ws.rs.core.StreamingOutput;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.api.Priority;
import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
@@ -1324,7 +1325,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
* @param ctx query context
* @throws LensException the lens exception
*/
- private void rewriteAndSelect(final AbstractQueryContext ctx) throws LensException {
+ void rewriteAndSelect(final AbstractQueryContext ctx) throws LensException {
logSegregationContext.setLogSegragationAndQueryId(ctx.getLogHandle());
MethodMetricsContext parallelCallGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
PARALLEL_CALL_GAUGE);
@@ -1419,7 +1420,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
ctx.setSelectedDriver(driver);
QueryCost selectedDriverQueryCost = ctx.getDriverContext().getDriverQueryCost(driver);
ctx.setSelectedDriverQueryCost(selectedDriverQueryCost);
- driver.decidePriority(ctx);
+ Priority priority = driver.decidePriority(ctx);
+ ctx.setPriority(priority == null ? Priority.NORMAL : priority);
selectGauge.markSuccess();
} finally {
parallelCallGauge.markSuccess();
http://git-wip-us.apache.org/repos/asf/lens/blob/f2113099/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index df13ba2..6bf077d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -253,6 +253,21 @@ public class TestQueryService extends LensJerseyTest {
assertTrue(lensQuery.getFinishTime() > 0);
}
+ @Test
+ public void testPriorityOnMockQuery() throws Exception {
+ String query = "select mock, fail from " + TEST_TABLE;
+ QueryContext ctx = queryService.createContext(query, null, new LensConf(), new Configuration(), 5000L);
+ ctx.setLensSessionIdentifier(lensSessionId.getPublicId().toString());
+ queryService.acquire(lensSessionId);
+ try {
+ queryService.rewriteAndSelect(ctx);
+ } finally {
+ queryService.release(lensSessionId);
+ }
+ assertNotNull(ctx.getSelectedDriver());
+ assertEquals(ctx.getPriority(), Priority.NORMAL);
+ }
+
// test with execute async post, get all queries, get query context,
// get wrong uuid query