You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/03/19 13:26:53 UTC
incubator-lens git commit: LENS-356 : Run query cost estimate for
drivers in parallel (Jaideep Dhok via amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/master 948184015 -> b8cc840f7
LENS-356 : Run query cost estimate for drivers in parallel (Jaideep Dhok via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b8cc840f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b8cc840f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b8cc840f
Branch: refs/heads/master
Commit: b8cc840f79cfca8ad95ba135ebae2ced1aca52b7
Parents: 9481840
Author: Jaideep Dhok <jd...@paache.org>
Authored: Thu Mar 19 17:56:34 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Mar 19 17:56:34 2015 +0530
----------------------------------------------------------------------
.../apache/lens/driver/cube/RewriteUtil.java | 180 +++++++++------
.../apache/lens/driver/cube/TestRewriting.java | 121 ++++++----
.../lens/server/api/LensConfConstants.java | 29 +++
.../server/api/query/AbstractQueryContext.java | 122 +++++++---
.../api/query/DriverSelectorQueryContext.java | 8 +-
.../server/api/query/PreparedQueryContext.java | 9 +
.../lens/server/api/query/QueryContext.java | 10 +
.../lens/server/api/driver/MockDriver.java | 10 +
.../server/query/QueryExecutionServiceImpl.java | 228 +++++++++++++++++--
.../src/main/resources/lensserver-default.xml | 30 +++
.../lens/server/query/TestQueryService.java | 31 ++-
lens-server/src/test/resources/lens-site.xml | 5 +
src/site/apt/admin/config.apt | 132 ++++++-----
13 files changed, 679 insertions(+), 236 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
index de79423..3dc66ac 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
@@ -18,10 +18,7 @@
*/
package org.apache.lens.driver.cube;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -42,6 +39,8 @@ import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.log4j.Logger;
+import lombok.Getter;
+
/**
* The Class RewriteUtil.
*/
@@ -214,78 +213,123 @@ public final class RewriteUtil {
* @return the map
* @throws LensException the lens exception
*/
- public static Map<LensDriver, String> rewriteQuery(AbstractQueryContext ctx) throws LensException {
+ public static Map<LensDriver, DriverRewriterRunnable> rewriteQuery(AbstractQueryContext ctx) throws LensException {
try {
String replacedQuery = getReplacedQuery(ctx.getUserQuery());
+ Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<LensDriver, DriverRewriterRunnable>();
+ List<RewriteUtil.CubeQueryInfo> cubeQueries = findCubePositions(replacedQuery, ctx.getHiveConf());
+
+ for (LensDriver driver : ctx.getDriverContext().getDrivers()) {
+ runnables.put(driver, new DriverRewriterRunnable(driver, ctx, cubeQueries, replacedQuery));
+ }
+
+ return runnables;
+ } catch (Exception e) {
+ throw new LensException("Rewriting failed, cause :" + e.getMessage(), e);
+ }
+ }
+
+ public static class DriverRewriterRunnable implements Runnable {
+ @Getter
+ private final LensDriver driver;
+ private final AbstractQueryContext ctx;
+ private final List<CubeQueryInfo> cubeQueries;
+ private final String replacedQuery;
+
+ @Getter
+ /** Indicate if rewrite operation succeeded */
+ private boolean succeeded;
+
+ @Getter
+ /** Get cause of rewrite failure if rewrite operation failed */
+ private String failureCause = null;
+
+ @Getter
+ /** Get eventual rewritten query */
+ private String rewrittenQuery;
+
+ public DriverRewriterRunnable(LensDriver driver,
+ AbstractQueryContext ctx,
+ List<CubeQueryInfo> cubeQueries,
+ String replacedQuery) {
+ this.driver = driver;
+ this.ctx = ctx;
+ this.cubeQueries = cubeQueries;
+ this.replacedQuery = replacedQuery;
+ }
+
+ @Override
+ public void run() {
String lowerCaseQuery = replacedQuery.toLowerCase();
- Map<LensDriver, String> driverQueries = new HashMap<LensDriver, String>();
- StringBuilder rewriteFailure = new StringBuilder();
- String failureCause = null;
- boolean useBuilder = false;
if (lowerCaseQuery.startsWith("add") || lowerCaseQuery.startsWith("set")) {
- for (LensDriver driver : ctx.getDriverContext().getDrivers()) {
- driverQueries.put(driver, replacedQuery);
+ rewrittenQuery = replacedQuery;
+ return;
+ }
+
+ MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
+ REWRITE_QUERY_GAUGE);
+ StringBuilder builder = new StringBuilder();
+ int start = 0;
+ CubeQueryRewriter rewriter = null;
+ try {
+ if (cubeQueries.size() > 0) {
+ // avoid creating rewriter if there are no cube queries
+ rewriter = getCubeRewriter(ctx.getDriverContext().getDriverConf(driver), ctx.getHiveConf());
+ ctx.setOlapQuery(true);
}
- } else {
- List<RewriteUtil.CubeQueryInfo> cubeQueries = findCubePositions(replacedQuery, ctx.getHiveConf());
- for (LensDriver driver : ctx.getDriverContext().getDrivers()) {
- MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
- REWRITE_QUERY_GAUGE);
- StringBuilder builder = new StringBuilder();
- int start = 0;
- CubeQueryRewriter rewriter = null;
- try {
- if (cubeQueries.size() > 0) {
- // avoid creating rewriter if there are no cube queries
- rewriter = getCubeRewriter(ctx.getDriverContext().getDriverConf(driver), ctx.getHiveConf());
- ctx.setOlapQuery(true);
- }
- for (RewriteUtil.CubeQueryInfo cqi : cubeQueries) {
- LOG.debug("Rewriting cube query:" + cqi.query);
- if (start != cqi.startPos) {
- builder.append(replacedQuery.substring(start, cqi.startPos));
- }
- CubeQueryContext cqc = rewriter.rewrite(cqi.query);
- MethodMetricsContext toHQLGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
- TOHQL_GAUGE);
- String hqlQuery = cqc.toHQL();
- toHQLGauge.markSuccess();
- LOG.debug("Rewritten query:" + hqlQuery);
- builder.append(hqlQuery);
- start = cqi.endPos;
- }
- builder.append(replacedQuery.substring(start));
- String finalQuery = builder.toString();
- LOG.info("Final rewritten query for driver:" + driver + " is: " + finalQuery);
- driverQueries.put(driver, finalQuery);
- } catch (Exception e) {
- driverQueries.remove(driver);
- ctx.setDriverRewriteError(driver, e);
- // we are catching all exceptions sothat other drivers can be picked in case of driver bugs
- LOG.warn("Driver : " + driver.getClass().getName() + " Skipped for the query rewriting due to ", e);
- rewriteFailure.append(" Driver :").append(driver.getClass().getName());
- rewriteFailure.append(" Cause :" + e.getLocalizedMessage());
- if (failureCause != null && !failureCause.equals(e.getLocalizedMessage())) {
- useBuilder = true;
- }
- if (failureCause == null) {
- failureCause = e.getLocalizedMessage();
- }
- } finally {
- if (rewriter != null) {
- rewriter.clear();
- }
+
+ // We have to rewrite each sub cube query which might be present in the original
+ // user query. We are looping through all sub queries here.
+ for (RewriteUtil.CubeQueryInfo cqi : cubeQueries) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Rewriting cube query:" + cqi.query);
+ }
+
+ if (start != cqi.startPos) {
+ builder.append(replacedQuery.substring(start, cqi.startPos));
+ }
+
+ // Parse and rewrite individual cube query
+ CubeQueryContext cqc = rewriter.rewrite(cqi.query);
+ MethodMetricsContext toHQLGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
+ TOHQL_GAUGE);
+ // toHQL actually generates the rewritten query
+ String hqlQuery = cqc.toHQL();
+ toHQLGauge.markSuccess();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Rewritten query:" + hqlQuery);
}
- rewriteGauge.markSuccess();
+
+ builder.append(hqlQuery);
+ start = cqi.endPos;
}
+
+ builder.append(replacedQuery.substring(start));
+
+ rewrittenQuery = builder.toString();
+ succeeded = true;
+ ctx.setDriverQuery(driver, rewrittenQuery);
+ LOG.info("Final rewritten query for driver:" + driver + " is: " + rewrittenQuery);
+ } catch (Exception e) {
+ // we are catching all exceptions sothat other drivers can be picked in case of driver bugs
+ LOG.warn("Driver : " + driver + " Skipped for the query rewriting due to ", e);
+ ctx.setDriverRewriteError(driver, e);
+ failureCause = new StringBuilder(" Driver :")
+ .append(driver.getClass().getName())
+ .append(" Cause :" + e.getLocalizedMessage())
+ .toString();
+ } finally {
+ if (rewriter != null) {
+ rewriter.clear();
+ }
+ rewriteGauge.markSuccess();
}
- if (driverQueries.isEmpty()) {
- throw new LensException("No driver accepted the query, because "
- + (useBuilder ? rewriteFailure.toString() : failureCause));
- }
- return driverQueries;
- } catch (Exception e) {
- throw new LensException("Rewriting failed, cause :" + e.getMessage(), e);
+ }
+
+ @Override
+ public String toString() {
+ return "Rewrite runnable for " + driver;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
index 00a0397..34d7dca 100644
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
+++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
@@ -20,10 +20,7 @@ package org.apache.lens.driver.cube;
import static org.mockito.Matchers.any;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensException;
@@ -65,7 +62,6 @@ import com.codahale.metrics.MetricRegistry;
@PowerMockIgnore({"org.apache.log4j.*", "javax.management.*", "javax.xml.*",
"com.sun.org.apache.xerces.internal.jaxp.*", "ch.qos.logback.*", "org.slf4j.*", "org.w3c.dom*"})
public class TestRewriting {
-
/**
* We need a special {@link IObjectFactory}.
*
@@ -149,6 +145,26 @@ public class TestRewriting {
return context;
}
+ private void runRewrites(Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnableMap) {
+ for (LensDriver driver: runnableMap.keySet()) {
+ RewriteUtil.DriverRewriterRunnable r = runnableMap.get(driver);
+ r.run();
+ Assert.assertTrue(r.getDriver() == driver);
+
+ // Failure cause should be set only when rewrite fails
+ if (r.isSucceeded()) {
+ Assert.assertNull(r.getFailureCause(), driver
+ + " succeeded but failure cause is set to " + r.getFailureCause());
+ Assert.assertNotNull(r.getRewrittenQuery(), driver + " succeeded but rewritten query is not set");
+ } else {
+ Assert.assertNotNull(r.getFailureCause(), driver + " failed but failure cause is not set");
+ Assert.assertNull(r.getRewrittenQuery(),
+ driver + " failed but rewritten query is set to " + r.getRewrittenQuery());
+ }
+
+ }
+ }
+
/**
* Test cube query.
*
@@ -172,7 +188,7 @@ public class TestRewriting {
List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(q1, hconf);
Assert.assertEquals(cubeQueries.size(), 0);
QueryContext ctx = new QueryContext(q1, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName());
driver.configure(conf);
@@ -182,7 +198,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
@@ -196,7 +212,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "insert overwrite local directory '/tmp/rewrite' cube select name from table";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -204,7 +220,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "insert overwrite local directory '/tmp/example-output' cube select id,name from dim_table";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -212,7 +228,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select id,name from dim_table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "explain cube select name from table";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -220,7 +236,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table) a";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -228,7 +244,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "insert overwrite directory '/tmp/rewrite' select * from (cube select name from table) a";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -236,7 +252,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table)a";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -244,7 +260,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from ( cube select name from table ) a";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -252,7 +268,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from ( cube select name from table where"
+ " (name = 'ABC'||name = 'XYZ')&&(key=100) ) a";
@@ -262,7 +278,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from"
+ " table where (name = 'ABC' OR name = 'XYZ') AND (key=100)");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table) a join (cube select" + " name2 from table2) b";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -271,7 +287,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table) a full outer join"
+ " (cube select name2 from table2) b on a.name=b.name2";
@@ -281,7 +297,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table) a join (select name2 from table2) b";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -289,13 +305,13 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table union all cube select name2 from table2) u";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 2);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
@@ -305,7 +321,7 @@ public class TestRewriting {
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 2);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
@@ -316,7 +332,7 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select u.* from (select name from table union all cube select name2 from table2)u";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
@@ -324,14 +340,14 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table union all cube select name2"
+ " from table2 union all cube select name3 from table3) u";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 3);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
@@ -342,7 +358,7 @@ public class TestRewriting {
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 3);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
@@ -355,13 +371,13 @@ public class TestRewriting {
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name";
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 2);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
@@ -370,7 +386,7 @@ public class TestRewriting {
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
@@ -379,7 +395,7 @@ public class TestRewriting {
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 2);
Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
@@ -389,7 +405,7 @@ public class TestRewriting {
Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- RewriteUtil.rewriteQuery(ctx);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
Assert.assertEquals(cubeQueries.size(), 1);
Assert.assertEquals(cubeQueries.get(0).query,
"cube select name from table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')");
@@ -399,23 +415,42 @@ public class TestRewriting {
driver2.configure(conf);
drivers.add(driver2);
+ Assert.assertEquals(drivers.size(), 2);
+ Assert.assertTrue(drivers.contains(driver) && drivers.contains(driver2));
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- Map<LensDriver, String> dQueries = RewriteUtil.rewriteQuery(ctx);
- Assert.assertEquals(dQueries.size(), 1);
-
+ Map<LensDriver, RewriteUtil.DriverRewriterRunnable> dQueries = RewriteUtil.rewriteQuery(ctx);
+ Assert.assertEquals(dQueries.size(), 2);
+
+ // Check both driver runnables are present
+ Assert.assertNotNull(dQueries.get(driver), driver + " runnable not found");
+ Assert.assertEquals(dQueries.get(driver).getDriver(), driver);
+
+ Assert.assertNotNull(dQueries.get(driver2), driver2 + " runnable not found");
+ Assert.assertEquals(dQueries.get(driver2).getDriver(), driver2);
+
+ runRewrites(dQueries);
+
+ // We have to verify that query failed for the second driver
+ // First driver passes
+ Iterator<LensDriver> itr = dQueries.keySet().iterator();
+ itr.next();
+ LensDriver failedDriver = itr.next();
+ Assert.assertFalse(dQueries.get(failedDriver).isSucceeded(), failedDriver + " rewrite should have failed");
+ Assert.assertNull(dQueries.get(failedDriver).getRewrittenQuery(),
+ failedDriver + " rewritten query should not be set");
// running again will fail on both drivers
ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- Throwable th = null;
- try {
- RewriteUtil.rewriteQuery(ctx);
- Assert.fail("Shouldn't succeed");
- } catch (LensException e) {
- th = e;
- }
- Assert.assertNotNull(th);
+ Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnables = RewriteUtil.rewriteQuery(ctx);
+ runRewrites(runnables);
+
+ Assert.assertFalse(runnables.get(driver).isSucceeded());
+ Assert.assertNotNull(runnables.get(driver).getFailureCause());
+ Assert.assertNull(runnables.get(driver).getRewrittenQuery());
Assert.assertNotNull(ctx.getDriverRewriteError(driver));
+
+ Assert.assertFalse(runnables.get(driver2).isSucceeded());
+ Assert.assertNotNull(runnables.get(driver2).getFailureCause());
+ Assert.assertNull(runnables.get(driver2).getRewrittenQuery());
Assert.assertNotNull(ctx.getDriverRewriteError(driver2));
- Assert.assertTrue(
- th.getMessage().contains("Rewriting failed, cause :No driver accepted the query, because Mock fail"));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index cdc5fd7..6326b42 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -847,4 +847,33 @@ public final class LensConfConstants {
*/
public static final String QUERY_METRIC_DRIVER_STACK_NAME = QUERY_PFX + "metric.driver.stack.name";
+ /**
+ * Timeout for parallel query estimate calls. A driver needs to comeback with a query estimate within this timeout.
+ */
+ public static final String ESTIMATE_TIMEOUT_MILLIS = SERVER_PFX + "estimate.timeout.millis";
+
+ /**
+ * Default value for timeout for parallel estimate calls.
+ */
+ public static final long DEFAULT_ESTIMATE_TIMEOUT_MILLIS = 300000L; // 5 minutes
+
+
+ /**
+ * Key used to get minimum number of threads in the estimate thread pool
+ */
+ public static final String ESTIMATE_POOL_MIN_THREADS = SERVER_PFX + "estimate.pool.min.threads";
+ public static final int DEFAULT_ESTIMATE_POOL_MIN_THREADS = 3;
+
+ /**
+ * Key used to get maximum number of threads in the estimate thread pool
+ */
+ public static final String ESTIMATE_POOL_MAX_THREADS = SERVER_PFX + "estimate.pool.max.threads";
+ public static final int DEFAULT_ESTIMATE_POOL_MAX_THREADS = 100;
+
+ /**
+ * Key used to get keep alive time for threads in the estimate thread pool
+ */
+ public static final String ESTIMATE_POOL_KEEP_ALIVE_MILLIS = SERVER_PFX + "estimate.pool.keepalive.millis";
+ public static final int DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS = 60000; // 1 minute
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 3101ed6..6390bd4 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -19,9 +19,9 @@
package org.apache.lens.server.api.query;
import java.io.Serializable;
-import java.util.Collection;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensException;
@@ -112,6 +112,9 @@ public abstract class AbstractQueryContext implements Serializable {
@Setter
private boolean olapQuery = false;
+ /** Lock used to synchronize HiveConf access */
+ private final Lock hiveConfLock = new ReentrantLock();
+
protected AbstractQueryContext(final String query, final String user, final LensConf qconf, final Configuration conf,
final Collection<LensDriver> drivers, boolean mergeDriverConf) {
if (conf.getBoolean(LensConfConstants.ENABLE_QUERY_METRICS, LensConfConstants.DEFAULT_ENABLE_QUERY_METRICS)) {
@@ -147,40 +150,75 @@ public abstract class AbstractQueryContext implements Serializable {
* Estimate cost for each driver and set in context
*
* @throws LensException
+ *
*/
public void estimateCostForDrivers() throws LensException {
- StringBuilder detailedFailureCause = new StringBuilder();
- String failureCause = null;
- boolean useBuilder = false;
- boolean succeededOnAtleastOneDriver = false;
+ Map<LensDriver, DriverEstimateRunnable> estimateRunnables = getDriverEstimateRunnables();
+ for (LensDriver driver : estimateRunnables.keySet()) {
+ LOG.info("Running estimate for driver " + driver);
+ estimateRunnables.get(driver).run();
+ }
+ }
+
+ /**
+ * Get runnables wrapping estimate computation, which could be processed offline
+ */
+ public Map<LensDriver, DriverEstimateRunnable> getDriverEstimateRunnables() throws LensException {
+ Map<LensDriver, DriverEstimateRunnable> estimateRunnables = new HashMap<LensDriver, DriverEstimateRunnable>();
+
for (LensDriver driver : driverContext.getDrivers()) {
- final DriverQueryContext driverQueryContext = driverContext.driverQueryContextMap.get(driver);
- MethodMetricsContext estimateGauge = MethodMetricsFactory.createMethodGauge(getDriverConf(driver), true,
- "driverEstimate");
+ estimateRunnables.put(driver, new DriverEstimateRunnable(this, driver));
+ }
+
+ return estimateRunnables;
+ }
+
+ /**
+ * Runnable to wrap estimate computation for a driver. Failure cause and success status
+ * are stored as field members
+ */
+ public static class DriverEstimateRunnable implements Runnable {
+ private final AbstractQueryContext queryContext;
+ private final LensDriver driver;
+
+ @Getter
+ private String failureCause = null;
+
+ @Getter
+ private boolean succeeded = false;
+
+
+ public DriverEstimateRunnable(AbstractQueryContext queryContext,
+ LensDriver driver) {
+ this.queryContext = queryContext;
+ this.driver = driver;
+ }
+
+ @Override
+ public void run() {
+ MethodMetricsContext estimateGauge =
+ MethodMetricsFactory.createMethodGauge(queryContext.getDriverConf(driver), true, "driverEstimate");
+ DriverQueryContext driverQueryContext = queryContext.getDriverContext().getDriverQueryContextMap().get(driver);
if (driverQueryContext.getDriverQueryRewriteError() != null) {
// skip estimate
- continue;
+ return;
}
+
try {
- driverQueryContext.setDriverCost(driver.estimate(this));
- succeededOnAtleastOneDriver = true;
+ driverQueryContext.setDriverCost(driver.estimate(queryContext));
+ succeeded = true;
} catch (Exception e) {
- LOG.error("Setting driver cost failed for driver " + driver, e);
String expMsg = LensUtil.getCauseMessage(e);
driverQueryContext.setDriverQueryCostEstimateError(e);
- detailedFailureCause.append("\n Driver :").append(driver.getClass().getName());
- detailedFailureCause.append(" Cause :" + expMsg);
- if (failureCause != null && !failureCause.equals(expMsg)) {
- useBuilder = true;
- }
- if (failureCause == null) {
- failureCause = expMsg;
- }
+ failureCause = new StringBuilder("Driver :")
+ .append(driver.getClass().getName())
+ .append(" Cause :")
+ .append(expMsg)
+ .toString();
+ LOG.error("Setting driver cost failed for driver " + driver + " Cause: " + failureCause, e);
+ } finally {
+ estimateGauge.markSuccess();
}
- estimateGauge.markSuccess();
- }
- if (!succeededOnAtleastOneDriver) {
- throw new LensException(useBuilder ? detailedFailureCause.toString() : failureCause);
}
}
@@ -333,10 +371,15 @@ public abstract class AbstractQueryContext implements Serializable {
* Should be called judiciously, because constructing HiveConf from conf object is costly.
* @return
*/
- public synchronized HiveConf getHiveConf() {
- if (hiveConf == null) {
- hiveConf = new HiveConf(this.conf, this.getClass());
- hiveConf.setClassLoader(this.conf.getClassLoader());
+ public HiveConf getHiveConf() {
+ hiveConfLock.lock();
+ try {
+ if (hiveConf == null) {
+ hiveConf = new HiveConf(this.conf, this.getClass());
+ hiveConf.setClassLoader(this.conf.getClassLoader());
+ }
+ } finally {
+ hiveConfLock.unlock();
}
return hiveConf;
}
@@ -350,4 +393,23 @@ public abstract class AbstractQueryContext implements Serializable {
public void setFinalDriverQuery(LensDriver driver, String rewrittenQuery) {
driverContext.driverQueryContextMap.get(driver).setFinalDriverQuery(rewrittenQuery);
}
+
+ /**
+ * Set query for a given driver
+ * @param driver driver instance
+ * @param query query string
+ * @throws LensException
+ */
+ public void setDriverQuery(LensDriver driver, String query) {
+ driverContext.setDriverQuery(driver, query);
+ isDriverQueryExplicitlySet = true;
+ }
+
+ /**
+ * Get handle of the query for logging purposes
+ * @return
+ */
+ public String getLogHandle() {
+ return this.getUserQuery();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index b53c4b9..fef834f 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -163,11 +163,15 @@ public class DriverSelectorQueryContext {
*/
void setDriverQueries(Map<LensDriver, String> driverQueries) {
for (LensDriver driver : driverQueries.keySet()) {
- final DriverQueryContext driverQueryContext = driverQueryContextMap.get(driver);
- driverQueryContext.setQuery(driverQueries.get(driver));
+ setDriverQuery(driver, driverQueries.get(driver));
}
}
+ public void setDriverQuery(LensDriver driver, String driverQuery) {
+ final DriverQueryContext driverQueryContext = driverQueryContextMap.get(driver);
+ driverQueryContext.setQuery(driverQuery);
+ }
+
/**
* Sets driver queries, generates plans for each driver by calling explain with respective queries,
* Sets driverQueryPlans
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
index d20c0d0..1ce89ac 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
@@ -152,4 +152,13 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye
.getCanonicalName() : null, getDriverContext().getSelectedDriverQuery(),
lensConf);
}
+
+ /**
+ * Get prepared query handle string
+ * @return
+ */
+ @Override
+ public String getLogHandle() {
+ return prepareHandle.getPrepareHandleId().toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 202ee1e..1d0727d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -390,4 +390,14 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
public String getClusterUser() {
return conf.get(LensConfConstants.SESSION_CLUSTER_USER, getSubmittedUser());
}
+
+
+ /**
+ * Get query handle string
+ * @return
+ */
+ @Override
+ public String getLogHandle() {
+ return queryHandle.getHandleId().toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 12f6833..67c73b8 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -24,6 +24,7 @@ import java.io.ObjectOutput;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lens.api.LensException;
import org.apache.lens.api.query.QueryCost;
@@ -46,6 +47,7 @@ import lombok.Setter;
* The Class MockDriver.
*/
public class MockDriver implements LensDriver {
+ private static AtomicInteger mockDriverId = new AtomicInteger();
/**
* The conf.
@@ -62,10 +64,18 @@ public class MockDriver implements LensDriver {
*/
private int ioTestVal = -1;
+ private final int driverId;
+
/**
* Instantiates a new mock driver.
*/
public MockDriver() {
+ driverId = mockDriverId.incrementAndGet();
+ }
+
+ @Override
+ public String toString() {
+ return "MockDriver:" + driverId;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 8da3fd5..18eb2dc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -23,6 +23,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.NotFoundException;
@@ -67,6 +68,8 @@ import org.codehaus.jackson.*;
import org.codehaus.jackson.map.*;
import org.codehaus.jackson.map.module.SimpleModule;
+import lombok.Getter;
+
/**
* The Class QueryExecutionServiceImpl.
*/
@@ -224,6 +227,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LensServerDAO lensServerDao;
/**
+ * Thread pool used for running query estimates in parallel
+ */
+ private ExecutorService estimatePool;
+
+ /**
* The driver event listener.
*/
final LensEventListener<DriverEvent> driverEventListener = new LensEventListener<DriverEvent>() {
@@ -237,6 +245,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
};
+
/**
* Instantiates a new query execution service impl.
*
@@ -859,7 +868,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
module.addSerializer(ColumnDescriptor.class, new JsonSerializer<ColumnDescriptor>() {
@Override
public void serialize(ColumnDescriptor columnDescriptor, JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
+ SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField("name", columnDescriptor.getName());
jsonGenerator.writeStringField("comment", columnDescriptor.getComment());
@@ -910,6 +919,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LOG.error("Error waiting for thread: " + th.getName(), e);
}
}
+
+ estimatePool.shutdownNow();
LOG.info("Query execution service stopped");
}
@@ -944,34 +955,209 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
statusPoller.start();
queryPurger.start();
prepareQueryPurger.start();
+
+ startEstimatePool();
}
- private static final String ALL_REWRITES_GAUGE = "ALL_CUBE_REWRITES";
- private static final String ALL_DRIVERS_ESTIMATE_GAUGE = "ALL_DRIVER_ESTIMATES";
+ private void startEstimatePool() {
+ int minPoolSize = conf.getInt(LensConfConstants.ESTIMATE_POOL_MIN_THREADS,
+ LensConfConstants.DEFAULT_ESTIMATE_POOL_MIN_THREADS);
+ int maxPoolSize = conf.getInt(LensConfConstants.ESTIMATE_POOL_MAX_THREADS,
+ LensConfConstants.DEFAULT_ESTIMATE_POOL_MAX_THREADS);
+ int keepAlive = conf.getInt(LensConfConstants.ESTIMATE_POOL_KEEP_ALIVE_MILLIS,
+ LensConfConstants.DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS);
+
+ final ThreadFactory defaultFactory = Executors.defaultThreadFactory();
+ final AtomicInteger thId = new AtomicInteger();
+ // We are creating our own thread factory, just so that we can override thread name for easy debugging
+ ThreadFactory threadFactory = new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread th = defaultFactory.newThread(r);
+ th.setName("estimate-" + thId.incrementAndGet());
+ return th;
+ }
+ };
+
+ ThreadPoolExecutor estimatePool = new ThreadPoolExecutor(minPoolSize, maxPoolSize, keepAlive, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(), threadFactory);
+ estimatePool.allowCoreThreadTimeOut(true);
+ estimatePool.prestartCoreThread();
+ this.estimatePool = estimatePool;
+ }
+
+ private static final String REWRITE_GAUGE = "CUBE_REWRITE";
+ private static final String DRIVER_ESTIMATE_GAUGE = "DRIVER_ESTIMATE";
private static final String DRIVER_SELECTOR_GAUGE = "DRIVER_SELECTION";
+ private static final String PARALLEL_CALL_GAUGE = "PARALLEL_ESTIMATE";
+
/**
- * Rewrite and select.
+ * Rewrite the query for each driver, and estimate query cost for the rewritten queries.
+ * Finally, select the driver using driver selector.
*
- * @param ctx the ctx
+ * @param ctx query context
* @throws LensException the lens exception
*/
- private void rewriteAndSelect(AbstractQueryContext ctx) throws LensException {
- MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
- ALL_REWRITES_GAUGE);
- ctx.setDriverQueries(RewriteUtil.rewriteQuery(ctx));
- rewriteGauge.markSuccess();
- MethodMetricsContext estimateGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
- ALL_DRIVERS_ESTIMATE_GAUGE);
- ctx.estimateCostForDrivers();
- estimateGauge.markSuccess();
+ private void rewriteAndSelect(final AbstractQueryContext ctx) throws LensException {
+ MethodMetricsContext parallelCallGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
+ PARALLEL_CALL_GAUGE);
+ try {
+ // Initially we obtain individual runnables for rewrite and estimate calls
+ // These are mapped against the driver, so that later it becomes easy to chain them
+ // for each driver.
+ Map<LensDriver, RewriteUtil.DriverRewriterRunnable> rewriteRunnables = RewriteUtil.rewriteQuery(ctx);
+ Map<LensDriver, AbstractQueryContext.DriverEstimateRunnable> estimateRunnables = ctx.getDriverEstimateRunnables();
+
+ int numDrivers = ctx.getDriverContext().getDrivers().size();
+ final CountDownLatch estimateCompletionLatch = new CountDownLatch(numDrivers);
+ List<RewriteEstimateRunnable> runnables = new ArrayList<RewriteEstimateRunnable>(numDrivers);
+ List<Future> estimateFutures = new ArrayList<Future>();
+
+ for (final LensDriver driver : ctx.getDriverContext().getDrivers()) {
+ RewriteEstimateRunnable r = new RewriteEstimateRunnable(driver,
+ rewriteRunnables.get(driver),
+ estimateRunnables.get(driver),
+ ctx, estimateCompletionLatch);
+
+ // Submit composite rewrite + estimate operation to background pool
+ estimateFutures.add(estimatePool.submit(r));
+ runnables.add(r);
+ }
+
+ // Wait for all rewrite and estimates to finish
+ try {
+ long estimateLatchTimeout = ctx.getConf().getLong(LensConfConstants.ESTIMATE_TIMEOUT_MILLIS,
+ LensConfConstants.DEFAULT_ESTIMATE_TIMEOUT_MILLIS);
+ boolean completed = estimateCompletionLatch.await(estimateLatchTimeout, TimeUnit.MILLISECONDS);
+
+ // log operations yet to complete and check if we can proceed with at least one driver
+ if (!completed) {
+ String debugInfo = "session: " + ctx.getLensSessionIdentifier() + " query:" + ctx.getLogHandle();
+ int inCompleteDrivers = 0;
+
+ for (int i = 0; i < runnables.size(); i++) {
+ RewriteEstimateRunnable r = runnables.get(i);
+ if (!r.isCompleted()) {
+ ++inCompleteDrivers;
+ // Cancel the corresponding task
+ estimateFutures.get(i).cancel(true);
+ LOG.warn("Timeout reached for estimate task for driver " + r.getDriver() + " " + debugInfo);
+ }
+ }
+
+ if (inCompleteDrivers == ctx.getDriverContext().getDrivers().size()) {
+ throw new LensException("None of the drivers could complete within given timeout: " + estimateLatchTimeout
+ + ". " + debugInfo);
+ }
+ }
+ } catch (InterruptedException exc) {
+ throw new LensException("At least one of the estimate operation failed to complete in time", exc);
+ }
+
+ // Evaluate success of rewrite and estimate
+ boolean succeededOnce = false;
+ List<String> failureCauses = new ArrayList<String>(numDrivers);
+
+ for (RewriteEstimateRunnable r : runnables) {
+ if (r.isSucceeded()) {
+ succeededOnce = true;
+ } else {
+ failureCauses.add(r.getFailureCause());
+ }
+ }
+
+ // Throw exception if none of the rewrite+estimates are successful.
+ if (!succeededOnce) {
+ throw new LensException(StringUtils.join(failureCauses, '\n'));
+ }
+
+ MethodMetricsContext selectGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
+ DRIVER_SELECTOR_GAUGE);
+ // 2. select driver to run the query
+ LensDriver driver = driverSelector.select(ctx, conf);
+ selectGauge.markSuccess();
+
+ ctx.setSelectedDriver(driver);
+ } finally {
+ parallelCallGauge.markSuccess();
+ }
+ }
+
+ /**
+ * Chains driver specific rewrite and estimate of the query in a single runnable, which can be
+ * processed in a background thread
+ */
+ public class RewriteEstimateRunnable implements Runnable {
+ @Getter
+ private final LensDriver driver;
+ private final RewriteUtil.DriverRewriterRunnable rewriterRunnable;
+ private final AbstractQueryContext.DriverEstimateRunnable estimateRunnable;
+ private final AbstractQueryContext ctx;
+ private final CountDownLatch estimateCompletionLatch;
+
+ @Getter
+ private boolean succeeded;
+ @Getter
+ private String failureCause = null;
- MethodMetricsContext selectGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
- DRIVER_SELECTOR_GAUGE);
- // 2. select driver to run the query
- LensDriver driver = driverSelector.select(ctx, conf);
- selectGauge.markSuccess();
+ @Getter
+ private volatile boolean completed;
- ctx.setSelectedDriver(driver);
+ public RewriteEstimateRunnable(
+ LensDriver driver,
+ RewriteUtil.DriverRewriterRunnable rewriterRunnable,
+ AbstractQueryContext.DriverEstimateRunnable estimateRunnable,
+ AbstractQueryContext ctx,
+ CountDownLatch estimateCompletionLatch) {
+ this.driver = driver;
+ this.rewriterRunnable = rewriterRunnable;
+ this.estimateRunnable = estimateRunnable;
+ this.ctx = ctx;
+ this.estimateCompletionLatch = estimateCompletionLatch;
+ }
+
+ @Override
+ public void run() {
+ try {
+ acquire(ctx.getLensSessionIdentifier());
+ MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
+ REWRITE_GAUGE);
+ // 1. Rewrite for driver
+ rewriterRunnable.run();
+ succeeded = rewriterRunnable.isSucceeded();
+ failureCause = rewriterRunnable.getFailureCause();
+
+ rewriteGauge.markSuccess();
+
+ // 2. Estimate for driver only if rewrite succeeded.
+ if (succeeded) {
+ MethodMetricsContext estimateGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
+ DRIVER_ESTIMATE_GAUGE);
+
+ estimateRunnable.run();
+ succeeded = estimateRunnable.isSucceeded();
+ failureCause = estimateRunnable.getFailureCause();
+
+ if (!succeeded) {
+ LOG.error("Estimate failed for driver " + driver + " cause: " + failureCause);
+ }
+ estimateGauge.markSuccess();
+ } else {
+ LOG.error("Estimate skipped since rewrite failed for driver " + driver + " cause: " + failureCause);
+ }
+ } catch (Throwable th) {
+ LOG.error("Error computing estimate for driver " + driver, th);
+ } finally {
+ completed = true;
+ try {
+ release(ctx.getLensSessionIdentifier());
+ } catch (LensException e) {
+ LOG.error("Could not release session: " + ctx.getLensSessionIdentifier(), e);
+ } finally {
+ estimateCompletionLatch.countDown();
+ }
+ }
+ }
}
/**
@@ -1793,7 +1979,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
Configuration qconf = getLensConf(sessionHandle, lensConf);
ExplainQueryContext estimateQueryContext = new ExplainQueryContext(query,
getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values());
-
+ estimateQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
accept(query, qconf, SubmitOp.ESTIMATE);
rewriteAndSelect(estimateQueryContext);
return new EstimateResult(estimateQueryContext.getSelectedDriverQueryCost());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index 57fbbc7..9fe7ab5 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -644,4 +644,34 @@
will be ignored</description>
</property>
+ <property>
+ <name>lens.server.estimate.timeout.millis</name>
+ <value>300000</value>
+ <description>Timeout for parallel query estimate calls in milliseconds.
+ A driver needs to comeback with a query estimate within this timeout. If the timeout is reached, only the drivers
+ that have provided an estimate would be considered for query selection. If the timeout is reached and none
+ of the drivers have provided an estimate then estimate calls fails with an exception.
+ </description>
+ </property>
+
+ <property>
+ <name>lens.server.estimate.pool.min.threads</name>
+ <value>3</value>
+ <description>Minimum number of threads in the estimate thread pool</description>
+ </property>
+
+ <property>
+ <name>lens.server.estimate.pool.max.threads</name>
+ <value>100</value>
+ <description>Maximum number of threads in the estimate thread pool</description>
+ </property>
+
+ <property>
+ <name>lens.server.estimate.pool.keepalive.millis</name>
+ <value>60000</value>
+ <description>Thread keep alive time in milliseconds for the estimate thread pool.
+ If there are no estimate requests for this period,then cached threads are released from the pool.
+ </description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 0f20bee..4c84c8b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -47,6 +47,7 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.metrics.LensMetricsRegistry;
import org.apache.lens.server.api.metrics.MetricsService;
+import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.session.SessionService;
import org.apache.lens.server.session.HiveSessionService;
@@ -1476,8 +1477,8 @@ public class TestQueryService extends LensJerseyTest {
Assert.assertNull(result.getCost());
Assert.assertTrue(result.isError());
System.out.println("ERROR for REWRITE:" + result.getErrorMsg());
- Assert.assertTrue(result.getErrorMsg().contains("No driver accepted the query, because Neither cube nor dimensions"
- + " accessed in the query"));
+ Assert.assertTrue(result.getErrorMsg().contains("Cause :Neither cube nor dimensions accessed in the query"),
+ result.getErrorMsg());
}
@Test
@@ -1488,12 +1489,14 @@ public class TestQueryService extends LensJerseyTest {
for (LensDriver driver : queryService.getDrivers()) {
ctx.setDriverRewriteError(driver, new LensException());
}
- try {
- ctx.estimateCostForDrivers();
- Assert.fail("estimate should have failed");
- } catch (LensException e) {
- // expected
+
+ // All estimates should be skipped.
+ Map<LensDriver, AbstractQueryContext.DriverEstimateRunnable> estimateRunnables = ctx.getDriverEstimateRunnables();
+ for (LensDriver driver : estimateRunnables.keySet()) {
+ estimateRunnables.get(driver).run();
+ Assert.assertFalse(estimateRunnables.get(driver).isSucceeded(), driver + " estimate should have been skipped");
}
+
for (LensDriver driver : queryService.getDrivers()) {
Assert.assertNull(ctx.getDriverQueryCost(driver));
}
@@ -1535,8 +1538,16 @@ public class TestQueryService extends LensJerseyTest {
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-ALL_CUBE_REWRITES",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-ALL_DRIVER_ESTIMATES",
- "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION")));
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-CUBE_REWRITE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-DRIVER_ESTIMATE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-"
+ + "org.apache.lens.driver.cube.RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-JDBCDriver-CUBE_REWRITE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-JDBCDriver-DRIVER_ESTIMATE",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-JDBCDriver-"
+ + "org.apache.lens.driver.cube.RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")),
+ reg.getGauges().keySet().toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index 2adf762..d966f47 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -153,4 +153,9 @@
<value>target/resources</value>
</property>
+ <property>
+ <name>lens.server.estimate.timeout.millis</name>
+ <value>120000</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8cc840f/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 9e06da3..a017f1c 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -75,128 +75,136 @@ Lens server configuration
*--+--+---+--+
|24|lens.server.enable.resource.method.metering|false|Whether to Enable metering for all resource methods.|
*--+--+---+--+
-|25|lens.server.event.service.thread.pool.size| |The size of thread pool for notifying events. The no value is specified, it uses the available processors as the number.|
+|25|lens.server.estimate.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the estimate thread pool. If there are no estimate requests for this period,then cached threads are released from the pool.|
*--+--+---+--+
-|26|lens.server.index.ws.resource.impl|org.apache.lens.server.IndexResource|Implementation class for Index Resource|
+|26|lens.server.estimate.pool.max.threads|100|Maximum number of threads in the estimate thread pool|
*--+--+---+--+
-|27|lens.server.mail.from.address|blah@company.com|The from field in the notifier mail to the submitter.|
+|27|lens.server.estimate.pool.min.threads|3|Minimum number of threads in the estimate thread pool|
*--+--+---+--+
-|28|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail|
+|28|lens.server.estimate.timeout.millis|300000|Timeout for parallel query estimate calls in milliseconds. A driver needs to comeback with a query estimate within this timeout. If the timeout is reached, only the drivers that have provided an estimate would be considered for query selection. If the timeout is reached and none of the drivers have provided an estimate then estimate calls fails with an exception.|
*--+--+---+--+
-|29|lens.server.mail.port|25|SMTP Port|
+|29|lens.server.event.service.thread.pool.size| |The size of thread pool for notifying events. The no value is specified, it uses the available processors as the number.|
*--+--+---+--+
-|30|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.|
+|30|lens.server.index.ws.resource.impl|org.apache.lens.server.IndexResource|Implementation class for Index Resource|
*--+--+---+--+
-|31|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.|
+|31|lens.server.mail.from.address|blah@company.com|The from field in the notifier mail to the submitter.|
*--+--+---+--+
-|32|lens.server.max.finished.queries|100|Maximum number of finished queries which lens server will keep in memory before purging.|
+|32|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail|
*--+--+---+--+
-|33|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
+|33|lens.server.mail.port|25|SMTP Port|
*--+--+---+--+
-|34|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource|
+|34|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.|
*--+--+---+--+
-|35|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.|
+|35|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.|
*--+--+---+--+
-|36|lens.server.metrics.ganglia.host| |The ganglia host name|
+|36|lens.server.max.finished.queries|100|Maximum number of finished queries which lens server will keep in memory before purging.|
*--+--+---+--+
-|37|lens.server.metrics.ganglia.port| |The ganglia port|
+|37|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
*--+--+---+--+
-|38|lens.server.metrics.graphite.host| |The graphite host name|
+|38|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource|
*--+--+---+--+
-|39|lens.server.metrics.graphite.port| |The graphite port|
+|39|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.|
*--+--+---+--+
-|40|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
+|40|lens.server.metrics.ganglia.host| |The ganglia host name|
*--+--+---+--+
-|41|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.|
+|41|lens.server.metrics.ganglia.port| |The ganglia port|
*--+--+---+--+
-|42|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource|
+|42|lens.server.metrics.graphite.host| |The graphite host name|
*--+--+---+--+
-|43|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location|
+|43|lens.server.metrics.graphite.port| |The graphite port|
*--+--+---+--+
-|44|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
+|44|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
*--+--+---+--+
-|45|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
+|45|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.|
*--+--+---+--+
-|46|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
+|46|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource|
*--+--+---+--+
-|47|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
+|47|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location|
*--+--+---+--+
-|48|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
+|48|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
*--+--+---+--+
-|49|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
+|49|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
*--+--+---+--+
-|50|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
+|50|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
*--+--+---+--+
-|51|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
+|51|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
*--+--+---+--+
-|52|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
+|52|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
*--+--+---+--+
-|53|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
+|53|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
*--+--+---+--+
-|54|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
+|54|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
*--+--+---+--+
-|55|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
+|55|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
*--+--+---+--+
-|56|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|56|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
*--+--+---+--+
-|57|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|57|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
*--+--+---+--+
-|58|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|58|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
*--+--+---+--+
-|59|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
+|59|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|60|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|60|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|61|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|61|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|62|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|62|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|63|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|63|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
*--+--+---+--+
-|64|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|64|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|65|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
+|65|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|66|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|66|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|67|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
+|67|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|68|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|68|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|69|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|69|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
*--+--+---+--+
-|70|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|70|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|71|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|71|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|72|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|72|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|73|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|73|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|74|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|74|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|75|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|75|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|76|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|76|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|77|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|77|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|78|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|78|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|79|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|79|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|80|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|80|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|81|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|81|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|82|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|82|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|83|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|83|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|84|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|84|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|85|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|85|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|86|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|86|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+*--+--+---+--+
+|87|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+*--+--+---+--+
+|88|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+*--+--+---+--+
+|89|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+*--+--+---+--+
+|90|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values