You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:50:08 UTC
[34/50] [abbrv] incubator-lens git commit: LENS-457 : Adds
Configurability in QueryExecutionServiceImpl (Rajat Khandelwal via
amareshwari)
LENS-457 : Adds Configurability in QueryExecutionServiceImpl (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/e90dc33b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/e90dc33b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/e90dc33b
Branch: refs/heads/current-release-line
Commit: e90dc33bcb54770e5a54d0067337ee52e345622a
Parents: 21102e6
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Apr 9 06:39:31 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Apr 9 06:39:31 2015 +0530
----------------------------------------------------------------------
.../lens/server/api/LensConfConstants.java | 8 +
.../server/query/QueryExecutionServiceImpl.java | 88 +++++++----
.../src/main/resources/lensserver-default.xml | 15 ++
.../lens/server/query/BlahQueryAcceptor.java | 37 +++++
.../lens/server/query/TestQueryService.java | 16 ++
lens-server/src/test/resources/lens-site.xml | 6 +
src/site/apt/admin/config.apt | 150 ++++++++++---------
7 files changed, 220 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e90dc33b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 6326b42..07bcae6 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -55,7 +55,15 @@ public final class LensConfConstants {
* The Constant DRIVER_CLASSES.
*/
public static final String DRIVER_CLASSES = SERVER_PFX + "drivers";
+ /**
+ * The Constant DRIVER_SELECTOR_CLASS.
+ */
+ public static final String DRIVER_SELECTOR_CLASS = SERVER_PFX + "driver.selector.class";
+ /**
+ * The Constant ACCEPTOR_CLASSES.
+ */
+ public static final String ACCEPTOR_CLASSES = SERVER_PFX + "query.acceptors";
/**
* The Constant SERVICE_NAMES.
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e90dc33b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 3e0e0db..3cfd03b 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -126,7 +126,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* The accepted queries.
*/
- private PriorityBlockingQueue<QueryContext> acceptedQueries = new PriorityBlockingQueue<QueryContext>();
+ private PriorityBlockingQueue<QueryContext> queuedQueries = new PriorityBlockingQueue<QueryContext>();
/**
* The launched queries.
@@ -262,7 +262,24 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* Initialize query acceptors and listeners.
*/
- private void initializeQueryAcceptorsAndListeners() {
+ private void initializeQueryAcceptors() throws LensException {
+ String[] acceptorClasses = conf.getStrings(LensConfConstants.ACCEPTOR_CLASSES);
+ if (acceptorClasses != null) {
+ for (String acceptorClass : acceptorClasses) {
+ try {
+ Class<?> clazz = Class.forName(acceptorClass);
+ QueryAcceptor acceptor = (QueryAcceptor) clazz.newInstance();
+ LOG.info("initialized query acceptor: " + acceptor);
+ queryAcceptors.add(acceptor);
+ } catch (Exception e) {
+ LOG.warn("Could not load the acceptor:" + acceptorClass, e);
+ throw new LensException("Could not load acceptor" + acceptorClass, e);
+ }
+ }
+ }
+ }
+
+ private void initializeListeners() {
if (conf.getBoolean(LensConfConstants.QUERY_STATE_LOGGER_ENABLED, true)) {
getEventService().addListenerForType(new QueryStatusLogger(), StatusChange.class);
LOG.info("Registered query state logger");
@@ -281,7 +298,6 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* @throws LensException the lens exception
*/
private void loadDriversAndSelector() throws LensException {
- conf.get(LensConfConstants.DRIVER_CLASSES);
String[] driverClasses = conf.getStrings(LensConfConstants.DRIVER_CLASSES);
if (driverClasses != null) {
for (String driverClass : driverClasses) {
@@ -304,7 +320,17 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
} else {
throw new LensException("No drivers specified");
}
- driverSelector = new MinQueryCostSelector();
+ try {
+ Class<? extends DriverSelector> driverSelectorClass = conf.getClass(LensConfConstants.DRIVER_SELECTOR_CLASS,
+ MinQueryCostSelector.class,
+ DriverSelector.class);
+ LOG.info("Using driver selector class: " + driverSelectorClass.getCanonicalName());
+ driverSelector = driverSelectorClass.newInstance();
+ } catch (Exception e) {
+ throw new LensException("Couldn't instantiate driver selector class. Class name: "
+ + conf.get(LensConfConstants.DRIVER_SELECTOR_CLASS) + ". Please supply a valid value for "
+ + LensConfConstants.DRIVER_SELECTOR_CLASS);
+ }
}
protected LensEventService getEventService() {
@@ -462,7 +488,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LOG.info("Starting QuerySubmitter thread");
while (!pausedForTest && !stopped && !querySubmitter.isInterrupted()) {
try {
- QueryContext ctx = acceptedQueries.take();
+ QueryContext ctx = queuedQueries.take();
synchronized (ctx) {
if (ctx.getStatus().getStatus().equals(Status.QUEUED)) {
LOG.info("Launching query:" + ctx.getUserQuery());
@@ -608,7 +634,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
// before would be null in case of server restart
if (before != null) {
if (before.getStatus().equals(Status.QUEUED)) {
- acceptedQueries.remove(ctx);
+ queuedQueries.remove(ctx);
} else {
launchedQueries.remove(ctx);
}
@@ -844,7 +870,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public synchronized void init(HiveConf hiveConf) {
super.init(hiveConf);
this.conf = hiveConf;
- initializeQueryAcceptorsAndListeners();
+ try {
+ initializeQueryAcceptors();
+ } catch (LensException e) {
+ throw new IllegalStateException("Could not load acceptors");
+ }
+ initializeListeners();
try {
loadDriversAndSelector();
} catch (LensException e) {
@@ -873,7 +904,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
module.addSerializer(ColumnDescriptor.class, new JsonSerializer<ColumnDescriptor>() {
@Override
public void serialize(ColumnDescriptor columnDescriptor, JsonGenerator jsonGenerator,
- SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
+ SerializerProvider serializerProvider) throws IOException, JsonProcessingException {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField("name", columnDescriptor.getName());
jsonGenerator.writeStringField("comment", columnDescriptor.getComment());
@@ -997,8 +1028,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private static final String PARALLEL_CALL_GAUGE = "PARALLEL_ESTIMATE";
/**
- * Rewrite the query for each driver, and estimate query cost for the rewritten queries.
- * Finally, select the driver using driver selector.
+ * Rewrite the query for each driver, and estimate query cost for the rewritten queries. Finally, select the driver
+ * using driver selector.
*
* @param ctx query context
* @throws LensException the lens exception
@@ -1089,8 +1120,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
/**
- * Chains driver specific rewrite and estimate of the query in a single runnable, which can be
- * processed in a background thread
+ * Chains driver specific rewrite and estimate of the query in a single runnable, which can be processed in a
+ * background thread
*/
public class RewriteEstimateRunnable implements Runnable {
@Getter
@@ -1181,7 +1212,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
String rejectionCause = acceptor.accept(query, conf, submitOp);
if (rejectionCause != null) {
getEventService().notifyEvent(new QueryRejected(System.currentTimeMillis(), query, rejectionCause, null));
- throw new LensException("Query not accepted because " + cause);
+ throw new BadRequestException("Query not accepted because " + cause);
}
}
getEventService().notifyEvent(new QueryAccepted(System.currentTimeMillis(), null, query, null));
@@ -1231,11 +1262,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (resultSet == null) {
if (ctx.isPersistent() && ctx.getQueryOutputFormatter() != null) {
resultSets
- .put(queryHandle,
- new LensPersistentResult(
- ctx.getQueryOutputFormatter().getMetadata(),
- ctx.getQueryOutputFormatter().getFinalOutputPath(),
- ctx.getQueryOutputFormatter().getNumRows()));
+ .put(queryHandle,
+ new LensPersistentResult(
+ ctx.getQueryOutputFormatter().getMetadata(),
+ ctx.getQueryOutputFormatter().getFinalOutputPath(),
+ ctx.getQueryOutputFormatter().getNumRows()));
} else if (allQueries.get(queryHandle).isResultAvailableInDriver()) {
resultSet = allQueries.get(queryHandle).getSelectedDriver().fetchResultSet(allQueries.get(queryHandle));
resultSets.put(queryHandle, resultSet);
@@ -1471,7 +1502,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
ctx.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
QueryStatus before = ctx.getStatus();
ctx.setStatus(new QueryStatus(0.0, QueryStatus.Status.QUEUED, "Query is queued", false, null, null));
- acceptedQueries.add(ctx);
+ queuedQueries.add(ctx);
allQueries.put(ctx.getQueryHandle(), ctx);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
LOG.info("Returning handle " + ctx.getQueryHandle().getHandleId());
@@ -2135,7 +2166,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
switch (ctx.getStatus().getStatus()) {
case NEW:
case QUEUED:
- acceptedQueries.add(ctx);
+ queuedQueries.add(ctx);
break;
case LAUNCHED:
case RUNNING:
@@ -2289,7 +2320,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public long getQueuedQueriesCount() {
- return acceptedQueries.size();
+ return queuedQueries.size();
}
@Override
@@ -2321,6 +2352,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LOG.warn("Lens session went away for sessionid:" + lensSession);
return;
}
+
try {
LensSessionImpl session = getSession(sessionHandle);
acquire(sessionHandle);
@@ -2353,7 +2385,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* Add session's resources to selected driver if needed
- * @param ctx the query context
+ *
+ * @param ctx QueryContext for executinf queries
* @throws LensException
*/
protected void addSessionResourcesToDriver(final AbstractQueryContext ctx) {
@@ -2412,17 +2445,18 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* Add resources to hive driver, returning resources which failed to be added
- * @param resources collection of resources intented to be added to hive driver
+ *
+ * @param resources collection of resources intented to be added to hive driver
* @param sessionHandle
* @param hiveDriver
* @return resources which could not be added to hive driver
*/
private List<ResourceEntry> addResources(Collection<ResourceEntry> resources,
- LensSessionHandle sessionHandle,
- HiveDriver hiveDriver) {
+ LensSessionHandle sessionHandle,
+ HiveDriver hiveDriver) {
List<ResourceEntry> failedResources = new ArrayList<ResourceEntry>();
for (ResourceEntry res : resources) {
- try{
+ try {
addSingleResourceToHive(hiveDriver, res, sessionHandle);
} catch (LensException exc) {
failedResources.add(res);
@@ -2434,7 +2468,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
private void addSingleResourceToHive(HiveDriver driver, ResourceEntry res,
- LensSessionHandle sessionHandle) throws LensException {
+ LensSessionHandle sessionHandle) throws LensException {
String sessionIdentifier = sessionHandle.getPublicId().toString();
String uri = res.getLocation();
// Hive doesn't and URIs starting with file:/ correctly, so we have to change it to file:///
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e90dc33b/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index c153193..1a3f82f 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -29,6 +29,21 @@
</property>
<property>
+ <name>lens.server.driver.selector.class</name>
+ <value>org.apache.lens.server.api.driver.MinQueryCostSelector</value>
+ <description>Class for selecting best driver given the query context</description>
+ </property>
+
+ <property>
+ <name>lens.server.query.acceptors</name>
+ <value></value>
+ <description>Query Acceptors configured. Query acceptors are consulted first, before anything happens for the
+ given query. They can either return null or return a messaging indicating why the given query shouldn't be
+ accepted. These can be used to filter out queries at the earliest.
+ </description>
+ </property>
+
+ <property>
<name>lens.server.servicenames</name>
<value>session,query,metastore,scheduler,quota</value>
<description>These services would be started in the specified order when lens-server starts up</description>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e90dc33b/lens-server/src/test/java/org/apache/lens/server/query/BlahQueryAcceptor.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/BlahQueryAcceptor.java b/lens-server/src/test/java/org/apache/lens/server/query/BlahQueryAcceptor.java
new file mode 100644
index 0000000..e51817f
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/BlahQueryAcceptor.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query;
+
+import org.apache.lens.api.LensException;
+import org.apache.lens.api.query.SubmitOp;
+import org.apache.lens.server.api.query.QueryAcceptor;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class BlahQueryAcceptor implements QueryAcceptor {
+ public static final String MSG = "Query can't start with blah";
+
+ @Override
+ public String accept(String query, Configuration conf, SubmitOp submitOp) throws LensException {
+ if (query.toLowerCase().startsWith("blah")) {
+ return MSG;
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e90dc33b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index bce0ca9..bd2aa4c 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -1516,4 +1516,20 @@ public class TestQueryService extends LensJerseyTest {
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")),
reg.getGauges().keySet().toString());
}
+ @Test
+ public void testQueryRejection() throws InterruptedException, IOException {
+ final WebTarget target = target().path("queryapi/queries");
+
+ final FormDataMultiPart mp = new FormDataMultiPart();
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
+ MediaType.APPLICATION_XML_TYPE));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "blah select ID from "
+ + TEST_TABLE));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(),
+ MediaType.APPLICATION_XML_TYPE));
+
+ Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE));
+ Assert.assertEquals(response.getStatus(), 400);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e90dc33b/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index d966f47..3bcbf0e 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -28,6 +28,12 @@
</property>
<property>
+ <name>lens.server.query.acceptors</name>
+ <value>org.apache.lens.server.query.BlahQueryAcceptor</value>
+ <description>Query Acceptors configured</description>
+ </property>
+
+ <property>
<name>test.lens.site.key</name>
<value>gsvalue</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/e90dc33b/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 2462f85..0ff204b 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -63,150 +63,154 @@ Lens server configuration
*--+--+---+--+
|18|lens.server.domain|company.com|Server domain. This will be used in authentication and Sending emails to users. For the server, the username will be without domain and domain will come through conf.|
*--+--+---+--+
-|19|lens.server.drivers|org.apache.lens.driver.hive.HiveDriver|Drivers enabled for this lens server instance|
+|19|lens.server.driver.selector.class|org.apache.lens.server.api.driver.MinQueryCostSelector|Class for selecting best driver given the query context|
*--+--+---+--+
-|20|lens.server.enable.console.metrics|false|Enable metrics to be reported on console|
+|20|lens.server.drivers|org.apache.lens.driver.hive.HiveDriver|Drivers enabled for this lens server instance|
*--+--+---+--+
-|21|lens.server.enable.csv.metrics|false|Enable metrics to be reported as csv files in a directory|
+|21|lens.server.enable.console.metrics|false|Enable metrics to be reported on console|
*--+--+---+--+
-|22|lens.server.enable.ganglia.metrics|false|Enable metrics to be reported on ganglia|
+|22|lens.server.enable.csv.metrics|false|Enable metrics to be reported as csv files in a directory|
*--+--+---+--+
-|23|lens.server.enable.graphite.metrics|false|Enable metrics to be reported on graphite|
+|23|lens.server.enable.ganglia.metrics|false|Enable metrics to be reported on ganglia|
*--+--+---+--+
-|24|lens.server.enable.resource.method.metering|false|Whether to Enable metering for all resource methods.|
+|24|lens.server.enable.graphite.metrics|false|Enable metrics to be reported on graphite|
*--+--+---+--+
-|25|lens.server.estimate.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the estimate thread pool. If there are no estimate requests for this period,then cached threads are released from the pool.|
+|25|lens.server.enable.resource.method.metering|false|Whether to Enable metering for all resource methods.|
*--+--+---+--+
-|26|lens.server.estimate.pool.max.threads|100|Maximum number of threads in the estimate thread pool|
+|26|lens.server.estimate.pool.keepalive.millis|60000|Thread keep alive time in milliseconds for the estimate thread pool. If there are no estimate requests for this period,then cached threads are released from the pool.|
*--+--+---+--+
-|27|lens.server.estimate.pool.min.threads|3|Minimum number of threads in the estimate thread pool|
+|27|lens.server.estimate.pool.max.threads|100|Maximum number of threads in the estimate thread pool|
*--+--+---+--+
-|28|lens.server.estimate.timeout.millis|300000|Timeout for parallel query estimate calls in milliseconds. A driver needs to comeback with a query estimate within this timeout. If the timeout is reached, only the drivers that have provided an estimate would be considered for query selection. If the timeout is reached and none of the drivers have provided an estimate then estimate calls fails with an exception.|
+|28|lens.server.estimate.pool.min.threads|3|Minimum number of threads in the estimate thread pool|
*--+--+---+--+
-|29|lens.server.event.service.thread.pool.size| |The size of thread pool for notifying events. The no value is specified, it uses the available processors as the number.|
+|29|lens.server.estimate.timeout.millis|300000|Timeout for parallel query estimate calls in milliseconds. A driver needs to comeback with a query estimate within this timeout. If the timeout is reached, only the drivers that have provided an estimate would be considered for query selection. If the timeout is reached and none of the drivers have provided an estimate then estimate calls fails with an exception.|
*--+--+---+--+
-|30|lens.server.index.ws.resource.impl|org.apache.lens.server.IndexResource|Implementation class for Index Resource|
+|30|lens.server.event.service.thread.pool.size| |The size of thread pool for notifying events. The no value is specified, it uses the available processors as the number.|
*--+--+---+--+
-|31|lens.server.mail.from.address|blah@company.com|The from field in the notifier mail to the submitter.|
+|31|lens.server.index.ws.resource.impl|org.apache.lens.server.IndexResource|Implementation class for Index Resource|
*--+--+---+--+
-|32|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail|
+|32|lens.server.mail.from.address|blah@company.com|The from field in the notifier mail to the submitter.|
*--+--+---+--+
-|33|lens.server.mail.port|25|SMTP Port|
+|33|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail|
*--+--+---+--+
-|34|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.|
+|34|lens.server.mail.port|25|SMTP Port|
*--+--+---+--+
-|35|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.|
+|35|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.|
*--+--+---+--+
-|36|lens.server.max.finished.queries|100|Maximum number of finished queries which lens server will keep in memory before purging.|
+|36|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.|
*--+--+---+--+
-|37|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
+|37|lens.server.max.finished.queries|100|Maximum number of finished queries which lens server will keep in memory before purging.|
*--+--+---+--+
-|38|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource|
+|38|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
*--+--+---+--+
-|39|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.|
+|39|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource|
*--+--+---+--+
-|40|lens.server.metrics.ganglia.host| |The ganglia host name|
+|40|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.|
*--+--+---+--+
-|41|lens.server.metrics.ganglia.port| |The ganglia port|
+|41|lens.server.metrics.ganglia.host| |The ganglia host name|
*--+--+---+--+
-|42|lens.server.metrics.graphite.host| |The graphite host name|
+|42|lens.server.metrics.ganglia.port| |The ganglia port|
*--+--+---+--+
-|43|lens.server.metrics.graphite.port| |The graphite port|
+|43|lens.server.metrics.graphite.host| |The graphite host name|
*--+--+---+--+
-|44|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
+|44|lens.server.metrics.graphite.port| |The graphite port|
*--+--+---+--+
-|45|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.|
+|45|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
*--+--+---+--+
-|46|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource|
+|46|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.|
*--+--+---+--+
-|47|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location|
+|47|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource|
*--+--+---+--+
-|48|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
+|48|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location|
*--+--+---+--+
-|49|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
+|49|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.|
*--+--+---+--+
-|50|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
+|50|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
*--+--+---+--+
-|51|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
+|51|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
*--+--+---+--+
-|52|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
+|52|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
*--+--+---+--+
-|53|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
+|53|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
*--+--+---+--+
-|54|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
+|54|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
*--+--+---+--+
-|55|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
+|55|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
*--+--+---+--+
-|56|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
+|56|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
*--+--+---+--+
-|57|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
+|57|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
*--+--+---+--+
-|58|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
+|58|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
*--+--+---+--+
-|59|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
+|59|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
*--+--+---+--+
-|60|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|60|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
*--+--+---+--+
-|61|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|61|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|62|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|62|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|63|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
+|63|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|64|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|64|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|65|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|65|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
*--+--+---+--+
-|66|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|66|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|67|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|67|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|68|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|68|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|69|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
+|69|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|70|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
+|70|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|71|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|71|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
*--+--+---+--+
-|72|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
+|72|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
*--+--+---+--+
-|73|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|73|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|74|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|74|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|75|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|75|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|76|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|76|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|77|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|77|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|78|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|78|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|79|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|79|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|80|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|80|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|81|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|81|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|82|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|82|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|83|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|83|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|84|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|84|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|85|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|85|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|86|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|86|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|87|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|87|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|88|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|88|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|89|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|89|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
-|90|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|90|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|91|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|91|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+*--+--+---+--+
+|92|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+*--+--+---+--+
+|93|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values