You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2016/09/21 09:26:04 UTC
lens git commit: LENS-1320: Limit the maximum number of scheduled
jobs per user
Repository: lens
Updated Branches:
refs/heads/master e81402633 -> 6ca6157f3
LENS-1320: Limit the maximum number of scheduled jobs per user
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/6ca6157f
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/6ca6157f
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/6ca6157f
Branch: refs/heads/master
Commit: 6ca6157f38e03b7bf4638267211b4c27aa83ce55
Parents: e814026
Author: Lavkesh Lahngir <la...@linux.com>
Authored: Wed Sep 21 14:54:54 2016 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed Sep 21 14:54:54 2016 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/lens-errors.conf | 7 +-
.../lens/server/api/LensConfConstants.java | 11 +++
.../server/error/LensSchedulerErrorCode.java | 3 +-
.../lens/server/scheduler/SchedulerDAO.java | 24 ++++--
.../server/scheduler/SchedulerServiceImpl.java | 25 ++++--
.../src/main/resources/lensserver-default.xml | 6 ++
.../lens/server/scheduler/SchedulerDAOTest.java | 24 +++---
.../server/scheduler/SchedulerRestartTest.java | 2 +
.../scheduler/TestSchedulerServiceImpl.java | 25 +++++-
lens-server/src/test/resources/lens-site.xml | 4 +
src/site/apt/admin/config.apt | 86 ++++++++++----------
11 files changed, 146 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-api/src/main/resources/lens-errors.conf
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf
index 162fb1d..29e24cf 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -409,7 +409,12 @@ lensSchedulerErrors = [
{
errorCode = 5008
httpStatusCode = ${INTERNAL_SERVER_ERROR}
- errorMsg = "Cannot % job in AlarmService with handle %s"
+ errorMsg = "Cannot %s job in AlarmService with handle %s"
+ }
+ {
+ errorCode = 5009
+ httpStatusCode = ${BAD_REQUEST}
+ errorMsg = "Cannot submit the job because user already has %d scheduled jobs"
}
]
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 8f1983e..8cf617b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -1169,4 +1169,15 @@ public final class LensConfConstants {
*/
public static final int DEFAULT_GRIZZLY_MAX_POOL_SIZE = 40;
+ /**
+ * Maximum Scheduled job per user.
+ */
+ public static final String MAX_SCHEDULED_JOB_PER_USER = SERVER_PFX + "scheduler.max.job.per.user";
+
+ /**
+ * -1 represents that the default is unlimited
+ */
+
+ public static final int DEFAULT_MAX_SCHEDULED_JOB_PER_USER = -1;
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/main/java/org/apache/lens/server/error/LensSchedulerErrorCode.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/error/LensSchedulerErrorCode.java b/lens-server/src/main/java/org/apache/lens/server/error/LensSchedulerErrorCode.java
index 12c4851..8cf1618 100644
--- a/lens-server/src/main/java/org/apache/lens/server/error/LensSchedulerErrorCode.java
+++ b/lens-server/src/main/java/org/apache/lens/server/error/LensSchedulerErrorCode.java
@@ -28,7 +28,8 @@ public enum LensSchedulerErrorCode {
JOB_IS_NOT_SCHEDULED(5005, 0),
JOB_INSTANCE_IS_NOT_YET_RUN(5006, 0),
CANT_UPDATE_RESOURCE_WITH_HANDLE(5007, 0),
- FAILED_ALARM_SERVICE_OPERATION(5008, 0);
+ FAILED_ALARM_SERVICE_OPERATION(5008, 0),
+ MAX_SCHEDULED_JOB_EXCEEDED(5009, 0);
private final LensErrorInfo errorInfo;
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
index b924167..69e81c1 100644
--- a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerDAO.java
@@ -238,7 +238,7 @@ public class SchedulerDAO {
try {
return store.getAllJobInstances(id.getHandleIdString());
} catch (SQLException e) {
- log.error("Error while getting instances of a job with id {}" , id.getHandleIdString(), e);
+ log.error("Error while getting instances of a job with id {}", id.getHandleIdString(), e);
return null;
}
}
@@ -247,14 +247,15 @@ public class SchedulerDAO {
* Gets all jobs which match the filter requirements.
*
* @param username : User name of the job
- * @param jobState : state of the job
* @param startTime : Created on should be greater than this start time.
* @param endTime : Created on should be less than the end time.
+ * @param jobStates : Multiple states of the jobs
* @return List of Job handles
*/
- public List<SchedulerJobHandle> getJobs(String username, SchedulerJobState jobState, Long startTime, Long endTime) {
+ public List<SchedulerJobHandle> getJobs(String username, Long startTime, Long endTime,
+ SchedulerJobState... jobStates) {
try {
- return store.getJobs(username, jobState == null ? null : jobState.name(), startTime, endTime);
+ return store.getJobs(username, jobStates == null ? new SchedulerJobState[] {} : jobStates, startTime, endTime);
} catch (SQLException e) {
log.error("Error while getting jobs ", e);
return null;
@@ -478,20 +479,26 @@ public class SchedulerDAO {
* Gets all the jobs which match the filter requirements.
*
* @param username
- * @param status
+ * @param states
* @param starttime
* @param endtime
* @return the list of job handles.
* @throws SQLException
*/
- public List<SchedulerJobHandle> getJobs(String username, String status, Long starttime, Long endtime)
+ public List<SchedulerJobHandle> getJobs(String username, SchedulerJobState[] states, Long starttime, Long endtime)
throws SQLException {
String whereClause = "";
if (username != null && !username.isEmpty()) {
whereClause += ((whereClause.isEmpty()) ? " WHERE " : " AND ") + COLUMN_USER + " = '" + username + "'";
}
- if (status != null && !status.isEmpty()) {
- whereClause += ((whereClause.isEmpty()) ? " WHERE " : " AND ") + COLUMN_STATUS + " = '" + status + "'";
+ if (states.length > 0) {
+ whereClause += ((whereClause.isEmpty()) ? " WHERE " : " AND ") + COLUMN_STATUS + " IN (";
+ String internalWhere = "";
+ for (SchedulerJobState state : states) {
+ internalWhere += ((internalWhere.isEmpty()) ? "'" : " , '") + state + "'";
+ }
+ whereClause += internalWhere;
+ whereClause += ")";
}
if (starttime != null && starttime > 0) {
whereClause += ((whereClause.isEmpty()) ? " WHERE " : " AND ") + COLUMN_CREATED_ON + " >= " + starttime;
@@ -499,6 +506,7 @@ public class SchedulerDAO {
if (endtime != null && endtime > 0) {
whereClause += ((whereClause.isEmpty()) ? " WHERE " : " AND ") + COLUMN_CREATED_ON + " < " + endtime;
}
+
String fetchSQL = "SELECT " + COLUMN_ID + " FROM " + JOB_TABLE + whereClause;
List<Object[]> result = runner.query(fetchSQL, multipleRowsHandler);
List<SchedulerJobHandle> resOut = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
index 969d740..c683a2c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
@@ -36,6 +36,7 @@ import org.apache.lens.api.scheduler.*;
import org.apache.lens.cube.parse.CubeQueryConfUtil;
import org.apache.lens.server.BaseLensService;
import org.apache.lens.server.LensServices;
+import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.LensErrorInfo;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.SchedulerAlarmEvent;
@@ -79,6 +80,8 @@ public class SchedulerServiceImpl extends BaseLensService implements SchedulerSe
@Getter
private AlarmService alarmService;
+ private int maxJobsPerUser = LensConfConstants.DEFAULT_MAX_SCHEDULED_JOB_PER_USER;
+
/**
* Instantiates a new scheduler service.
*
@@ -91,6 +94,7 @@ public class SchedulerServiceImpl extends BaseLensService implements SchedulerSe
@Override
public synchronized void init(HiveConf hiveConf) {
super.init(hiveConf);
+ maxJobsPerUser = hiveConf.getInt(LensConfConstants.MAX_SCHEDULED_JOB_PER_USER, maxJobsPerUser);
try {
schedulerDAO = new SchedulerDAO(hiveConf);
alarmService = LensServices.get().getService(AlarmService.NAME);
@@ -237,7 +241,7 @@ public class SchedulerServiceImpl extends BaseLensService implements SchedulerSe
@Override
public List<SchedulerJobHandle> getAllJobs(String user, SchedulerJobState state, Long start, Long end)
throws LensException {
- return this.schedulerDAO.getJobs(user, state, start, end);
+ return this.schedulerDAO.getJobs(user, start, end, state);
}
/**
@@ -247,7 +251,7 @@ public class SchedulerServiceImpl extends BaseLensService implements SchedulerSe
public SchedulerJobHandle submitJob(LensSessionHandle sessionHandle, XJob job) throws LensException {
LensSessionImpl session = getSession(sessionHandle);
// Validate XJob
- validateJob(job);
+ validateJob(session, job);
SchedulerJobHandle handle = UtilityMethods.generateSchedulerJobHandle();
long createdOn = System.currentTimeMillis();
SchedulerJobInfo info = new SchedulerJobInfo(handle, job, session.getLoggedInUser(), SchedulerJobState.NEW,
@@ -260,7 +264,18 @@ public class SchedulerServiceImpl extends BaseLensService implements SchedulerSe
}
}
- private void validateJob(XJob job) throws LensException {
+ private void validateJob(LensSessionImpl session, XJob job) throws LensException {
+ // Check if the number of scheduled jobs are not exceeding the configured global value.
+ if (maxJobsPerUser > 0) {
+ int currentJobs = schedulerDAO
+ .getJobs(session.getLoggedInUser(), null, null, SchedulerJobState.NEW, SchedulerJobState.SCHEDULED,
+ SchedulerJobState.SUSPENDED).size();
+ if (currentJobs >= maxJobsPerUser) {
+ throw new LensException(LensSchedulerErrorCode.MAX_SCHEDULED_JOB_EXCEEDED.getLensErrorInfo(), null,
+ currentJobs);
+ }
+
+ }
}
/**
@@ -479,8 +494,8 @@ public class SchedulerServiceImpl extends BaseLensService implements SchedulerSe
log.info("Killing instance {} for job {} ", instanceInfo.getId(), instanceInfo.getJobId());
return updateInstanceRun(latestRun, state);
} else {
- log.info("Killing instance {} for job {} with query handle {} ", instanceInfo.getId(),
- instanceInfo.getJobId(), handle);
+ log.info("Killing instance {} for job {} with query handle {} ", instanceInfo.getId(), instanceInfo.getJobId(),
+ handle);
// This will cause the QueryEnd event which will set the status of the instance to KILLED.
return queryService.cancelQuery(sessionHandle, handle);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index 84ff9f9..d88ab3c 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -905,4 +905,10 @@
<value>40</value>
<description>Max pool size for lens grizzly server</description>
</property>
+
+ <property>
+ <name>lens.server.scheduler.max.job.per.user</name>
+ <value>-1</value>
+ <description>The maximum number of jobs per user which can be scheduled. -1 means unlimited jobs.</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java b/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
index 16a6c78..e698e2b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerDAOTest.java
@@ -103,7 +103,7 @@ public class SchedulerDAOTest {
long currentTime = System.currentTimeMillis();
jobHandle = new SchedulerJobHandle(UUID.randomUUID());
SchedulerJobInfo info = new SchedulerJobInfo(jobHandle, job, "lens", SchedulerJobState.NEW, currentTime,
- currentTime);
+ currentTime);
// Store the job
schedulerDAO.storeJob(info);
// Retrive the stored job
@@ -116,10 +116,10 @@ public class SchedulerDAOTest {
long currentTime = System.currentTimeMillis();
SchedulerJobInstanceHandle instanceHandle = new SchedulerJobInstanceHandle(UUID.randomUUID());
SchedulerJobInstanceInfo firstInstance = new SchedulerJobInstanceInfo(instanceHandle, jobHandle, currentTime,
- new ArrayList<SchedulerJobInstanceRun>());
+ new ArrayList<SchedulerJobInstanceRun>());
SchedulerJobInstanceRun run1 = new SchedulerJobInstanceRun(instanceHandle, 1,
- new LensSessionHandle(UUID.randomUUID(), UUID.randomUUID()), currentTime, currentTime, "/tmp/",
- QueryHandle.fromString(UUID.randomUUID().toString()), SchedulerJobInstanceState.WAITING);
+ new LensSessionHandle(UUID.randomUUID(), UUID.randomUUID()), currentTime, currentTime, "/tmp/",
+ QueryHandle.fromString(UUID.randomUUID().toString()), SchedulerJobInstanceState.WAITING);
instances.put(firstInstance.getId(), firstInstance);
schedulerDAO.storeJobInstance(firstInstance);
schedulerDAO.storeJobInstanceRun(run1);
@@ -129,10 +129,10 @@ public class SchedulerDAOTest {
currentTime = System.currentTimeMillis();
instanceHandle = new SchedulerJobInstanceHandle(UUID.randomUUID());
SchedulerJobInstanceInfo secondInstance = new SchedulerJobInstanceInfo(instanceHandle, jobHandle, currentTime,
- new ArrayList<SchedulerJobInstanceRun>());
+ new ArrayList<SchedulerJobInstanceRun>());
SchedulerJobInstanceRun run2 = new SchedulerJobInstanceRun(instanceHandle, 1,
- new LensSessionHandle(UUID.randomUUID(), UUID.randomUUID()), currentTime, currentTime, "/tmp/",
- QueryHandle.fromString(UUID.randomUUID().toString()), SchedulerJobInstanceState.WAITING);
+ new LensSessionHandle(UUID.randomUUID(), UUID.randomUUID()), currentTime, currentTime, "/tmp/",
+ QueryHandle.fromString(UUID.randomUUID().toString()), SchedulerJobInstanceState.WAITING);
instances.put(secondInstance.getId(), secondInstance);
schedulerDAO.storeJobInstance(secondInstance);
schedulerDAO.storeJobInstanceRun(run2);
@@ -184,15 +184,15 @@ public class SchedulerDAOTest {
XJob job = getTestJob();
long currentTime = System.currentTimeMillis();
SchedulerJobInfo info = new SchedulerJobInfo(SchedulerJobHandle.fromString(UUID.randomUUID().toString()), job,
- "lens", SchedulerJobState.NEW, currentTime, currentTime);
+ "lens", SchedulerJobState.NEW, currentTime, currentTime);
// Store the job
schedulerDAO.storeJob(info);
info = new SchedulerJobInfo(SchedulerJobHandle.fromString(UUID.randomUUID().toString()), job, "lens",
- SchedulerJobState.NEW, currentTime, currentTime);
+ SchedulerJobState.NEW, currentTime, currentTime);
schedulerDAO.storeJob(info);
// There should be 3 jobs till now.
- Assert.assertEquals(schedulerDAO.getJobs("lens", null, null, null).size(), 3);
- Assert.assertEquals(schedulerDAO.getJobs("lens", SchedulerJobState.NEW, 1L, System.currentTimeMillis()).size(), 2);
- Assert.assertEquals(schedulerDAO.getJobs("Alice", SchedulerJobState.NEW, null, null).size(), 0);
+ Assert.assertEquals(schedulerDAO.getJobs("lens", null, null, SchedulerJobState.values()).size(), 3);
+ Assert.assertEquals(schedulerDAO.getJobs("lens", 1L, System.currentTimeMillis(), SchedulerJobState.NEW).size(), 2);
+ Assert.assertEquals(schedulerDAO.getJobs("Alice", null, null, SchedulerJobState.NEW).size(), 0);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerRestartTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerRestartTest.java b/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerRestartTest.java
index de36499..03f5d9b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerRestartTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/scheduler/SchedulerRestartTest.java
@@ -120,5 +120,7 @@ public class SchedulerRestartTest {
Assert.assertEquals(previousInstanceInfo.getInstanceRunList().get(0).getResultPath(), "/tmp/path");
Assert.assertEquals(previousInstanceInfo.getInstanceRunList().get(0).getInstanceState(),
SchedulerJobInstanceState.SUCCEEDED);
+ scheduler.expireJob(sessionHandle, jobHandle);
+ scheduler.closeSession(sessionHandle);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/test/java/org/apache/lens/server/scheduler/TestSchedulerServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/scheduler/TestSchedulerServiceImpl.java b/lens-server/src/test/java/org/apache/lens/server/scheduler/TestSchedulerServiceImpl.java
index aa5c897..2d40c49 100644
--- a/lens-server/src/test/java/org/apache/lens/server/scheduler/TestSchedulerServiceImpl.java
+++ b/lens-server/src/test/java/org/apache/lens/server/scheduler/TestSchedulerServiceImpl.java
@@ -29,7 +29,9 @@ import org.apache.lens.server.EventServiceImpl;
import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.scheduler.SchedulerService;
+import org.apache.lens.server.error.LensSchedulerErrorCode;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
@@ -51,8 +53,6 @@ public class TestSchedulerServiceImpl {
System.setProperty(LensConfConstants.CONFIG_LOCATION, "target/test-classes/");
}
-
-
@Test(priority = 1)
public void testScheduler() throws Exception {
LensServices.get().init(LensServerConf.getHiveConf());
@@ -159,4 +159,25 @@ public class TestSchedulerServiceImpl {
Assert.assertEquals(scheduler.getSchedulerDAO().getJobState(jobHandle), SchedulerJobState.EXPIRED);
scheduler.closeSession(sessionHandle);
}
+
+ @Test(priority = 2)
+ public void testAdminControl() throws Exception {
+ long currentTime = System.currentTimeMillis();
+ XJob job = getTestJob("0/5 * * * * ?", queryString, currentTime, currentTime + 180000);
+ LensSessionHandle sessionHandle = scheduler.openSessionAsUser(user);
+ SchedulerJobHandle jobHandle1 = scheduler.submitJob(sessionHandle, job);
+ SchedulerJobHandle jobHandle2 = scheduler.submitJob(sessionHandle, job);
+ SchedulerJobHandle jobHandle3 = scheduler.submitJob(sessionHandle, job);
+ // Fourth should throw an error.
+ try {
+ SchedulerJobHandle jobHandle = scheduler.submitJob(sessionHandle, job);
+ } catch (LensException e) {
+ Assert.assertEquals(e.getErrorCode(), 5009);
+ Assert.assertEquals(e.getErrorInfo().getErrorName(), LensSchedulerErrorCode.MAX_SCHEDULED_JOB_EXCEEDED.name());
+ }
+ scheduler.expireJob(sessionHandle, jobHandle1);
+ scheduler.expireJob(sessionHandle, jobHandle2);
+ scheduler.expireJob(sessionHandle, jobHandle3);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index 7d72928..d96659f 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -197,5 +197,9 @@
<name>lens.server.duplicate.query.allowed</name>
<value>true</value>
</property>
+ <property>
+ <name>lens.server.scheduler.max.job.per.user</name>
+ <value>3</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/6ca6157f/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 24d5fd6..a1d4d1a 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -191,88 +191,90 @@ Lens server configuration
*--+--+---+--+
|81|lens.server.savedquery.ws.resource.impl|org.apache.lens.server.query.save.SavedQueryResource|Implementation class for Saved query Resource|
*--+--+---+--+
-|82|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service|
+|82|lens.server.scheduler.max.job.per.user|-1|The maximum number of jobs per user which can be scheduled. -1 means unlimited jobs.|
*--+--+---+--+
-|83|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.|
+|83|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.SchedulerServiceImpl|Implementation class for query scheduler service|
*--+--+---+--+
-|84|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
+|84|lens.server.scheduler.store.class|org.apache.lens.server.scheduler.SchedulerDAO$SchedulerHsqlDBStore|A subclass of SchedulerDBStore class used for storing scheduler related information.|
*--+--+---+--+
-|85|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.|
+|85|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
*--+--+---+--+
-|86|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
+|86|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.|
*--+--+---+--+
-|87|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
+|87|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
*--+--+---+--+
-|88|lens.server.servicenames|session,alarm,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
+|88|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
*--+--+---+--+
-|89|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
+|89|lens.server.servicenames|session,alarm,query,savedquery,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|90|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|90|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
*--+--+---+--+
-|91|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|91|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|92|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|92|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|93|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|93|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|94|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.|
+|94|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|95|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds|
+|95|lens.server.state.persistence.enabled|true|If flag is enabled, state of all the services will be persisted periodically to a location specified by lens.server.persist.location and on server restart all the services will be started from last saved state.|
*--+--+---+--+
-|96|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|96|lens.server.state.persistence.interval.millis|300000|Lens server state persistence time interval in milliseconds|
*--+--+---+--+
-|97|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|97|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|98|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|98|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|99|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|99|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|100|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.|
+|100|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|101|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.|
+|101|lens.server.status.update.exponential.wait.millis|30000|Number of millis that would grow exponentially for next update, incase of transient failures.|
*--+--+---+--+
-|102|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.|
+|102|lens.server.status.update.maximum.delay.secs|1800|The maximum delay in seconds for next status update to happen after any transient failure. This will be used a maximum delay sothat exponential wait times not to grow to bigger value.|
*--+--+---+--+
-|103|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.|
+|103|lens.server.status.update.num.retries|10|The number of retries a status update will tried with exponentital back off, in case of transient issues, upon which query will be marked FAILED.|
*--+--+---+--+
-|104|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|104|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.|
*--+--+---+--+
-|105|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|105|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|106|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|106|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|107|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|107|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|108|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|108|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|109|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|109|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|110|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|110|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|111|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|111|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|112|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|112|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|113|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|113|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|114|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|114|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|115|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|115|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|116|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|116|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|117|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|117|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|118|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|118|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|119|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
+|119|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
-|120|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|120|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
*--+--+---+--+
-|121|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|121|lens.server.ws.featurenames|multipart,moxyjson,moxyjsonconfigresovler|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|122|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|122|lens.server.ws.filternames|requestlogger,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|123|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|123|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+*--+--+---+--+
+|124|lens.server.ws.resourcenames|session,metastore,query,savedquery,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values