You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/06/09 08:38:15 UTC
[2/3] incubator-lens git commit: LENS-294: Make SessionExpiryService
run period configurable
LENS-294: Make SessionExpiryService run period configurable
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/10d647ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/10d647ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/10d647ff
Branch: refs/heads/master
Commit: 10d647fff0dfcefe8ac6c46d0ce6a3876f9c4ef1
Parents: 713104d
Author: Raju Bairishetti <ra...@inmobi.com>
Authored: Tue Jun 9 12:05:48 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:05:48 2015 +0530
----------------------------------------------------------------------
.../lens/server/api/LensConfConstants.java | 8 +++
.../lens/server/session/HiveSessionService.java | 15 ++++-
.../src/main/resources/lensserver-default.xml | 5 ++
.../lens/server/session/TestSessionExpiry.java | 45 ++++++++++++-
src/site/apt/admin/config.apt | 66 ++++++++++----------
5 files changed, 105 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index e53e66c..ad27d78 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -712,6 +712,14 @@ public final class LensConfConstants {
*/
public static final String AUX_JARS = SESSION_PFX + "aux.jars";
+ /**
+ * Interval at which lens session expiry service runs
+ */
+ public static final String SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS = SERVER_PFX
+ + "session.expiry.serivce.interval.secs";
+
+ public static final int DEFAULT_SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS = 3600;
+
// Statistics Store configuration keys
/**
* The Constant STATS_STORE_CLASS.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index 6dc4ea6..fa061ec 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -81,6 +81,11 @@ public class HiveSessionService extends LensService implements SessionService {
private DatabaseResourceService databaseResourceService;
/**
+ * The conf.
+ */
+ private Configuration conf;
+
+ /**
* Instantiates a new hive session service.
*
* @param cliService the cli service
@@ -334,6 +339,7 @@ public class HiveSessionService extends LensService implements SessionService {
public synchronized void init(HiveConf hiveConf) {
this.databaseResourceService = new DatabaseResourceService(DatabaseResourceService.NAME);
addService(this.databaseResourceService);
+ this.conf = hiveConf;
super.init(hiveConf);
}
@@ -347,7 +353,9 @@ public class HiveSessionService extends LensService implements SessionService {
super.start();
sessionExpiryThread = Executors.newSingleThreadScheduledExecutor();
- sessionExpiryThread.scheduleWithFixedDelay(sessionExpiryRunnable, 60, 60, TimeUnit.MINUTES);
+ int sessionExpiryInterval = getSessionExpiryInterval();
+ sessionExpiryThread.scheduleWithFixedDelay(sessionExpiryRunnable, sessionExpiryInterval,
+ sessionExpiryInterval, TimeUnit.SECONDS);
// Restore sessions if any
if (restorableSessions == null || restorableSessions.size() <= 0) {
@@ -392,6 +400,11 @@ public class HiveSessionService extends LensService implements SessionService {
log.info("Session service restoed " + restorableSessions.size() + " sessions");
}
+ private int getSessionExpiryInterval() {
+ return conf.getInt(LensConfConstants.SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS,
+ LensConfConstants.DEFAULT_SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS);
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index 1a3f82f..da00c0d 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -695,4 +695,9 @@
</description>
</property>
+ <property>
+ <name>lens.server.session.expiry.service.interval.secs</name>
+ <value>3600</value>
+ <description>Interval at which lens session expiry service runs</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
index f494fd5..52402a0 100644
--- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
+++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
@@ -29,14 +29,18 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.hadoop.hive.conf.HiveConf;
+
import org.apache.hive.service.cli.CLIService;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestSessionExpiry.
*/
@Test(groups = "unit-test")
+@Slf4j
public class TestSessionExpiry {
/**
@@ -45,7 +49,7 @@ public class TestSessionExpiry {
* @throws Exception the exception
*/
public void testSessionExpiry() throws Exception {
- HiveConf conf = LensServerConf.getHiveConf();
+ HiveConf conf = LensServerConf.createHiveConf();
conf.setVar(HiveConf.ConfVars.HIVE_SESSION_IMPL_CLASSNAME, LensSessionImpl.class.getName());
conf.setLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, 1L);
CLIService cliService = new CLIService();
@@ -64,6 +68,45 @@ public class TestSessionExpiry {
assertFalse(session.isActive());
// run the expiry thread
lensService.getSessionExpiryRunnable().run();
+ log.info("Keeping a sleep of 3 seconds to make sure SessionExpiryService gets enough time to close"
+ + " inactive sessions");
+ Thread.sleep(3000);
+ assertTrue(metricSvc.getTotalExpiredSessions() >= 1);
+ assertTrue(metricSvc.getTotalClosedSessions() >= 1);
+
+ try {
+ lensService.getSession(sessionHandle);
+ // should throw exception since session should be expired by now
+ fail("Expected get session to fail for session " + sessionHandle.getPublicId());
+ } catch (Exception e) {
+ // pass
+ }
+ } finally {
+ lensService.stop();
+ }
+ }
+
+ public void testSessionExpiryInterval() throws Exception {
+ HiveConf conf = LensServerConf.createHiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_SESSION_IMPL_CLASSNAME, LensSessionImpl.class.getName());
+ conf.setLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, 1L);
+ conf.setInt(LensConfConstants.SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS, 1);
+ CLIService cliService = new CLIService();
+ cliService.init(conf);
+ HiveSessionService lensService = new HiveSessionService(cliService);
+ lensService.init(conf);
+ lensService.start();
+ MetricsService metricSvc = (MetricsService) LensServices.get().getService(MetricsService.NAME);
+ try {
+ LensSessionHandle sessionHandle = lensService.openSession("foo", "bar", new HashMap<String, String>());
+ LensSessionImpl session = lensService.getSession(sessionHandle);
+ assertTrue(session.isActive());
+ session.setLastAccessTime(session.getLastAccessTime() - 2000
+ * conf.getLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, LensConfConstants.SESSION_TIMEOUT_SECONDS_DEFAULT));
+ assertFalse(session.isActive());
+ log.info("Keeping a sleep of 3 seconds to make sure SessionExpiryService is running and gets enough time to"
+ + "close inactive sessions");
+ Thread.sleep(3000);
assertTrue(metricSvc.getTotalExpiredSessions() >= 1);
assertTrue(metricSvc.getTotalClosedSessions() >= 1);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 0ff204b..b4e63bc 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -149,68 +149,70 @@ Lens server configuration
*--+--+---+--+
|61|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|62|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|62|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
*--+--+---+--+
-|63|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|63|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|64|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|64|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|65|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
+|65|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|66|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|66|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
*--+--+---+--+
-|67|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|67|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|68|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|68|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|69|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|69|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|70|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|70|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|71|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
+|71|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|72|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
+|72|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
*--+--+---+--+
-|73|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|73|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
*--+--+---+--+
-|74|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
+|74|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|75|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|75|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|76|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|76|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|77|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|77|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|78|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|78|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|79|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|79|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|80|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|80|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|81|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|81|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|82|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|82|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|83|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|83|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|84|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|84|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|85|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|85|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|86|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|86|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|87|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|87|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|88|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|88|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|89|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|89|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|90|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|90|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
-|91|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|91|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|92|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|92|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|93|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|93|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+*--+--+---+--+
+|94|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values