You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2015/10/08 11:45:52 UTC
[3/3] lens git commit: LENS-446: Remove server params from session
and driver configurations
LENS-446: Remove server params from session and driver configurations
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/7080b76c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/7080b76c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/7080b76c
Branch: refs/heads/master
Commit: 7080b76c63a82e09b5d42c2d2b7458f80b1085f8
Parents: 02760a3
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Thu Oct 8 15:14:57 2015 +0530
Committer: Raju Bairishetti <ra...@im0933-x0.corp.inmobi.com>
Committed: Thu Oct 8 15:14:57 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/LensApplication.java | 2 +-
.../org/apache/lens/server/LensServerConf.java | 24 ++++++++++++++++----
.../server/query/QueryExecutionServiceImpl.java | 2 +-
.../lens/server/session/LensSessionImpl.java | 23 ++++++++++++++-----
.../lens/server/query/TestQueryService.java | 4 ++++
.../server/session/TestSessionResource.java | 9 +++++++-
6 files changed, 50 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
index a446ab6..71ff9ea 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
@@ -38,7 +38,7 @@ import lombok.extern.slf4j.Slf4j;
public class LensApplication extends Application {
/** The conf. */
- public static final Configuration CONF = LensServerConf.getConf();
+ public static final Configuration CONF = LensServerConf.getHiveConf();
@Override
public Set<Class<?>> getClasses() {
http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
index 4d333c6..6db720d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
@@ -18,6 +18,11 @@
*/
package org.apache.lens.server;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.lens.server.api.LensConfConstants;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -32,13 +37,22 @@ public final class LensServerConf {
private static final class ConfHolder {
public static final HiveConf HIVE_CONF = new HiveConf();
// configuration object which does not load defaults and loads only lens*.xml files.
- public static final Configuration CONF = new Configuration(false);
+ // and removes any server specific configuration, that should not be passed to drivers
+ private static final Configuration OVERRIDING_CONF_FOR_DRIVER = new Configuration(false);
static {
HIVE_CONF.addResource("lensserver-default.xml");
HIVE_CONF.addResource("lens-site.xml");
- CONF.addResource("lensserver-default.xml");
- CONF.addResource("lens-site.xml");
+ Configuration conf = new Configuration(false);
+ conf.addResource("lens-site.xml");
+ Iterator<Map.Entry<String, String>> confItr = conf.iterator();
+ while (confItr.hasNext()) {
+ Map.Entry<String, String> prop = confItr.next();
+ if (!prop.getKey().startsWith(LensConfConstants.SERVER_PFX)) {
+ OVERRIDING_CONF_FOR_DRIVER.set(prop.getKey(), prop.getValue());
+ }
+ }
+
}
}
@@ -57,8 +71,8 @@ public final class LensServerConf {
*
* @return the conf
*/
- public static Configuration getConf() {
- return ConfHolder.CONF;
+ public static Configuration getConfForDrivers() {
+ return ConfHolder.OVERRIDING_CONF_FOR_DRIVER;
}
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 73179e1..d57812f 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -338,7 +338,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
for (Class<?> driverClass : driverClasses) {
try {
LensDriver driver = (LensDriver) driverClass.newInstance();
- driver.configure(LensServerConf.getConf());
+ driver.configure(LensServerConf.getConfForDrivers());
if (driver instanceof HiveDriver) {
driver.registerDriverEventListener(driverEventListener);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
index d0677cc..9d8f198 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
@@ -65,7 +65,7 @@ public class LensSessionImpl extends HiveSessionImpl {
private long sessionTimeout;
/** The conf. */
- private Configuration conf = createDefaultConf();
+ private Configuration conf = new Configuration(createDefaultConf());
/**
* Keep track of DB static resources which failed to be added to this session
@@ -102,16 +102,27 @@ public class LensSessionImpl extends HiveSessionImpl {
persistInfo.setSessionConf(sessionConf);
}
+ private static Configuration sessionDefaultConfig;
/**
* Creates the default conf.
*
* @return the configuration
*/
- public static Configuration createDefaultConf() {
- Configuration conf = new Configuration(false);
- conf.addResource("lenssession-default.xml");
- conf.addResource("lens-site.xml");
- return conf;
+ public static synchronized Configuration createDefaultConf() {
+ if (sessionDefaultConfig == null) {
+ Configuration conf = new Configuration(false);
+ conf.addResource("lenssession-default.xml");
+ conf.addResource("lens-site.xml");
+ sessionDefaultConfig = new Configuration(false);
+ Iterator<Map.Entry<String, String>> confItr = conf.iterator();
+ while (confItr.hasNext()) {
+ Map.Entry<String, String> prop = confItr.next();
+ if (!prop.getKey().startsWith(LensConfConstants.SERVER_PFX)) {
+ sessionDefaultConfig.set(prop.getKey(), prop.getValue());
+ }
+ }
+ }
+ return sessionDefaultConfig;
}
/** The default hive session conf. */
http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index ea75ffb..6218882 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -1167,6 +1167,8 @@ public class TestQueryService extends LensJerseyTest {
assertNull(conf.get("hive.metastore.warehouse.dir"));
// core default should not be loaded
assertNull(conf.get("fs.default.name"));
+ // server configuration should not set
+ assertNull(conf.get("lens.server.persist.location"));
// Test server config. Hive configs overriden should be set
assertFalse(Boolean.parseBoolean(queryService.getHiveConf().get("hive.server2.log.redirection.enabled")));
@@ -1202,6 +1204,8 @@ public class TestQueryService extends LensJerseyTest {
assertEquals(dconf.get("lens.driver.test.key"), "set");
// core default should not be loaded
assertNull(dconf.get("fs.default.name"));
+ // server configuration should not set
+ assertNull(dconf.get("lens.server.persist.location"));
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/7080b76c/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java
index 8fffbc9..3055ce5 100644
--- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java
+++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionResource.java
@@ -169,7 +169,14 @@ public class TestSessionResource extends LensJerseyTest {
System.out.println("Session params:" + sessionParams.getElements());
Assert.assertEquals(sessionParams.getElements().size(), 1);
Assert.assertTrue(sessionParams.getElements().contains("my.conf=myvalue"));
-
+ // get server params on session
+ try {
+ paramtarget.queryParam("sessionid", handle).queryParam("key", "lens.server.persist.location").request()
+ .get(StringList.class);
+ Assert.fail("Expected 404");
+ } catch (Exception ne) {
+ Assert.assertTrue(ne instanceof NotFoundException);
+ }
// get all params verbose
sessionParams = paramtarget.queryParam("sessionid", handle).queryParam("verbose", true).request()
.get(StringList.class);