You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/04/15 21:50:10 UTC
[36/50] [abbrv] incubator-lens git commit: LENS-480 : Fix NPE while
running Hive MR jobs (amareshwari)
LENS-480 : Fix NPE while running Hive MR jobs (amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/8750e721
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/8750e721
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/8750e721
Branch: refs/heads/current-release-line
Commit: 8750e72169fa9b7c01b376bf165e9abcc9206e8d
Parents: 44f4664
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Thu Apr 9 06:46:45 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Apr 9 06:46:45 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/LensApplication.java | 4 +--
.../java/org/apache/lens/server/LensServer.java | 6 +---
.../org/apache/lens/server/LensServerConf.java | 31 +++++++++++++++++---
.../org/apache/lens/server/LensServices.java | 2 --
.../lens/server/LensServletContextListener.java | 2 +-
.../server/query/QueryExecutionServiceImpl.java | 3 +-
.../org/apache/lens/server/LensJerseyTest.java | 4 +--
.../org/apache/lens/server/TestLensServer.java | 2 +-
.../apache/lens/server/TestServiceProvider.java | 2 +-
.../lens/server/query/TestEventService.java | 2 +-
.../lens/server/query/TestQueryService.java | 16 ++++++++++
.../server/session/TestSessionClassLoaders.java | 2 +-
.../lens/server/session/TestSessionExpiry.java | 2 +-
.../server/stats/TestLogStatisticsStore.java | 2 +-
.../TestStatisticsLogPartitionHandler.java | 4 +--
.../lens/server/user/TestUserConfigLoader.java | 2 +-
.../src/test/resources/hivedriver-site.xml | 5 ++++
.../src/test/resources/jdbcdriver-site.xml | 5 ++++
18 files changed, 70 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
index dd3a4b3..cb452e8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
@@ -28,7 +28,7 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.conf.Configuration;
/**
* The Class LensApplication.
@@ -40,7 +40,7 @@ public class LensApplication extends Application {
public static final Log LOG = LogFactory.getLog(LensApplication.class);
/** The conf. */
- public static final HiveConf CONF = LensServerConf.get();
+ public static final Configuration CONF = LensServerConf.getConf();
@Override
public Set<Class<?>> getClasses() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/main/java/org/apache/lens/server/LensServer.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
index fe348a1..2f83fe1 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
@@ -62,9 +62,6 @@ public class LensServer {
@Getter
private final List<HttpServer> serverList = new ArrayList<HttpServer>();
- /** The conf. */
- final HiveConf conf;
-
/**
* This flag indicates that the lens server can run, When this is set to false, main thread bails out.
*/
@@ -87,7 +84,6 @@ public class LensServer {
* @throws IOException Signals that an I/O exception has occurred.
*/
private LensServer(HiveConf conf) throws IOException {
- this.conf = conf;
startServices(conf);
String baseURI = conf.get(LensConfConstants.SERVER_BASE_URL, LensConfConstants.DEFAULT_SERVER_BASE_URL);
HttpServer server = GrizzlyHttpServerFactory.createHttpServer(UriBuilder.fromUri(baseURI).build(), getApp(),
@@ -187,7 +183,7 @@ public class LensServer {
printStartupMessage();
try {
- final LensServer thisServer = LensServer.createLensServer(LensServerConf.get());
+ final LensServer thisServer = LensServer.createLensServer(LensServerConf.getHiveConf());
registerShutdownHook(thisServer);
registerDefaultExceptionHandler();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
index 38cd14f..4d333c6 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.server;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
/**
@@ -29,22 +30,44 @@ public final class LensServerConf {
}
private static final class ConfHolder {
- public static final HiveConf CONF = new HiveConf();
+ public static final HiveConf HIVE_CONF = new HiveConf();
+ // configuration object which does not load defaults and loads only lens*.xml files.
+ public static final Configuration CONF = new Configuration(false);
static {
+ HIVE_CONF.addResource("lensserver-default.xml");
+ HIVE_CONF.addResource("lens-site.xml");
CONF.addResource("lensserver-default.xml");
CONF.addResource("lens-site.xml");
}
}
/**
+ * The HiveConf object with lensserver-default.xml and lens-site.xml added.
+ *
* @return the hive conf
*/
- public static HiveConf get() {
+ public static HiveConf getHiveConf() {
+ return ConfHolder.HIVE_CONF;
+ }
+
+ /**
+ * The configuration object which does not load any defaults and loads only lens*.xml files. This is passed to
+ * all drivers in configure
+ *
+ * @return the conf
+ */
+ public static Configuration getConf() {
return ConfHolder.CONF;
}
- public static HiveConf create() {
- return new HiveConf(ConfHolder.CONF);
+ /**
+ * Creates a new configuration object from Server HiveConf, Creation should would be called usually from tests
+ * to modify some configurations.
+ *
+ * @return
+ */
+ public static HiveConf createHiveConf() {
+ return new HiveConf(ConfHolder.HIVE_CONF);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index 1484e47..f6cb365 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -143,8 +143,6 @@ public class LensServices extends CompositeService implements ServiceProvider {
public synchronized void init(HiveConf hiveConf) {
if (getServiceState() == STATE.NOTINITED) {
conf = hiveConf;
- conf.addResource("lensserver-default.xml");
- conf.addResource("lens-site.xml");
conf.setVar(HiveConf.ConfVars.HIVE_SESSION_IMPL_CLASSNAME, LensSessionImpl.class.getCanonicalName());
serviceMode = conf.getEnum(SERVER_MODE,
SERVICE_MODE.valueOf(DEFAULT_SERVER_MODE));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java b/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
index a801e3c..61a0dd8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
@@ -63,7 +63,7 @@ public class LensServletContextListener implements ServletContextListener {
}
// start up all lens services
- HiveConf conf = LensServerConf.get();
+ HiveConf conf = LensServerConf.getHiveConf();
LensServices services = LensServices.get();
services.init(conf);
services.start();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 3cfd03b..d531010 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -40,6 +40,7 @@ import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
import org.apache.lens.driver.cube.RewriteUtil;
import org.apache.lens.driver.hive.HiveDriver;
+import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.LensService;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
@@ -304,7 +305,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
Class<?> clazz = Class.forName(driverClass);
LensDriver driver = (LensDriver) clazz.newInstance();
- driver.configure(conf);
+ driver.configure(LensServerConf.getConf());
if (driver instanceof HiveDriver) {
driver.registerDriverEventListener(driverEventListener);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
index 1f3fe68..7913845 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
@@ -98,7 +98,7 @@ public abstract class LensJerseyTest extends JerseyTest {
}
public HiveConf getServerConf() {
- return LensServerConf.get();
+ return LensServerConf.getHiveConf();
}
/**
@@ -119,7 +119,7 @@ public abstract class LensJerseyTest extends JerseyTest {
LensTestUtil.createTestDatabaseResources(new String[]{LensTestUtil.DB_WITH_JARS, LensTestUtil.DB_WITH_JARS_2},
hiveConf);
- LensServices.get().init(LensServerConf.get());
+ LensServices.get().init(LensServerConf.getHiveConf());
LensServices.get().start();
// Check if mock service is started
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/TestLensServer.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestLensServer.java b/lens-server/src/test/java/org/apache/lens/server/TestLensServer.java
index d7ac06c..eb27f61 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestLensServer.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestLensServer.java
@@ -34,7 +34,7 @@ public class TestLensServer {
*/
@Test
public void testUIServer() throws IOException {
- HiveConf conf = new HiveConf(LensServerConf.get());
+ HiveConf conf = LensServerConf.createHiveConf();
LensServer thisServer = LensServer.createLensServer(conf);
Assert.assertEquals(thisServer.getServerList().size(), 2);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/TestServiceProvider.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServiceProvider.java b/lens-server/src/test/java/org/apache/lens/server/TestServiceProvider.java
index 5dfe780..af14fd6 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServiceProvider.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServiceProvider.java
@@ -42,7 +42,7 @@ public class TestServiceProvider extends LensAllApplicationJerseyTest {
*/
@Test
public void testServiceProvider() throws Exception {
- HiveConf conf = LensServerConf.get();
+ HiveConf conf = LensServerConf.getHiveConf();
Class<? extends ServiceProviderFactory> spfClass = conf.getClass(LensConfConstants.SERVICE_PROVIDER_FACTORY, null,
ServiceProviderFactory.class);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
index 3684af1..ec752ff 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
@@ -179,7 +179,7 @@ public class TestEventService {
*/
@BeforeTest
public void setup() throws Exception {
- LensServices.get().init(LensServerConf.get());
+ LensServices.get().init(LensServerConf.getHiveConf());
LensServices.get().start();
service = LensServices.get().getService(LensEventService.NAME);
assertNotNull(service);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index bd2aa4c..0ab68c7 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -1291,6 +1291,22 @@ public class TestQueryService extends LensJerseyTest {
Assert.assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader(),
ctx.getDriverContext().getDriverConf(queryService.getDrivers().iterator().next()).getClassLoader());
Assert.assertTrue(ctx.isDriverQueryExplicitlySet());
+ for (LensDriver driver : queryService.getDrivers()) {
+ Configuration dconf = ctx.getDriverConf(driver);
+ Assert.assertEquals(dconf.get("test.session.key"), "svalue");
+ // query specific conf
+ Assert.assertEquals(dconf.get("test.query.conf"), "qvalue");
+ // lenssession default should be loaded
+ Assert.assertNotNull(dconf.get("lens.query.enable.persistent.resultset"));
+ // lens site should be loaded
+ Assert.assertEquals(dconf.get("test.lens.site.key"), "gsvalue");
+ // hive default variables should not be set
+ Assert.assertNull(conf.get("hive.exec.local.scratchdir"));
+ // driver site should be loaded
+ Assert.assertEquals(dconf.get("lens.driver.test.key"), "set");
+ // core default should not be loaded
+ Assert.assertNull(dconf.get("fs.default.name"));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
index 6f6208a..7dd4486 100644
--- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
+++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
@@ -43,7 +43,7 @@ import org.testng.annotations.Test;
public class TestSessionClassLoaders {
private static final Log LOG = LogFactory.getLog(TestSessionClassLoaders.class);
- private final HiveConf conf = LensServerConf.create();
+ private final HiveConf conf = LensServerConf.createHiveConf();
private HiveSessionService sessionService;
private static final String DB1 = TestSessionClassLoaders.class.getSimpleName() + "_db1";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
index 6a30e90..c1cb9da 100644
--- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
+++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
@@ -43,7 +43,7 @@ public class TestSessionExpiry {
* @throws Exception the exception
*/
public void testSessionExpiry() throws Exception {
- HiveConf conf = LensServerConf.get();
+ HiveConf conf = LensServerConf.getHiveConf();
conf.setVar(HiveConf.ConfVars.HIVE_SESSION_IMPL_CLASSNAME, LensSessionImpl.class.getName());
conf.setLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, 1L);
CLIService cliService = new CLIService();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/stats/TestLogStatisticsStore.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/stats/TestLogStatisticsStore.java b/lens-server/src/test/java/org/apache/lens/server/stats/TestLogStatisticsStore.java
index ce74f9d..c8a0ea5 100644
--- a/lens-server/src/test/java/org/apache/lens/server/stats/TestLogStatisticsStore.java
+++ b/lens-server/src/test/java/org/apache/lens/server/stats/TestLogStatisticsStore.java
@@ -79,7 +79,7 @@ public class TestLogStatisticsStore {
@Test
public void testLogStatisticsStore() throws Exception {
LogStatisticsStore store = new LogStatisticsStore();
- store.initialize(LensServerConf.get());
+ store.initialize(LensServerConf.getHiveConf());
// Do some initialization work
StringWriter writer = new StringWriter();
Logger l = Logger.getLogger(MyLoggableLens.class);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogPartitionHandler.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogPartitionHandler.java b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogPartitionHandler.java
index 2084ced..f59131b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogPartitionHandler.java
+++ b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogPartitionHandler.java
@@ -89,7 +89,7 @@ public class TestStatisticsLogPartitionHandler {
@Test
public void testQueryExecutionStatisticsTableCreation() throws Exception {
QueryExecutionStatistics stats = new QueryExecutionStatistics(System.currentTimeMillis());
- HiveConf conf = LensServerConf.get();
+ HiveConf conf = LensServerConf.getHiveConf();
Table t = stats.getHiveTable(conf);
Hive h = getHiveClient(conf);
h.createTable(t);
@@ -117,7 +117,7 @@ public class TestStatisticsLogPartitionHandler {
*/
private HiveConf configureHiveTables() {
assertNotNull(System.getProperty("hadoop.bin.path"));
- HiveConf conf = LensServerConf.get();
+ HiveConf conf = LensServerConf.getHiveConf();
try {
Hive hive = getHiveClient(conf);
Database database = new Database();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
index 5187f2e..dd9351a 100644
--- a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
+++ b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
@@ -58,7 +58,7 @@ public class TestUserConfigLoader {
*/
@BeforeTest(alwaysRun = true)
public void init() {
- conf = new HiveConf(LensServerConf.get());
+ conf = new HiveConf(LensServerConf.getHiveConf());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/resources/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/hivedriver-site.xml b/lens-server/src/test/resources/hivedriver-site.xml
index e4208b4..cd9c655 100644
--- a/lens-server/src/test/resources/hivedriver-site.xml
+++ b/lens-server/src/test/resources/hivedriver-site.xml
@@ -72,4 +72,9 @@
<value>3</value>
</property>
+ <property>
+ <name>lens.driver.test.key</name>
+ <value>set</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8750e721/lens-server/src/test/resources/jdbcdriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/jdbcdriver-site.xml b/lens-server/src/test/resources/jdbcdriver-site.xml
index 752372a..1b14f54 100644
--- a/lens-server/src/test/resources/jdbcdriver-site.xml
+++ b/lens-server/src/test/resources/jdbcdriver-site.xml
@@ -47,4 +47,9 @@
<name>lens.driver.jdbc.explain.keyword</name>
<value>explain plan for</value>
</property>
+ <property>
+ <name>lens.driver.test.key</name>
+ <value>set</value>
+ </property>
+
</configuration>