You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by sh...@apache.org on 2015/12/30 08:10:47 UTC
[28/50] [abbrv] lens git commit: LENS-123 : Adds ability to load
different instances of same driver class
LENS-123 : Adds ability to load different instances of same driver class
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/114dab34
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/114dab34
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/114dab34
Branch: refs/heads/LENS-581
Commit: 114dab34642929152230cd049f1436ad796bedc2
Parents: 404d451
Author: Puneet Gupta <pu...@gmail.com>
Authored: Mon Nov 30 11:06:12 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Nov 30 11:06:12 2015 +0530
----------------------------------------------------------------------
.../lens/api/query/LensPreparedQuery.java | 8 +-
.../org/apache/lens/api/query/LensQuery.java | 4 +-
.../lens/cli/commands/LensQueryCommands.java | 2 +-
.../drivers/hive/hive1/hivedriver-site.xml | 80 ++++++++++
.../org/apache/lens/client/LensStatement.java | 2 +-
.../drivers/hive/hive1/hivedriver-site.xml | 80 ++++++++++
.../lens/driver/cube/TestMinCostSelector.java | 8 +-
.../org/apache/lens/driver/es/ESDriver.java | 13 +-
.../org/apache/lens/driver/es/ESDriverTest.java | 2 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 51 ++++---
.../apache/lens/driver/hive/TestHiveDriver.java | 4 +-
.../lens/driver/hive/TestRemoteHiveDriver.java | 16 +-
.../drivers/hive/hive1/hivedriver-site.xml | 49 +++++++
.../src/test/resources/hivedriver-site.xml | 49 -------
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 38 ++---
.../driver/jdbc/TestColumnarSQLRewriter.java | 6 +-
.../apache/lens/driver/jdbc/TestJDBCFinal.java | 2 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 12 +-
.../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 70 +++++++++
.../src/test/resources/jdbcdriver-site.xml | 70 ---------
.../drivers/hive/hive1/hivedriver-site.xml | 80 ++++++++++
lens-ml-lib/src/test/resources/lens-site.xml | 2 +-
.../lib/query/TestAbstractFileFormatter.java | 2 +-
.../lens/server/api/LensConfConstants.java | 20 ++-
.../server/api/driver/AbstractLensDriver.java | 71 +++++++++
.../lens/server/api/driver/LensDriver.java | 13 +-
.../server/api/query/AbstractQueryContext.java | 2 +-
.../api/query/DriverSelectorQueryContext.java | 4 +-
.../server/api/query/FinishedLensQuery.java | 15 +-
.../server/api/query/PreparedQueryContext.java | 7 +-
.../lens/server/api/query/QueryContext.java | 3 +-
.../lens/server/api/driver/MockDriver.java | 14 +-
.../lens/server/api/query/MockQueryContext.java | 2 +-
.../api/query/TestAbstractQueryContext.java | 4 +-
.../apache/lens/server/query/LensServerDAO.java | 10 +-
.../server/query/QueryExecutionServiceImpl.java | 146 ++++++++++++++-----
.../apache/lens/server/rewrite/RewriteUtil.java | 2 +-
.../lens/server/session/LensSessionImpl.java | 15 +-
.../src/main/resources/lensserver-default.xml | 2 +-
.../org/apache/lens/server/LensJerseyTest.java | 3 +
.../apache/lens/server/TestServerRestart.java | 10 +-
.../lens/server/query/TestEventService.java | 2 +
.../apache/lens/server/query/TestLensDAO.java | 2 +-
.../lens/server/query/TestQueryConstraints.java | 60 +-------
.../lens/server/query/TestQueryService.java | 38 +++--
.../lens/server/rewrite/TestRewriting.java | 17 ++-
.../drivers/hive/hive1/hivedriver-site.xml | 85 +++++++++++
.../drivers/hive/hive2/hivedriver-site.xml | 85 +++++++++++
.../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 58 ++++++++
.../mock/fail1/failing-query-driver-site.xml | 32 ++++
.../mockHive/mockHive1/hivedriver-site.xml | 95 ++++++++++++
.../mockHive/mockHive2/hivedriver-site.xml | 95 ++++++++++++
.../resources/failing-query-driver-site.xml | 27 ----
.../src/test/resources/hivedriver-site.xml | 80 ----------
.../src/test/resources/jdbcdriver-site.xml | 55 -------
lens-server/src/test/resources/lens-site.xml | 4 +-
src/site/apt/admin/config-server.apt | 37 ++++-
src/site/apt/admin/config.apt | 2 +-
src/site/apt/lenshome/install-and-run.apt | 18 ++-
.../drivers/hive/hive1/hivedriver-site.xml | 57 ++++++++
.../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 50 +++++++
.../server/hivedriver-site.xml | 57 --------
.../server/jdbcdriver-site.xml | 50 -------
tools/conf-pseudo-distr/server/lens-site.xml | 2 +-
.../drivers/hive/hive1/hivedriver-site.xml | 41 ++++++
.../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 50 +++++++
tools/conf/server/hivedriver-site.xml | 41 ------
tools/conf/server/jdbcdriver-site.xml | 50 -------
tools/conf/server/lens-site.xml | 2 +-
69 files changed, 1463 insertions(+), 722 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java
index 9595ce9..f5bc35d 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/LensPreparedQuery.java
@@ -50,8 +50,8 @@ import lombok.NoArgsConstructor;
* the prepared time
* @param preparedUser
* the prepared user
- * @param selectedDriverClassName
- * the selected driver class name
+ * @param selectedDriverName
+ * the selected driver's fully qualified name
* @param driverQuery
* the driver query
* @param conf
@@ -93,11 +93,11 @@ public class LensPreparedQuery {
private String preparedUser;
/**
- * The selected driver class name.
+ * The selected driver's fully qualified name.
*/
@XmlElement
@Getter
- private String selectedDriverClassName;
+ private String selectedDriverName;
/**
* The driver query.
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
index 204ecee..23c4dec 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
@@ -46,7 +46,7 @@ import lombok.*;
* the priority
* @param isPersistent
* the is persistent
- * @param selectedDriverClassName
+ * @param selectedDriverName
* the selected driver class name
* @param driverQuery
* the driver query
@@ -121,7 +121,7 @@ public class LensQuery {
*/
@XmlElement
@Getter
- private String selectedDriverClassName;
+ private String selectedDriverName;
/**
* The driver query.
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 83b919b..fe9b84d 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -364,7 +364,7 @@ public class LensQueryCommands extends BaseLensCommand {
.append("Prepare handle:").append(prepared.getPrepareHandle()).append("\n")
.append("User:" + prepared.getPreparedUser()).append("\n")
.append("Prepared at:").append(prepared.getPreparedTime()).append("\n")
- .append("Selected driver :").append(prepared.getSelectedDriverClassName()).append("\n")
+ .append("Selected driver :").append(prepared.getSelectedDriverName()).append("\n")
.append("Driver query:").append(prepared.getDriverQuery()).append("\n");
if (prepared.getConf() != null) {
sb.append("Conf:").append(prepared.getConf().getProperties()).append("\n");
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
new file mode 100644
index 0000000..f2aed88
--- /dev/null
+++ b/lens-cli/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>lens.driver.hive.connection.class</name>
+ <value>org.apache.lens.driver.hive.RemoteThriftConnection</value>
+ <description>The connection class from HiveDriver to HiveServer.</description>
+ </property>
+
+ <property>
+
+ <name>hive.metastore.local</name>
+
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${project.build.directory}/hive/warehouse</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>hive.lock.manager</name>
+ <value>org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.bind.host</name>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.port</name>
+ <value>12345</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.retry.limit</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.connect.retry.limit</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>lens.driver.test.key</name>
+ <value>set</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
index 0a511f0..40e6d76 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
@@ -213,7 +213,7 @@ public class LensStatement {
}
}
LensClient.getCliLooger().info("User query: '{}' was submitted to {}", query.getUserQuery(),
- query.getSelectedDriverClassName());
+ query.getSelectedDriverName());
if (query.getDriverQuery() != null) {
LensClient.getCliLooger().info(" Driver query: '{}' and Driver handle: {}", query.getDriverQuery(),
query.getDriverOpHandle());
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
new file mode 100644
index 0000000..f2aed88
--- /dev/null
+++ b/lens-client/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>lens.driver.hive.connection.class</name>
+ <value>org.apache.lens.driver.hive.RemoteThriftConnection</value>
+ <description>The connection class from HiveDriver to HiveServer.</description>
+ </property>
+
+ <property>
+
+ <name>hive.metastore.local</name>
+
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${project.build.directory}/hive/warehouse</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>hive.lock.manager</name>
+ <value>org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.bind.host</name>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.port</name>
+ <value>12345</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.retry.limit</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.connect.retry.limit</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>lens.driver.test.key</name>
+ <value>set</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
index 72f1497..8267229 100644
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
+++ b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestMinCostSelector.java
@@ -66,13 +66,13 @@ public class TestMinCostSelector {
LensConf qconf = new LensConf();
MockDriver d1 = new MockDriver();
- d1.configure(conf);
+ d1.configure(conf, null, null);
MockDriver d2 = new MockDriver();
- d2.configure(conf);
+ d2.configure(conf, null, null);
MockFailDriver fd1 = new MockFailDriver();
- fd1.configure(conf);
+ fd1.configure(conf, null, null);
MockFailDriver fd2 = new MockFailDriver();
- fd2.configure(conf);
+ fd2.configure(conf, null, null);
drivers.add(d1);
drivers.add(d2);
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
index 14d9f99..d166e43 100644
--- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
@@ -48,6 +48,7 @@ import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.commons.lang.Validate;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -62,13 +63,14 @@ import org.antlr.runtime.tree.Tree;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import lombok.extern.slf4j.Slf4j;
/**
* Driver for elastic search
*/
@Slf4j
-public class ESDriver implements LensDriver {
+public class ESDriver extends AbstractLensDriver {
private static final AtomicInteger THID = new AtomicInteger();
private static final double STREAMING_PARTITION_COST = 0;
@@ -317,10 +319,11 @@ public class ESDriver implements LensDriver {
}
@Override
- public void configure(Configuration conf) throws LensException {
+ public void configure(Configuration conf, String driverType, String driverName) throws LensException {
+ super.configure(conf, driverType, driverName);
this.conf = new Configuration(conf);
this.conf.addResource("esdriver-default.xml");
- this.conf.addResource("esdriver-site.xml");
+ this.conf.addResource(getDriverResourcePath("esdriver-site.xml"));
config = new ESDriverConfig(this.conf);
Class klass;
try {
@@ -341,10 +344,10 @@ public class ESDriver implements LensDriver {
| InstantiationException
| IllegalAccessException
| InvocationTargetException e) {
- log.error("ES driver cannot start!", e);
+ log.error("ES driver {} cannot start!", getFullyQualifiedName(), e);
throw new LensException("Cannot start es driver", e);
}
- log.debug("ES Driver configured");
+ log.info("ES Driver {} configured", getFullyQualifiedName());
asyncQueryPool = Executors.newCachedThreadPool(new ThreadFactory() {
@Override
public Thread newThread(Runnable runnable) {
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java
index f453416..ab6f22b 100644
--- a/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java
+++ b/lens-driver-es/src/test/java/org/apache/lens/driver/es/ESDriverTest.java
@@ -35,7 +35,7 @@ public abstract class ESDriverTest {
public void beforeTest() throws LensException {
initializeConfig(config);
esDriverConfig = new ESDriverConfig(config);
- driver.configure(config);
+ driver.configure(config, "es", "es1");
mockClientES = (MockClientES) driver.getESClient();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 19c4793..a84c679 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -50,6 +50,7 @@ import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
import org.apache.lens.server.api.query.priority.QueryPriorityDecider;
import org.apache.commons.lang.StringUtils;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -65,6 +66,7 @@ import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
import com.google.common.collect.ImmutableSet;
+
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -72,7 +74,7 @@ import lombok.extern.slf4j.Slf4j;
* The Class HiveDriver.
*/
@Slf4j
-public class HiveDriver implements LensDriver {
+public class HiveDriver extends AbstractLensDriver {
/** The Constant HIVE_CONNECTION_CLASS. */
public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
@@ -331,16 +333,18 @@ public class HiveDriver implements LensDriver {
* @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration)
*/
@Override
- public void configure(Configuration conf) throws LensException {
+ public void configure(Configuration conf, String driverType, String driverName) throws LensException {
+ super.configure(conf, driverType, driverName);
this.driverConf = new Configuration(conf);
+ String driverConfPath = getDriverResourcePath("hivedriver-site.xml");
this.driverConf.addResource("hivedriver-default.xml");
- this.driverConf.addResource("hivedriver-site.xml");
+ this.driverConf.addResource(driverConfPath);
// resources have to be added separately on hiveConf again because new HiveConf() overrides hive.* properties
// from HiveConf
this.hiveConf = new HiveConf(conf, HiveDriver.class);
this.hiveConf.addResource("hivedriver-default.xml");
- this.hiveConf.addResource("hivedriver-site.xml");
+ this.hiveConf.addResource(driverConfPath);
connectionClass = this.driverConf.getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class,
ThriftConnection.class);
@@ -366,6 +370,7 @@ public class HiveDriver implements LensDriver {
}
queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, driverConf);
selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, driverConf);
+ log.info("Hive driver {} configured successfully", getFullyQualifiedName());
}
private QueryCost calculateQueryCost(AbstractQueryContext qctx) throws LensException {
@@ -378,7 +383,7 @@ public class HiveDriver implements LensDriver {
@Override
public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
- log.info("Estimate: {}", qctx.getDriverQuery(this));
+ log.info("{} Estimate: {}", getFullyQualifiedName(), qctx.getDriverQuery(this));
if (qctx.getDriverQuery(this) == null) {
throw new NullPointerException("Null driver query for " + qctx.getUserQuery());
}
@@ -413,7 +418,7 @@ public class HiveDriver implements LensDriver {
// explain called again and again
return (HiveQueryPlan) explainCtx.getDriverContext().getDriverQueryPlan(this);
}
- log.info("Explain: {}", explainCtx.getDriverQuery(this));
+ log.info("{} Explain: {}", getFullyQualifiedName(), explainCtx.getDriverQuery(this));
Configuration explainConf = new Configuration(explainCtx.getDriverConf(this));
explainConf.setClassLoader(explainCtx.getConf().getClassLoader());
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
@@ -751,7 +756,7 @@ public class HiveDriver implements LensDriver {
*/
@Override
public void close() {
- log.info("CloseDriver");
+ log.info("CloseDriver {}", getFullyQualifiedName());
// Close this driver and release all resources
sessionLock.lock();
try {
@@ -812,8 +817,8 @@ public class HiveDriver implements LensDriver {
connection = new ExpirableConnection(tconn, connectionExpiryTimeout);
thriftConnExpiryQueue.offer(connection);
threadConnections.put(connectionKey, connection);
- log.info("New thrift connection {} for thread: {} for user: {} connection ID={}", connectionClass,
- Thread.currentThread().getId(), user, connection.getConnId());
+ log.info("New thrift connection {} for thread: {} for user: {} connection ID={} on driver:{}",
+ connectionClass, Thread.currentThread().getId(), user, connection.getConnId(), getFullyQualifiedName());
} catch (Exception e) {
throw new LensException(e);
}
@@ -885,7 +890,7 @@ public class HiveDriver implements LensDriver {
context.unSetDriverPersistent();
hiveQuery = context.getSelectedDriverQuery();
}
- log.info("Hive driver query:{}", hiveQuery);
+ log.info("Hive driver {} query:{}", getFullyQualifiedName(), hiveQuery);
context.setSelectedDriverQuery(hiveQuery);
}
@@ -914,14 +919,15 @@ public class HiveDriver implements LensDriver {
try {
hiveSession = getClient().openSession(ctx.getClusterUser(), "");
lensToHiveSession.put(sessionDbKey, hiveSession);
- log.info("New hive session for user: {} , lens session: {} hive session handle: {}", ctx.getClusterUser(),
- sessionDbKey, hiveSession.getHandleIdentifier());
+ log.info("New hive session for user: {} , lens session: {} , hive session handle: {} , driver : {}",
+ ctx.getClusterUser(), sessionDbKey, hiveSession.getHandleIdentifier(), getFullyQualifiedName());
for (LensEventListener<DriverEvent> eventListener : driverListeners) {
try {
eventListener.onEvent(new DriverSessionStarted(System.currentTimeMillis(), this, lensSession, hiveSession
.getSessionId().toString()));
} catch (Exception exc) {
- log.error("Error sending driver start event to listener {}", eventListener, exc);
+ log.error("Error sending driver {} start event to listener {}", getFullyQualifiedName(), eventListener,
+ exc);
}
}
} catch (Exception e) {
@@ -1006,7 +1012,7 @@ public class HiveDriver implements LensDriver {
return;
}
} catch (LensException e) {
- log.debug("query handle: {} Not yet launched on driver", handle);
+ log.debug("query handle: {} Not yet launched on driver {}", handle, getFullyQualifiedName());
}
Thread.sleep(pollInterval);
timeSpent += pollInterval;
@@ -1069,9 +1075,9 @@ public class HiveDriver implements LensDriver {
QueryHandle qhandle = (QueryHandle) in.readObject();
OperationHandle opHandle = new OperationHandle((TOperationHandle) in.readObject());
hiveHandles.put(qhandle, opHandle);
- log.debug("Hive driver recovered {}:{}", qhandle, opHandle);
+ log.debug("Hive driver {} recovered {}:{}", getFullyQualifiedName(), qhandle, opHandle);
}
- log.info("HiveDriver recovered {} queries", hiveHandles.size());
+ log.info("Hive driver {} recovered {} queries", getFullyQualifiedName(), hiveHandles.size());
int numSessions = in.readInt();
for (int i = 0; i < numSessions; i++) {
String lensId = in.readUTF();
@@ -1079,7 +1085,7 @@ public class HiveDriver implements LensDriver {
TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6);
lensToHiveSession.put(lensId, sHandle);
}
- log.info("HiveDriver recovered {} sessions", lensToHiveSession.size());
+ log.info("Hive driver {} recovered {} sessions", getFullyQualifiedName(), lensToHiveSession.size());
}
}
@@ -1096,15 +1102,15 @@ public class HiveDriver implements LensDriver {
for (Map.Entry<QueryHandle, OperationHandle> entry : hiveHandles.entrySet()) {
out.writeObject(entry.getKey());
out.writeObject(entry.getValue().toTOperationHandle());
- log.debug("Hive driver persisted {}:{}", entry.getKey(), entry.getValue());
+ log.debug("Hive driver {} persisted {}:{}", getFullyQualifiedName(), entry.getKey(), entry.getValue());
}
- log.info("HiveDriver persisted {} queries ", hiveHandles.size());
+ log.info("Hive driver {} persisted {} queries ", getFullyQualifiedName(), hiveHandles.size());
out.writeInt(lensToHiveSession.size());
for (Map.Entry<String, SessionHandle> entry : lensToHiveSession.entrySet()) {
out.writeUTF(entry.getKey());
out.writeObject(entry.getValue().toTSessionHandle());
}
- log.info("HiveDriver persisted {} sessions", lensToHiveSession.size());
+ log.info("Hive driver {} persisted {} sessions", getFullyQualifiedName(), lensToHiveSession.size());
}
}
@@ -1166,14 +1172,15 @@ public class HiveDriver implements LensDriver {
for (SessionHandle session : hiveSessionsToCheck) {
if (isSessionInvalid(exc, session)) {
// We have to expire previous session
- log.info("Hive server session {} for lens session {} has become invalid", session, lensSession);
+ log.info("{} Hive server session {} for lens session {} has become invalid", getFullyQualifiedName(), session,
+ lensSession);
sessionLock.lock();
try {
// We should close all connections and clear the session map since
// most likely all sessions are gone
closeAllConnections();
lensToHiveSession.clear();
- log.info("Cleared all sessions");
+ log.info("{} Cleared all sessions", getFullyQualifiedName());
} finally {
sessionLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 722a2da..11efd3c 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -112,13 +112,13 @@ public class TestHiveDriver {
protected void createDriver() throws LensException {
conf = new HiveConf();
- conf.addResource("hivedriver-site.xml");
+ conf.addResource("drivers/hive/hive1/hivedriver-site.xml");
conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
- driver.configure(conf);
+ driver.configure(conf, "hive", "hive1");
drivers = Lists.<LensDriver>newArrayList(driver);
System.out.println("TestHiveDriver created");
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 98edc28..ab5ada9 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -133,11 +133,11 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
protected void createDriver() throws LensException {
dataBase = TestRemoteHiveDriver.class.getSimpleName().toLowerCase();
conf = new HiveConf(remoteConf);
- conf.addResource("hivedriver-site.xml");
+ conf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driver = new HiveDriver();
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
- driver.configure(conf);
+ driver.configure(conf, "hive", "hive1");
drivers = Lists.<LensDriver>newArrayList(driver);
System.out.println("TestRemoteHiveDriver created");
}
@@ -155,7 +155,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
HiveConf thConf = new HiveConf(conf, TestRemoteHiveDriver.class);
thConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
final HiveDriver thrDriver = new HiveDriver();
- thrDriver.configure(thConf);
+ thrDriver.configure(thConf, "hive", "hive1");
QueryContext ctx = createContext("USE " + dataBase, conf, thrDriver);
thrDriver.execute(ctx);
@@ -235,12 +235,12 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
public void testHiveDriverPersistence() throws Exception {
System.out.println("@@@@ start_persistence_test");
HiveConf driverConf = new HiveConf(remoteConf, TestRemoteHiveDriver.class);
- driverConf.addResource("hivedriver-site.xml");
+ driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driverConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
final HiveDriver oldDriver = new HiveDriver();
- oldDriver.configure(driverConf);
+ oldDriver.configure(driverConf, "hive", "hive1");
driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
@@ -284,7 +284,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
ByteArrayInputStream driverInput = new ByteArrayInputStream(driverBytes.toByteArray());
HiveDriver newDriver = new HiveDriver();
newDriver.readExternal(new ObjectInputStream(driverInput));
- newDriver.configure(driverConf);
+ newDriver.configure(driverConf, "hive", "hive1");
driverInput.close();
ctx1 = readContext(ctx1bytes, newDriver);
@@ -311,7 +311,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
boolean isDriverAvailable = (ctx.getSelectedDriver() != null);
out.writeBoolean(isDriverAvailable);
if (isDriverAvailable) {
- out.writeUTF(ctx.getSelectedDriver().getClass().getName());
+ out.writeUTF(ctx.getSelectedDriver().getFullyQualifiedName());
}
} finally {
out.flush();
@@ -340,7 +340,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
ctx.setConf(driver.getConf());
boolean driverAvailable = in.readBoolean();
if (driverAvailable) {
- String clsName = in.readUTF();
+ String driverQualifiedName = in.readUTF();
ctx.setSelectedDriver(driver);
}
} finally {
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
new file mode 100644
index 0000000..613938d
--- /dev/null
+++ b/lens-driver-hive/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+
+ <name>hive.metastore.local</name>
+
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${project.build.directory}/hive/warehouse</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/lens-results</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-hive/src/test/resources/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/resources/hivedriver-site.xml b/lens-driver-hive/src/test/resources/hivedriver-site.xml
deleted file mode 100644
index 613938d..0000000
--- a/lens-driver-hive/src/test/resources/hivedriver-site.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<configuration>
-
- <property>
-
- <name>hive.metastore.local</name>
-
- <value>true</value>
- </property>
-
- <property>
- <name>hive.metastore.warehouse.dir</name>
- <value>${project.build.directory}/hive/warehouse</value>
- </property>
-
- <property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
- <description>JDBC connect string for a JDBC metastore</description>
- </property>
-
- <property>
- <name>lens.query.result.parent.dir</name>
- <value>target/lens-results</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index a8b980f..d3fa904 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -60,12 +60,14 @@ import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
+
+ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import com.google.common.collect.ImmutableSet;
+
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
@@ -75,7 +77,7 @@ import lombok.extern.slf4j.Slf4j;
* This driver is responsible for running queries against databases which can be queried using the JDBC API.
*/
@Slf4j
-public class JDBCDriver implements LensDriver {
+public class JDBCDriver extends AbstractLensDriver {
/** The Constant THID. */
public static final AtomicInteger THID = new AtomicInteger();
@@ -420,17 +422,17 @@ public class JDBCDriver implements LensDriver {
return conf;
}
- /**
- * Configure driver with {@link org.apache.hadoop.conf.Configuration} passed
+ /*
+ * (non-Javadoc)
*
- * @param conf The configuration object
- * @throws LensException the lens exception
+ * @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration)
*/
@Override
- public void configure(Configuration conf) throws LensException {
+ public void configure(Configuration conf, String driverType, String driverName) throws LensException {
+ super.configure(conf, driverType, driverName);
this.conf = new Configuration(conf);
this.conf.addResource("jdbcdriver-default.xml");
- this.conf.addResource("jdbcdriver-site.xml");
+ this.conf.addResource(getDriverResourcePath("jdbcdriver-site.xml"));
init(conf);
try {
queryHook = this.conf.getClass(
@@ -440,7 +442,7 @@ public class JDBCDriver implements LensDriver {
throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
}
configured = true;
- log.info("JDBC Driver configured");
+ log.info("JDBC Driver {} configured", getFullyQualifiedName());
}
/**
@@ -514,9 +516,9 @@ public class JDBCDriver implements LensDriver {
DummyQueryRewriter.class, QueryRewriter.class);
try {
rewriter = queryRewriterClass.newInstance();
- log.info("Initialized :{}", queryRewriterClass);
+ log.info("{} Initialized :{}", getFullyQualifiedName(), queryRewriterClass);
} catch (Exception e) {
- log.error("Unable to create rewriter object", e);
+ log.error("{} Unable to create rewriter object", getFullyQualifiedName(), e);
throw new LensException(e);
}
rewriter.init(conf);
@@ -636,7 +638,7 @@ public class JDBCDriver implements LensDriver {
explainQuery = rewrittenQuery.replaceAll("select ", "select "
+ explainKeyword + " ");
}
- log.info("Explain Query : {}", explainQuery);
+ log.info("{} Explain Query : {}", getFullyQualifiedName(), explainQuery);
QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null,
new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier(), false);
QueryResult result = null;
@@ -873,7 +875,7 @@ public class JDBCDriver implements LensDriver {
checkConfigured();
String rewrittenQuery = rewriteQuery(context);
- log.info("Execute {}", context.getQueryHandle());
+ log.info("{} Execute {}", getFullyQualifiedName(), context.getQueryHandle());
QueryResult result = executeInternal(context, rewrittenQuery);
return result.getLensResultSet(true);
@@ -919,7 +921,7 @@ public class JDBCDriver implements LensDriver {
throw new LensException("Query execution rejected: " + context.getQueryHandle() + " reason:" + e.getMessage(), e);
}
queryContextMap.put(context.getQueryHandle(), jdbcCtx);
- log.info("ExecuteAsync: {}", context.getQueryHandle());
+ log.info("{} ExecuteAsync: {}", getFullyQualifiedName(), context.getQueryHandle());
}
/**
@@ -1032,7 +1034,7 @@ public class JDBCDriver implements LensDriver {
context.setEndTime(System.currentTimeMillis());
}
context.closeResult();
- log.info("Cancelled query: {}", handle);
+ log.info("{} Cancelled query : {}", getFullyQualifiedName(), handle);
}
return cancelResult;
}
@@ -1053,7 +1055,7 @@ public class JDBCDriver implements LensDriver {
} finally {
queryContextMap.remove(handle);
}
- log.info("Closed query {}", handle.getHandleId());
+ log.info("{} Closed query {}", getFullyQualifiedName(), handle.getHandleId());
}
/**
@@ -1069,7 +1071,7 @@ public class JDBCDriver implements LensDriver {
try {
closeQuery(query);
} catch (LensException e) {
- log.warn("Error closing query : {}", query.getHandleId(), e);
+ log.warn("{} Error closing query : {}", getFullyQualifiedName(), query.getHandleId(), e);
}
}
for (QueryPrepareHandle query : preparedQueries.keySet()) {
@@ -1080,7 +1082,7 @@ public class JDBCDriver implements LensDriver {
throw new LensException();
}
} catch (LensException e) {
- log.warn("Error closing prapared query : {}", query, e);
+ log.warn("{} Error closing prapared query : {}", getFullyQualifiedName(), query, e);
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index cf795fa..c412cf0 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -135,7 +135,7 @@ public class TestColumnarSQLRewriter {
@BeforeTest
public void setup() throws Exception {
conf.addResource("jdbcdriver-default.xml");
- conf.addResource("jdbcdriver-site.xml");
+ conf.addResource("drivers/jdbc/jdbc1/jdbcdriver-site.xml");
qtest.init(conf);
List<FieldSchema> factColumns = new ArrayList<>();
@@ -435,8 +435,8 @@ public class TestColumnarSQLRewriter {
String actual = qtest.rewrite(query, conf, hconf);
String expected = "select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
- + "week((time_dim___time_dim . day )), "
- + "date(( time_dim___time_dim . day )), ( item_dim___item_dim . item_key ), "
+ + "week(( time_dim___time_dim . day )), "
+ + "date((time_dim___time_dim . day )), ( item_dim___item_dim . item_key ), "
+ "case when (sum(alias2) = 0 ) then 0.0 else sum(alias2) end as `dollars_sold` , "
+ "format(sum(alias3), 4 ), format(avg(alias4), '##################.###' ), "
+ "min(alias5), max(alias6) from (select sales_fact___fact.time_key, "
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
index 053e20d..4eee354 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJDBCFinal.java
@@ -75,7 +75,7 @@ public class TestJDBCFinal {
baseConf.set(JDBCDriverConfConstants.JDBC_EXPLAIN_KEYWORD_PARAM, "explain plan for ");
driver = new JDBCDriver();
- driver.configure(baseConf);
+ driver.configure(baseConf, "jdbc", "jdbc1");
assertNotNull(driver);
assertTrue(driver.configured);
System.out.println("Driver configured!");
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 425bd6f..2ad9fcb 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -88,7 +88,7 @@ public class TestJdbcDriver {
hConf = new HiveConf(baseConf, this.getClass());
driver = new JDBCDriver();
- driver.configure(baseConf);
+ driver.configure(baseConf, "jdbc", "jdbc1");
assertNotNull(driver);
assertTrue(driver.configured);
@@ -309,12 +309,12 @@ public class TestJdbcDriver {
metricConf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestJdbcDriver.class.getSimpleName());
driver.estimate(createExplainContext(query1, metricConf));
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
-
+ String driverQualifiledName = driver.getFullyQualifiedName();
Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-columnar-sql-rewrite",
- "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-jdbc-prepare-statement",
- "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-validate-thru-prepare",
- "lens.MethodMetricGauge.TestJdbcDriver-JDBCDriver-jdbc-check-allowed-query")));
+ "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-columnar-sql-rewrite",
+ "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-jdbc-prepare-statement",
+ "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-thru-prepare",
+ "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-jdbc-check-allowed-query")));
}
@Test
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml b/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
new file mode 100644
index 0000000..1202074
--- /dev/null
+++ b/lens-driver-jdbc/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>lens.driver.jdbc.estimate.db.user</name>
+ <value>estimateUser</value>
+ </property>
+
+ <!-- We have to use a different DB for estimate pool, because we are using an inmemory HSQL db.
+ There seem to be some issues regarding sharing of underlying inmemory db with different connection
+ pool. Whichever is constructed later is not able to get connections. -->
+ <property>
+ <name>lens.driver.jdbc.estimate.db.uri</name>
+ <value>jdbc:hsqldb:mem:jdbcTestDBEstimate</value>
+ </property>
+
+ <property>
+ <name>lens.driver.jdbc.estimate.connection.properties</name>
+ <value>random_key=random_value</value>
+ <description>Connection properties for jdbc connection.</description>
+ </property>
+
+
+ <property>
+ <name>lens.driver.jdbc.estimate.pool.max.size</name>
+ <value>50</value>
+ </property>
+
+ <property>
+ <name>lens.driver.jdbc.estimate.pool.idle.time</name>
+ <value>800</value>
+ </property>
+
+ <property>
+ <name>lens.driver.jdbc.estimate.get.connection.timeout</name>
+ <value>25000</value>
+ </property>
+
+ <property>
+ <name>lens.driver.jdbc.estimate.pool.max.statements</name>
+ <value>15</value>
+ </property>
+
+ <property>
+ <name>lens.driver.jdbc.regex.replacement.values</name>
+ <value>weekofyear=week, to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval$2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)</value>
+ <description>Rewriting the HQL to optimized sql queries</description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml b/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml
deleted file mode 100644
index 1202074..0000000
--- a/lens-driver-jdbc/src/test/resources/jdbcdriver-site.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<configuration>
- <property>
- <name>lens.driver.jdbc.estimate.db.user</name>
- <value>estimateUser</value>
- </property>
-
- <!-- We have to use a different DB for estimate pool, because we are using an inmemory HSQL db.
- There seem to be some issues regarding sharing of underlying inmemory db with different connection
- pool. Whichever is constructed later is not able to get connections. -->
- <property>
- <name>lens.driver.jdbc.estimate.db.uri</name>
- <value>jdbc:hsqldb:mem:jdbcTestDBEstimate</value>
- </property>
-
- <property>
- <name>lens.driver.jdbc.estimate.connection.properties</name>
- <value>random_key=random_value</value>
- <description>Connection properties for jdbc connection.</description>
- </property>
-
-
- <property>
- <name>lens.driver.jdbc.estimate.pool.max.size</name>
- <value>50</value>
- </property>
-
- <property>
- <name>lens.driver.jdbc.estimate.pool.idle.time</name>
- <value>800</value>
- </property>
-
- <property>
- <name>lens.driver.jdbc.estimate.get.connection.timeout</name>
- <value>25000</value>
- </property>
-
- <property>
- <name>lens.driver.jdbc.estimate.pool.max.statements</name>
- <value>15</value>
- </property>
-
- <property>
- <name>lens.driver.jdbc.regex.replacement.values</name>
- <value>weekofyear=week, to_date=date, format_number=format, date_sub\((.*?)\,\s*([0-9]+\s*)\)=date_sub($1\, interval$2 day), date_add\((.*?)\,\s*([0-9]+\s*)\)=date_add($1\, interval $2 day)</value>
- <description>Rewriting the HQL to optimized sql queries</description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
new file mode 100644
index 0000000..f2aed88
--- /dev/null
+++ b/lens-ml-lib/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+
+ <property>
+ <name>lens.driver.hive.connection.class</name>
+ <value>org.apache.lens.driver.hive.RemoteThriftConnection</value>
+ <description>The connection class from HiveDriver to HiveServer.</description>
+ </property>
+
+ <property>
+
+ <name>hive.metastore.local</name>
+
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>${project.build.directory}/hive/warehouse</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>hive.lock.manager</name>
+ <value>org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.bind.host</name>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.port</name>
+ <value>12345</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.retry.limit</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.connect.retry.limit</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <name>lens.driver.test.key</name>
+ <value>set</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-ml-lib/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/lens-site.xml b/lens-ml-lib/src/test/resources/lens-site.xml
index 3d5dbef..9be7850 100644
--- a/lens-ml-lib/src/test/resources/lens-site.xml
+++ b/lens-ml-lib/src/test/resources/lens-site.xml
@@ -25,7 +25,7 @@
<configuration>
<property>
<name>lens.server.drivers</name>
- <value>org.apache.lens.driver.hive.HiveDriver</value>
+ <value>hive:org.apache.lens.driver.hive.HiveDriver</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java b/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java
index 40e1cdc..c877516 100644
--- a/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java
+++ b/lens-query-lib/src/test/java/org/apache/lens/lib/query/TestAbstractFileFormatter.java
@@ -251,7 +251,7 @@ public abstract class TestAbstractFileFormatter {
protected QueryContext createContext(Configuration conf, String queryName) {
final LensDriver mockDriver = new MockDriver();
try {
- mockDriver.configure(conf);
+ mockDriver.configure(conf, null, null);
} catch (LensException e) {
Assert.fail(e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 7ee0749..88e5a01 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -52,9 +52,9 @@ public final class LensConfConstants {
public static final String METASTORE_PFX = "lens.metastore.";
/**
- * The Constant DRIVER_CLASSES.
+ * The Constant DRIVER_TYPES_AND_CLASSES
*/
- public static final String DRIVER_CLASSES = SERVER_PFX + "drivers";
+ public static final String DRIVER_TYPES_AND_CLASSES = SERVER_PFX + "drivers";
/**
* The Constant DRIVER_SELECTOR_CLASS.
*/
@@ -943,6 +943,21 @@ public final class LensConfConstants {
public static final int DEFAULT_FETCH_COUNT_SAVED_QUERY_LIST = 20;
/**
+ * This is the base directory where all drivers are available under lens-server's Conf directory.
+ */
+ public static final String DRIVERS_BASE_DIR = "drivers";
+
+ /**
+ * Name of the property that holds the path of "conf" directory of server
+ */
+ public static final String CONFIG_LOCATION = "config.location";
+
+ /**
+ * Default location of "conf" directory (wrt to lens-server/bin)
+ */
+ public static final String DEFAULT_CONFIG_LOCATION = "../conf";
+
+ /**
* The Constant RESULTSET_PURGE_ENABLED.
*/
public static final String RESULTSET_PURGE_ENABLED = SERVER_PFX + "resultset.purge.enabled";
@@ -981,5 +996,4 @@ public final class LensConfConstants {
* The Constant DEFAULT_HDFS_OUTPUT_RETENTION.
*/
public static final String DEFAULT_HDFS_OUTPUT_RETENTION = "1 day";
-
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
new file mode 100644
index 0000000..55f1535
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.commons.lang.StringUtils;
+
+import org.apache.hadoop.conf.Configuration;
+
+import lombok.Getter;
+
+/**
+ * Abstract class for Lens Driver Implementations. Provides default
+ * implementations and some utility methods for drivers
+ */
+public abstract class AbstractLensDriver implements LensDriver {
+ /**
+ * Separator used for constructing fully qualified name and driver resource path
+ */
+ private static final char SEPARATOR = '/';
+
+ /**
+ * Driver's fully qualified name ( Example hive/hive1, jdbc/mysql1)
+ */
+ @Getter
+ private String fullyQualifiedName = null;
+
+ @Override
+ public void configure(Configuration conf, String driverType, String driverName) throws LensException {
+ if (StringUtils.isBlank(driverType) || StringUtils.isBlank(driverName)) {
+ throw new LensException("Driver Type and Name can not be null or empty");
+ }
+ fullyQualifiedName = new StringBuilder(driverType).append(SEPARATOR).append(driverName).toString();
+ }
+
+ /**
+ * Gets the path (relative to lens server's conf location) for the driver resource in the system. This is a utility
+ * method that can be used by extending driver implementations to build path for their resources.
+ *
+ * @param resourceName
+ * @return
+ */
+ protected String getDriverResourcePath(String resourceName) {
+ return new StringBuilder(LensConfConstants.DRIVERS_BASE_DIR).append(SEPARATOR).append(getFullyQualifiedName())
+ .append(SEPARATOR).append(resourceName).toString();
+ }
+
+ @Override
+ public String toString() {
+ return getFullyQualifiedName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index a5a60d7..c220884 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -37,7 +37,6 @@ import com.google.common.collect.ImmutableSet;
* The Interface LensDriver.
*/
public interface LensDriver extends Externalizable {
-
/**
* Get driver configuration
*/
@@ -47,9 +46,11 @@ public interface LensDriver extends Externalizable {
* Configure driver with {@link Configuration} passed.
*
* @param conf The configuration object
+ * @param driverType Type of the driver (Example: hive, jdbc, el)
+ * @param driverName Name of this driver
* @throws LensException the lens exception
*/
- void configure(Configuration conf) throws LensException;
+ void configure(Configuration conf, String driverType, String driverName) throws LensException;
/**
* Estimate the cost of execution for given query.
@@ -199,4 +200,12 @@ public interface LensDriver extends Externalizable {
* null is never returned.
*/
ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies();
+
+
+ /**
+ * @return fully qualified name of this driver. This should be unique for each driver instance. This name can be used
+ * for referring to the driver while logging, persisting and restoring driver details,etc.
+ * (Examples: hive/hive1, jdbc/mysql1 )
+ */
+ String getFullyQualifiedName();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 0c980a2..2f20113 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -256,7 +256,7 @@ public abstract class AbstractQueryContext implements Serializable {
String expMsg = LensUtil.getCauseMessage(e);
driverQueryContext.setDriverQueryCostEstimateError(e);
failureCause = new StringBuilder("Driver :")
- .append(driver.getClass().getName())
+ .append(driver.getFullyQualifiedName())
.append(" Cause :")
.append(expMsg)
.toString();
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index feac938..5ff59bd 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -70,7 +70,7 @@ public class DriverSelectorQueryContext {
String metricId = ctx.driverSpecificConf.get(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
if (!StringUtils.isBlank(metricId)) {
ctx.driverSpecificConf.set(LensConfConstants.QUERY_METRIC_DRIVER_STACK_NAME,
- metricId + "-" + driver.getClass().getSimpleName());
+ metricId + "-" + driver.getFullyQualifiedName());
}
ctx.setQuery(userQuery);
driverQueryContextMap.put(driver, ctx);
@@ -198,7 +198,7 @@ public class DriverSelectorQueryContext {
log.error("Setting driver plan failed for driver {}", driver, e);
String expMsg = LensUtil.getCauseMessage(e);
driverQueryContext.setDriverQueryPlanGenerationError(e);
- detailedFailureCause.append("\n Driver :").append(driver.getClass().getName());
+ detailedFailureCause.append("\n Driver :").append(driver.getFullyQualifiedName());
detailedFailureCause.append(" Cause :" + expMsg);
if (failureCause != null && !failureCause.equals(expMsg)) {
useBuilder = true;
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index 89053aa..a57a6e4 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -159,11 +159,11 @@ public class FinishedLensQuery {
private String queryName;
/**
- * The selected driver class name.
+ * The selected driver's fully qualified name.
*/
@Getter
@Setter
- private String driverClass;
+ private String driverName;
@Getter
private LensDriver selectedDriver;
@@ -197,14 +197,14 @@ public class FinishedLensQuery {
}
this.selectedDriver = ctx.getSelectedDriver();
if (null != ctx.getSelectedDriver()) {
- this.driverClass = ctx.getSelectedDriver().getClass().getName();
+ this.driverName = ctx.getSelectedDriver().getFullyQualifiedName();
}
}
public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) {
- if (null == selectedDriver && null != driverClass) {
- selectedDriver = getDriverFromClassName(drivers);
+ if (null == selectedDriver && null != driverName) {
+ selectedDriver = getDriverFromName(drivers);
}
QueryContext qctx =
@@ -223,12 +223,11 @@ public class FinishedLensQuery {
return qctx;
}
- private LensDriver getDriverFromClassName(Collection<LensDriver> drivers) {
+ private LensDriver getDriverFromName(Collection<LensDriver> drivers) {
Iterator<LensDriver> iterator = drivers.iterator();
while (iterator.hasNext()) {
LensDriver driver = iterator.next();
- if (driverClass.equals(driver.getClass().getName())) {
- //TODO : LENS-123 - Ability to load different instances of same driver class
+ if (driverName.equals(driver.getFullyQualifiedName())) {
return driver;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
index b6f669b..0b08459 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
@@ -147,10 +147,9 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye
* @return the lens prepared query
*/
public LensPreparedQuery toPreparedQuery() {
- return new LensPreparedQuery(prepareHandle, userQuery, preparedTime, preparedUser,
- getDriverContext().getSelectedDriver() != null ? getDriverContext().getSelectedDriver().getClass()
- .getCanonicalName() : null, getDriverContext().getSelectedDriverQuery(),
- lensConf);
+ return new LensPreparedQuery(prepareHandle, userQuery, preparedTime, preparedUser, getDriverContext()
+ .getSelectedDriver() != null ? getDriverContext().getSelectedDriver().getFullyQualifiedName() : null,
+ getDriverContext().getSelectedDriverQuery(), lensConf);
}
public String getQueryHandleString() {
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 9b491d1..b637665 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -306,8 +306,7 @@ public class QueryContext extends AbstractQueryContext {
*/
public LensQuery toLensQuery() {
return new LensQuery(queryHandle, userQuery, super.getSubmittedUser(), priority, isPersistent,
- getSelectedDriver() != null ? getSelectedDriver().getClass()
- .getCanonicalName() : null,
+ getSelectedDriver() != null ? getSelectedDriver().getFullyQualifiedName() : null,
getSelectedDriverQuery(),
status,
resultSetPath, driverOpHandle, lensConf, submissionTime, launchTime, driverStatus.getDriverStartTime(),
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 2d86589..a20cf47 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -40,6 +40,7 @@ import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.hadoop.conf.Configuration;
+
import org.apache.hive.service.cli.ColumnDescriptor;
import com.beust.jcommander.internal.Sets;
@@ -48,7 +49,7 @@ import com.google.common.collect.ImmutableSet;
/**
* The Class MockDriver.
*/
-public class MockDriver implements LensDriver {
+public class MockDriver extends AbstractLensDriver {
private static AtomicInteger mockDriverId = new AtomicInteger();
/**
@@ -77,7 +78,7 @@ public class MockDriver implements LensDriver {
@Override
public String toString() {
- return "MockDriver:" + driverId;
+ return getFullyQualifiedName()+":"+driverId;
}
@Override
@@ -91,10 +92,15 @@ public class MockDriver implements LensDriver {
* @see org.apache.lens.server.api.driver.LensDriver#configure(org.apache.hadoop.conf.Configuration)
*/
@Override
- public void configure(Configuration conf) throws LensException {
+ public void configure(Configuration conf, String driverType, String driverName) throws LensException {
this.conf = conf;
ioTestVal = conf.getInt("mock.driver.test.val", -1);
- this.conf.addResource("failing-query-driver-site.xml");
+ this.conf.addResource(getDriverResourcePath("failing-query-driver-site.xml"));
+ }
+
+ @Override
+ public String getFullyQualifiedName() {
+ return "mock/fail1";
}
/**
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
index fd6b560..7768917 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
@@ -54,7 +54,7 @@ public class MockQueryContext extends QueryContext {
public static List<LensDriver> getDrivers(Configuration conf) throws LensException {
List<LensDriver> drivers = Lists.newArrayList();
MockDriver d = new MockDriver();
- d.configure(conf);
+ d.configure(conf, null, null);
drivers.add(d);
return drivers;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
index 02b652e..a37a4c8 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java
@@ -53,7 +53,7 @@ public class TestAbstractQueryContext {
String uniqueMetridId = ctx.getConf().get(QUERY_METRIC_UNIQUE_ID_CONF_KEY);
assertNotNull(uniqueMetridId);
assertEquals(ctx.getSelectedDriverConf().get(QUERY_METRIC_DRIVER_STACK_NAME),
- uniqueMetridId + "-" + MockDriver.class.getSimpleName());
+ uniqueMetridId + "-" + new MockDriver().getFullyQualifiedName());
}
@Test
@@ -70,7 +70,7 @@ public class TestAbstractQueryContext {
ctx.estimateCostForDrivers();
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestAbstractQueryContext-MockDriver-driverEstimate")));
+ "lens.MethodMetricGauge.TestAbstractQueryContext-"+new MockDriver().getFullyQualifiedName()+"-driverEstimate")));
}
@Test
http://git-wip-us.apache.org/repos/asf/lens/blob/114dab34/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index b9dd286..d8e654d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -85,7 +85,7 @@ public class LensServerDAO {
+ "userquery varchar(10000) not null," + "submitter varchar(255) not null," + "starttime bigint, "
+ "endtime bigint," + "result varchar(255)," + "status varchar(255), " + "metadata varchar(100000), "
+ "rows int, " + "filesize bigint, " + "errormessage varchar(10000), " + "driverstarttime bigint, "
- + "driverendtime bigint, " + "driverclass varchar(10000), "
+ + "driverendtime bigint, " + "drivername varchar(10000), "
+ "queryname varchar(255), " + "submissiontime bigint" + ")";
try {
QueryRunner runner = new QueryRunner(ds);
@@ -109,7 +109,7 @@ public class LensServerDAO {
Connection conn = null;
String sql = "insert into finished_queries (handle, userquery,submitter,"
+ "starttime,endtime,result,status,metadata,rows,filesize,"
- + "errormessage,driverstarttime,driverendtime, driverclass, queryname, submissiontime)"
+ + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime)"
+ " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
try {
conn = getConnection();
@@ -117,7 +117,7 @@ public class LensServerDAO {
runner.update(conn, sql, query.getHandle(), query.getUserQuery(), query.getSubmitter(), query.getStartTime(),
query.getEndTime(), query.getResult(), query.getStatus(), query.getMetadata(), query.getRows(),
query.getFileSize(), query.getErrorMessage(), query.getDriverStartTime(), query.getDriverEndTime(),
- query.getDriverClass(), query.getQueryName(), query.getSubmissionTime());
+ query.getDriverName(), query.getQueryName(), query.getSubmissionTime());
conn.commit();
} finally {
DbUtils.closeQuietly(conn);
@@ -158,7 +158,7 @@ public class LensServerDAO {
*
* @param state the state
* @param user the user
- * @param driverName the driverClass
+ * @param driverName the driver's fully qualified Name
* @param queryName the query name
* @param fromDate the from date
* @param toDate the to date
@@ -192,7 +192,7 @@ public class LensServerDAO {
}
if (StringUtils.isNotBlank(driverName)) {
- filters.add("lower(driverclass)=?");
+ filters.add("lower(drivername)=?");
params.add(driverName.toLowerCase());
}