You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/02/02 13:11:03 UTC
[3/3] incubator-lens git commit: LENS-248 : Fix checkstyle issues in
lens-driver-hive and lens-driver-jdbc (Rajat Khandelwal via amareshwari)
LENS-248 : Fix checkstyle issues in lens-driver-hive and lens-driver-jdbc (Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d0baff6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d0baff6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d0baff6b
Branch: refs/heads/master
Commit: d0baff6b88ede1ab8f9e3d1af53b1b8fa21db7e7
Parents: a4f1862
Author: Amareshwari Sriramdasu <am...@inmobi.com>
Authored: Mon Feb 2 17:40:51 2015 +0530
Committer: Amareshwari Sriramdasu <am...@inmobi.com>
Committed: Mon Feb 2 17:40:51 2015 +0530
----------------------------------------------------------------------
lens-driver-hive/pom.xml | 29 +-
.../driver/hive/EmbeddedThriftConnection.java | 7 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 57 +-
.../lens/driver/hive/HiveInMemoryResultSet.java | 30 +-
.../driver/hive/HivePersistentResultSet.java | 24 +-
.../apache/lens/driver/hive/HiveQueryPlan.java | 40 +-
.../lens/driver/hive/HiveResultSetMetadata.java | 10 +-
.../driver/hive/RemoteThriftConnection.java | 13 +-
.../lens/driver/hive/ThriftConnection.java | 11 +-
.../DurationBasedQueryPriorityDecider.java | 59 +-
.../src/main/resources/hivedriver-default.xml | 135 ++---
.../apache/lens/driver/hive/TestHiveDriver.java | 262 ++++-----
.../lens/driver/hive/TestRemoteHiveDriver.java | 96 ++--
.../src/test/resources/hive-site.xml | 64 ++-
.../src/test/resources/hivedriver-site.xml | 42 +-
lens-driver-hive/testng.xml | 22 +-
lens-driver-hive/testutils/hadoop | 4 +-
lens-driver-jdbc/pom.xml | 33 +-
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 302 +++++-----
.../lens/driver/jdbc/ConnectionProvider.java | 12 +-
.../jdbc/DataSourceConnectionProvider.java | 45 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 229 +++-----
.../driver/jdbc/JDBCDriverConfConstants.java | 10 +-
.../apache/lens/driver/jdbc/JDBCResultSet.java | 40 +-
.../src/main/resources/jdbcdriver-default.xml | 157 ++---
.../driver/jdbc/TestColumnarSQLRewriter.java | 571 ++++++++++---------
.../jdbc/TestDataSourceConnectionProvider.java | 17 +-
.../apache/lens/driver/jdbc/TestJDBCFinal.java | 74 +--
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 71 +--
.../src/test/resources/hive-site.xml | 32 +-
.../apache/lens/server/TestServerRestart.java | 8 +-
31 files changed, 1227 insertions(+), 1279 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index 19cdb75..dccb974 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -71,6 +71,33 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${lens.checkstyle.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>checkstyle-check</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <consoleOutput>true</consoleOutput>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <configLocation>checkstyle.xml</configLocation>
+ <headerLocation>checkstyle-java-header.txt</headerLocation>
+ <failOnViolation>true</failOnViolation>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -123,4 +150,4 @@
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java
index 634ec56..4de5bd7 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/EmbeddedThriftConnection.java
@@ -20,10 +20,11 @@ package org.apache.lens.driver.hive;
import java.io.IOException;
+import org.apache.lens.api.LensException;
+
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService;
import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient;
-import org.apache.lens.api.LensException;
/**
* The Class EmbeddedThriftConnection.
@@ -38,7 +39,7 @@ public class EmbeddedThriftConnection implements ThriftConnection {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.lens.driver.hive.ThriftConnection#getClient(org.apache.hadoop.hive.conf.HiveConf)
*/
@Override
@@ -52,7 +53,7 @@ public class EmbeddedThriftConnection implements ThriftConnection {
/*
* (non-Javadoc)
- *
+ *
* @see java.io.Closeable#close()
*/
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 2a5f5b8..6da1884 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -22,7 +22,10 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.DelayQueue;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
@@ -30,18 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.TaskStatus;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hive.service.cli.*;
-import org.apache.hive.service.cli.thrift.TOperationHandle;
-import org.apache.hive.service.cli.thrift.TProtocolVersion;
-import org.apache.hive.service.cli.thrift.TSessionHandle;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensException;
import org.apache.lens.api.LensSessionHandle;
@@ -56,7 +47,20 @@ import org.apache.lens.server.api.priority.QueryPriorityDecider;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.TaskStatus;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.service.cli.*;
+import org.apache.hive.service.cli.thrift.TOperationHandle;
+import org.apache.hive.service.cli.thrift.TProtocolVersion;
+import org.apache.hive.service.cli.thrift.TSessionHandle;
import org.apache.log4j.Logger;
+
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
@@ -162,8 +166,8 @@ public class HiveDriver implements LensDriver {
}
}
- /** The Constant connectionCounter. */
- private static final AtomicInteger connectionCounter = new AtomicInteger();
+ /** The Constant CONNECTION_COUNTER. */
+ private static final AtomicInteger CONNECTION_COUNTER = new AtomicInteger();
/**
* The Class ExpirableConnection.
@@ -194,7 +198,7 @@ public class HiveDriver implements LensDriver {
public ExpirableConnection(ThriftConnection conn, long timeout) {
this.conn = conn;
this.timeout = timeout;
- connId = connectionCounter.incrementAndGet();
+ connId = CONNECTION_COUNTER.incrementAndGet();
accessTime = System.currentTimeMillis();
}
@@ -316,7 +320,7 @@ public class HiveDriver implements LensDriver {
final String explainQuery = "EXPLAIN EXTENDED " + explainCtx.getDriverQuery(this);
QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery,
- explainCtx.getSubmittedUser(), new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier());
+ explainCtx.getSubmittedUser(), new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier());
// Get result set of explain
HiveInMemoryResultSet inMemoryResultSet = (HiveInMemoryResultSet) execute(explainQueryCtx);
List<String> explainOutput = new ArrayList<String>();
@@ -340,8 +344,8 @@ public class HiveDriver implements LensDriver {
* (non-Javadoc)
*
* @see
- * org.apache.lens.server.api.driver.LensDriver#explainAndPrepare(org.apache.lens.server.api.query.PreparedQueryContext
- * )
+ * org.apache.lens.server.api.driver.LensDriver#explainAndPrepare
+ * (org.apache.lens.server.api.query.PreparedQueryContext)
*/
@Override
public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException {
@@ -751,11 +755,11 @@ public class HiveDriver implements LensDriver {
*/
void addPersistentPath(QueryContext context) throws IOException {
String hiveQuery;
- if (context.isDriverPersistent()
- && context.getConf().getBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE,
- LensConfConstants.DEFAULT_ADD_INSERT_OVEWRITE)
+ boolean addInsertOverwrite = context.getConf().getBoolean(
+ LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, LensConfConstants.DEFAULT_ADD_INSERT_OVEWRITE);
+ if (context.isDriverPersistent() && addInsertOverwrite
&& (context.getSelectedDriverQuery().startsWith("SELECT")
- || context.getSelectedDriverQuery().startsWith("select"))) {
+ || context.getSelectedDriverQuery().startsWith("select"))) {
// store persistent data into user specified location
// If absent, take default home directory
Path resultSetPath = context.getHDFSResultDir();
@@ -927,11 +931,12 @@ public class HiveDriver implements LensDriver {
* (non-Javadoc)
*
* @see
- * org.apache.lens.server.api.driver.LensDriver#registerForCompletionNotification(org.apache.lens.api.query.QueryHandle
- * , long, org.apache.lens.server.api.driver.QueryCompletionListener)
+ * org.apache.lens.server.api.driver.LensDriver#registerForCompletionNotification
+ * (org.apache.lens.api.query.QueryHandle, long, org.apache.lens.server.api.driver.QueryCompletionListener)
*/
@Override
- public void registerForCompletionNotification(QueryHandle handle, long timeoutMillis, QueryCompletionListener listener)
+ public void registerForCompletionNotification(
+ QueryHandle handle, long timeoutMillis, QueryCompletionListener listener)
throws LensException {
Thread th = new Thread(new QueryCompletionNotifier(handle, timeoutMillis, listener));
th.start();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
index 97e9fca..2c8ee78 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
@@ -23,16 +23,12 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import org.apache.hive.service.cli.CLIServiceClient;
-import org.apache.hive.service.cli.FetchOrientation;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.OperationHandle;
-import org.apache.hive.service.cli.RowSet;
-import org.apache.hive.service.cli.TableSchema;
import org.apache.lens.api.LensException;
import org.apache.lens.api.query.ResultRow;
-import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.driver.InMemoryResultSet;
+import org.apache.lens.server.api.driver.LensResultSetMetadata;
+
+import org.apache.hive.service.cli.*;
/**
* The Class HiveInMemoryResultSet.
@@ -69,17 +65,13 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
/**
* Instantiates a new hive in memory result set.
*
- * @param hiveHandle
- * the hive handle
- * @param client
- * the client
- * @param closeAfterFecth
- * the close after fecth
- * @throws HiveSQLException
- * the hive sql exception
+ * @param hiveHandle the hive handle
+ * @param client the client
+ * @param closeAfterFecth the close after fecth
+ * @throws HiveSQLException the hive sql exception
*/
public HiveInMemoryResultSet(OperationHandle hiveHandle, CLIServiceClient client, boolean closeAfterFecth)
- throws HiveSQLException {
+ throws HiveSQLException {
this.client = client;
this.opHandle = hiveHandle;
this.closeAfterFecth = closeAfterFecth;
@@ -89,7 +81,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.lens.server.api.driver.LensResultSet#size()
*/
@Override
@@ -108,7 +100,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext()
*/
@Override
@@ -134,7 +126,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.lens.server.api.driver.InMemoryResultSet#next()
*/
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
index 9db4538..7a6486b 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
@@ -18,12 +18,16 @@
*/
package org.apache.lens.driver.hive;
-import org.apache.hadoop.fs.Path;
-import org.apache.hive.service.cli.*;
import org.apache.lens.api.LensException;
import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.driver.PersistentResultSet;
+import org.apache.hadoop.fs.Path;
+import org.apache.hive.service.cli.CLIServiceClient;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.OperationHandle;
+import org.apache.hive.service.cli.TableSchema;
+
/**
* The Class HivePersistentResultSet.
*/
@@ -38,24 +42,20 @@ public class HivePersistentResultSet extends PersistentResultSet {
/**
* Instantiates a new hive persistent result set.
*
- * @param resultSetPath
- * the result set path
- * @param opHandle
- * the op handle
- * @param client
- * the client
- * @throws HiveSQLException
- * the hive sql exception
+ * @param resultSetPath the result set path
+ * @param opHandle the op handle
+ * @param client the client
+ * @throws HiveSQLException the hive sql exception
*/
public HivePersistentResultSet(Path resultSetPath, OperationHandle opHandle, CLIServiceClient client)
- throws HiveSQLException {
+ throws HiveSQLException {
this.path = resultSetPath;
this.metadata = client.getResultSetMetadata(opHandle);
}
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.lens.server.api.driver.LensResultSet#size()
*/
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
index 4bda55d..e4b5fcc 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
@@ -23,15 +23,16 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.lens.api.query.QueryCost;
+import org.apache.lens.api.query.QueryPrepareHandle;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.driver.DriverQueryPlan;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.lens.api.query.QueryCost;
-import org.apache.lens.api.query.QueryPrepareHandle;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.DriverQueryPlan;
/**
* The Class HiveQueryPlan.
@@ -87,22 +88,18 @@ public class HiveQueryPlan extends DriverQueryPlan {
/** CREATE TABLE if destination is a table */
CREATE
- };
+ }
/**
* Instantiates a new hive query plan.
*
- * @param explainOutput
- * the explain output
- * @param prepared
- * the prepared
- * @param metastoreConf
- * the metastore conf
- * @throws HiveException
- * the hive exception
+ * @param explainOutput the explain output
+ * @param prepared the prepared
+ * @param metastoreConf the metastore conf
+ * @throws HiveException the hive exception
*/
public HiveQueryPlan(List<String> explainOutput, QueryPrepareHandle prepared, HiveConf metastoreConf)
- throws HiveException {
+ throws HiveException {
setPrepareHandle(prepared);
setExecMode(ExecMode.BATCH);
setScanMode(ScanMode.PARTIAL_SCAN);
@@ -114,12 +111,9 @@ public class HiveQueryPlan extends DriverQueryPlan {
/**
* Extract plan details.
*
- * @param explainOutput
- * the explain output
- * @param metastoreConf
- * the metastore conf
- * @throws HiveException
- * the hive exception
+ * @param explainOutput the explain output
+ * @param metastoreConf the metastore conf
+ * @throws HiveException the hive exception
*/
private void extractPlanDetails(List<String> explainOutput, HiveConf metastoreConf) throws HiveException {
ParserState state = ParserState.BEGIN;
@@ -229,10 +223,8 @@ public class HiveQueryPlan extends DriverQueryPlan {
/**
* Next state.
*
- * @param tr
- * the tr
- * @param state
- * the state
+ * @param tr the tr
+ * @param state the state
* @return the parser state
*/
private ParserState nextState(String tr, ParserState state) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveResultSetMetadata.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveResultSetMetadata.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveResultSetMetadata.java
index eb42f3a..be444b2 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveResultSetMetadata.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveResultSetMetadata.java
@@ -18,12 +18,14 @@
*/
package org.apache.lens.driver.hive;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import org.apache.hive.service.cli.ColumnDescriptor;
+import java.util.List;
+
import org.apache.lens.server.api.driver.LensResultSetMetadata;
-import java.util.List;
+import org.apache.hive.service.cli.ColumnDescriptor;
+
+import lombok.NoArgsConstructor;
+import lombok.Setter;
/**
* The top level Result Set metadata class which is used by the jackson to
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
index 6cf5950..de4645c 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
@@ -18,13 +18,14 @@
*/
package org.apache.lens.driver.hive;
+import org.apache.lens.api.LensException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.cli.CLIServiceClient;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.thrift.RetryingThriftCLIServiceClient;
-import org.apache.lens.api.LensException;
/**
* Connect to a remote Hive Server 2 service to run driver queries.
@@ -49,7 +50,7 @@ public class RemoteThriftConnection implements ThriftConnection {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.lens.driver.hive.ThriftConnection#getClient(org.apache.hadoop.hive.conf.HiveConf)
*/
@Override
@@ -57,11 +58,11 @@ public class RemoteThriftConnection implements ThriftConnection {
if (!connected) {
try {
LOG.info("HiveDriver connecting to HiveServer @ "
- + conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST) + ":"
- + conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
+ + conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST) + ":"
+ + conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
hs2Client = RetryingThriftCLIServiceClient.newRetryingCLIServiceClient(conf);
LOG.info("HiveDriver connected to HiveServer @ " + conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST)
- + ":" + conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
+ + ":" + conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
} catch (HiveSQLException e) {
throw new LensException(e);
@@ -73,7 +74,7 @@ public class RemoteThriftConnection implements ThriftConnection {
/*
* (non-Javadoc)
- *
+ *
* @see java.io.Closeable#close()
*/
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/ThriftConnection.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/ThriftConnection.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/ThriftConnection.java
index 282c6c1..8bf92f2 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/ThriftConnection.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/ThriftConnection.java
@@ -20,9 +20,10 @@ package org.apache.lens.driver.hive;
import java.io.Closeable;
+import org.apache.lens.api.LensException;
+
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.cli.CLIServiceClient;
-import org.apache.lens.api.LensException;
/**
* The Interface ThriftConnection.
@@ -32,11 +33,9 @@ public interface ThriftConnection extends Closeable {
/**
* Gets the client.
*
- * @param conf
- * the conf
+ * @param conf the conf
* @return the client
- * @throws LensException
- * the lens exception
+ * @throws LensException the lens exception
*/
- public CLIServiceClient getClient(HiveConf conf) throws LensException;
+ CLIServiceClient getClient(HiveConf conf) throws LensException;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
index 91e67df..4f471cb 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
@@ -18,17 +18,17 @@
*/
package org.apache.lens.driver.hive.priority;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.lens.api.LensException;
import org.apache.lens.api.Priority;
import org.apache.lens.server.api.priority.CostToPriorityRangeConf;
import org.apache.lens.server.api.priority.QueryPriorityDecider;
import org.apache.lens.server.api.query.AbstractQueryContext;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
@@ -36,22 +36,23 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
/** Partition Weights for priority calculation based on selected partitions **/
- /** weight of monthly partition **/
+ /** weight of monthly partition * */
private float monthlyPartitionWeight;
- /** weight of daily partition **/
+ /** weight of daily partition * */
private float dailyPartitionWeight;
- /** weight of hourly partition **/
+ /** weight of hourly partition * */
private float hourlyPartitionWeight;
/**
* Constructor. Takes three weights for partitions.
+ *
* @param ranges
* @param monthlyPartitoinWeight
* @param dailyPartitionWeight
* @param hourlyPartitionWeight
*/
public DurationBasedQueryPriorityDecider(String ranges,
- float monthlyPartitoinWeight, float dailyPartitionWeight, float hourlyPartitionWeight){
+ float monthlyPartitoinWeight, float dailyPartitionWeight, float hourlyPartitionWeight) {
this.costToPriorityRangeMap = new CostToPriorityRangeConf(ranges);
this.monthlyPartitionWeight = monthlyPartitoinWeight;
this.dailyPartitionWeight = dailyPartitionWeight;
@@ -60,10 +61,10 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
/**
* The Implementation
+ *
* @param abstractQueryContext
* @return decided Priority
- * @throws LensException
- * Exception occurs mostly when one of drivers/explained queries/plans is null
+ * @throws LensException Exception occurs mostly when one of drivers/explained queries/plans is null
*/
public Priority decidePriority(AbstractQueryContext abstractQueryContext) throws LensException {
float cost = getDurationCost(abstractQueryContext);
@@ -75,17 +76,17 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
/**
* Extract partitions from AbstractQueryContext. Hive currently gives partitions in the format
* {a:[dt partition1, dt partition2]...}. This method removes the "dt"
+ *
* @param queryContext
* @return all the tables along with their selected partitions.
* @throws LensException
*/
- protected Map<String,List<String>> extractPartitions(AbstractQueryContext queryContext) throws LensException{
+ protected Map<String, List<String>> extractPartitions(AbstractQueryContext queryContext) throws LensException {
Map<String, List<String>> partitions = new HashMap<String, List<String>>();
- for(Map.Entry<String, List<String>> entry: queryContext.getDriverContext().getSelectedDriverQueryPlan()
- .getPartitions().entrySet
- ()) {
+ for (Map.Entry<String, List<String>> entry : queryContext.getDriverContext().getSelectedDriverQueryPlan()
+ .getPartitions().entrySet()) {
partitions.put(entry.getKey(), new ArrayList<String>());
- for(String s: entry.getValue()) {
+ for (String s : entry.getValue()) {
String[] splits = s.split("\\s+");
partitions.get(entry.getKey()).add(splits[splits.length - 1]); //last split.
}
@@ -95,6 +96,7 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
/**
* Calculates total cost based on weights of selected tables and their selected partitions
+ *
* @param queryContext
* @return Query Cost
* @throws LensException
@@ -104,11 +106,11 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
final Map<String, List<String>> partitions = extractPartitions(queryContext);
LOG.info("partitions picked: " + partitions);
float cost = 0;
- for(String table: partitions.keySet()) {
- for(String partition: partitions.get(table)) {
- if(!partition.equals("latest")) {
+ for (String table : partitions.keySet()) {
+ for (String partition : partitions.get(table)) {
+ if (!partition.equals("latest")) {
cost += queryContext.getDriverContext().getSelectedDriverQueryPlan().getTableWeight(table)
- * getNormalizedPartitionCost(partition);
+ * getNormalizedPartitionCost(partition);
}
}
}
@@ -117,20 +119,21 @@ public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
/**
* Normalized cost of a partition. PartitionWeight multiplied by number of days in that partition.
+ *
* @param partition
* @return normalized cost.
* @throws LensException
*/
float getNormalizedPartitionCost(String partition) throws LensException {
switch (partition.length()) {
- case 7: //monthly
- return 30 * monthlyPartitionWeight;
- case 10: // daily
- return 1 * dailyPartitionWeight;
- case 13: // hourly
- return (1/24) * hourlyPartitionWeight;
- default:
- throw new LensException("Could not recognize partition: " + partition);
+ case 7: //monthly
+ return 30 * monthlyPartitionWeight;
+ case 10: // daily
+ return 1 * dailyPartitionWeight;
+ case 13: // hourly
+ return (1 / 24) * hourlyPartitionWeight;
+ default:
+ throw new LensException("Could not recognize partition: " + partition);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/main/resources/hivedriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml
index 11e7210..7d06fee 100644
--- a/lens-driver-hive/src/main/resources/hivedriver-default.xml
+++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml
@@ -23,72 +23,75 @@
<configuration>
-<property>
- <name>lens.driver.hive.connection.class</name>
- <value>org.apache.lens.driver.hive.EmbeddedThriftConnection</value>
- <description>The connection class from HiveDriver to HiveServer. The default is
- an embedded connection which does not require a remote hive server.
- For connecting to a hiveserver end point, remote connection should be used.
- The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection
- and org.apache.lens.driver.hive.RemoteThriftConnection. </description>
-</property>
-
-<property>
- <name>lens.driver.hive.hs2.connection.expiry.delay</name>
- <value>600000</value>
- <description>The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2</description>
-</property>
-
-<!-- Hive server client params -->
-
-<property>
- <name>hive.server2.thrift.bind.host</name>
- <value></value>
- <description>The host on which hive server is running</description>
-</property>
-
-<property>
- <name>hive.server2.thrift.port</name>
- <value>10000</value>
- <description>The port on which hive server is running</description>
-</property>
-
-<property>
- <name>hive.server2.thrift.client.retry.limit</name>
- <value>1</value>
- <description>Number of times to retry a Thrift service call upon failure </description>
-</property>
-
-<property>
- <name>hive.server2.thrift.client.connect.retry.limit</name>
- <value>1</value>
- <description>Number of times to retry a connection to a Thrift hive server</description>
-</property>
-
-<property>
- <name>hive.server2.thrift.client.retry.delay.seconds</name>
- <value>1</value>
- <description>Number of seconds the client should wait between connection attempts.</description>
-</property>
-
-<property>
- <name>hive.server.tcp.keepalive</name>
- <value>true</value>
- <description>TCP Keep alive socket option for HiveServer connection</description>
-</property>
-
-<property>
- <name>hive.server.read.socket.timeout</name>
- <value>10</value>
- <description>Socket timeout for the client connection</description>
-</property>
-
-<!-- query params -->
+ <property>
+ <name>lens.driver.hive.connection.class</name>
+ <value>org.apache.lens.driver.hive.EmbeddedThriftConnection</value>
+ <description>The connection class from HiveDriver to HiveServer. The default is
+ an embedded connection which does not require a remote hive server.
+ For connecting to a hiveserver end point, remote connection should be used.
+ The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection
+ and org.apache.lens.driver.hive.RemoteThriftConnection.
+ </description>
+ </property>
+
+ <property>
+ <name>lens.driver.hive.hs2.connection.expiry.delay</name>
+ <value>600000</value>
+ <description>The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2</description>
+ </property>
+
+ <!-- Hive server client params -->
+
+ <property>
+ <name>hive.server2.thrift.bind.host</name>
+ <value></value>
+ <description>The host on which hive server is running</description>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.port</name>
+ <value>10000</value>
+ <description>The port on which hive server is running</description>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.retry.limit</name>
+ <value>1</value>
+ <description>Number of times to retry a Thrift service call upon failure</description>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.connect.retry.limit</name>
+ <value>1</value>
+ <description>Number of times to retry a connection to a Thrift hive server</description>
+ </property>
+
+ <property>
+ <name>hive.server2.thrift.client.retry.delay.seconds</name>
+ <value>1</value>
+ <description>Number of seconds the client should wait between connection attempts.</description>
+ </property>
+
+ <property>
+ <name>hive.server.tcp.keepalive</name>
+ <value>true</value>
+ <description>TCP Keep alive socket option for HiveServer connection</description>
+ </property>
+
+ <property>
+ <name>hive.server.read.socket.timeout</name>
+ <value>10</value>
+ <description>Socket timeout for the client connection</description>
+ </property>
+
+ <!-- query params -->
<property>
<name>lens.cube.query.driver.supported.storages</name>
<value></value>
- <description>List of comma separated storage names that supported by a driver. If no value is specified, all storages are valid</description>
+ <description>List of comma separated storage names that supported by a driver. If no value is specified, all
+ storages are valid
+ </description>
</property>
<property>
@@ -118,10 +121,10 @@
The cost is calculated based on partition weights and fact weights. The interpretation of the
default config is: \
\
- cost \<= 7\ \ \ \ \ \ \ \ \ \ \ :\ \ \ \ \ Priority = VERY_HIGH \
- 7 \< cost \<= 30\ \ \ \ \ \ \ :\ \ \ \ \ Priority = HIGH \
- 30 \< cost \<= 90\ \ \ \ \ \ :\ \ \ \ \ Priority = NORMAL \
- 90 \< cost\ \ \ \ \ \ \ \ \ \ \ :\ \ \ \ \ Priority = LOW \
+ cost \<= 7\ \ \ \ \ \ \ \ \ \ \ :\ \ \ \ \ Priority = VERY_HIGH \
+ 7 \< cost \<= 30\ \ \ \ \ \ \ :\ \ \ \ \ Priority = HIGH \
+ 30 \< cost \<= 90\ \ \ \ \ \ :\ \ \ \ \ Priority = NORMAL \
+ 90 \< cost\ \ \ \ \ \ \ \ \ \ \ :\ \ \ \ \ Priority = LOW \
\
Some perspective wrt default weights and default ranges(1 for hourly, 0.75 for daily, 0.5 for monthly):\
For exclusively hourly data this translates to VERY_HIGH,7days,HIGH,30days,NORMAL,90days,LOW. \
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index d4a2280..0b118de 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -23,7 +23,18 @@ import static org.testng.Assert.*;
import java.io.*;
import java.util.*;
-import junit.framework.Assert;
+import org.apache.lens.api.LensConf;
+import org.apache.lens.api.LensException;
+import org.apache.lens.api.Priority;
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.driver.*;
+import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.ExplainQueryContext;
+import org.apache.lens.server.api.query.PreparedQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -37,23 +48,14 @@ import org.apache.hadoop.hive.ql.HiveDriverRunHookContext;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.ColumnDescriptor;
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensException;
-import org.apache.lens.api.Priority;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.*;
-import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.ExplainQueryContext;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
+
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import junit.framework.Assert;
+
/**
* The Class TestHiveDriver.
*/
@@ -63,7 +65,7 @@ public class TestHiveDriver {
public static final String TEST_DATA_FILE = "testdata/testdata1.data";
/** The test output dir. */
- public final String TEST_OUTPUT_DIR = "target/" + this.getClass().getSimpleName() + "/test-output";
+ private final String testOutputDir = "target/" + this.getClass().getSimpleName() + "/test-output";
/** The conf. */
protected HiveConf conf;
@@ -71,11 +73,11 @@ public class TestHiveDriver {
/** The driver. */
protected HiveDriver driver;
- /** Driver list **/
+ /** Driver list * */
protected Collection<LensDriver> drivers;
/** The data base. */
- public String DATA_BASE = this.getClass().getSimpleName().toLowerCase();
+ String dataBase = this.getClass().getSimpleName().toLowerCase();
protected String sessionid;
protected SessionState ss;
@@ -83,8 +85,7 @@ public class TestHiveDriver {
/**
* Before test.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@BeforeTest
public void beforeTest() throws Exception {
@@ -97,13 +98,13 @@ public class TestHiveDriver {
SessionState.start(ss);
Hive client = Hive.get(conf);
Database database = new Database();
- database.setName(DATA_BASE);
+ database.setName(dataBase);
client.createDatabase(database, true);
- SessionState.get().setCurrentDatabase(DATA_BASE);
+ SessionState.get().setCurrentDatabase(dataBase);
sessionid = SessionState.get().getSessionId();
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
- QueryContext context = createContext("USE " + DATA_BASE, conf);
+ QueryContext context = createContext("USE " + dataBase, conf);
driver.execute(context);
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
@@ -117,21 +118,28 @@ public class TestHiveDriver {
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
driver = new HiveDriver();
driver.configure(conf);
- drivers = new ArrayList<LensDriver>() {{ add
- (driver);}};
+ drivers = new ArrayList<LensDriver>() {
+ {
+ add(driver);
+ }
+ };
System.out.println("TestHiveDriver created");
}
@BeforeMethod
public void setDB() {
- SessionState.get().setCurrentDatabase(DATA_BASE);
+ SessionState.get().setCurrentDatabase(dataBase);
}
protected QueryContext createContext(final String query, Configuration conf) throws LensException {
QueryContext context = new QueryContext(query, "testuser", new LensConf(), conf, drivers);
// session id has to be set before calling setDriverQueriesAndPlans
context.setLensSessionIdentifier(sessionid);
- context.setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {{ put(driver, query); }} );
+ context.setDriverQueriesAndPlans(new HashMap<LensDriver, String>() {
+ {
+ put(driver, query);
+ }
+ });
context.setSelectedDriver(driver);
return context;
}
@@ -151,22 +159,19 @@ public class TestHiveDriver {
/**
* After test.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@AfterTest
public void afterTest() throws Exception {
driver.close();
- Hive.get(conf).dropDatabase(DATA_BASE, true, true, true);
+ Hive.get(conf).dropDatabase(dataBase, true, true, true);
}
/**
* Creates the test table.
*
- * @param tableName
- * the table name
- * @throws Exception
- * the exception
+ * @param tableName the table name
+ * @throws Exception the exception
*/
protected void createTestTable(String tableName) throws Exception {
System.out.println("Hadoop Location: " + System.getProperty("hadoop.bin.path"));
@@ -188,15 +193,14 @@ public class TestHiveDriver {
/**
* Creates the test table.
*
- * @param tableName
- * the table name
- * @throws Exception
- * the exception
+ * @param tableName the table name
+ * @throws Exception the exception
*/
protected void createPartitionedTable(String tableName) throws Exception {
System.out.println("Hadoop Location: " + System.getProperty("hadoop.bin.path"));
- String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)" + " PARTITIONED BY (dt string) TBLPROPERTIES ('"
- + LensConfConstants.STORAGE_COST + "'='500')";
+ String createTable = "CREATE TABLE IF NOT EXISTS " + tableName + "(ID STRING)"
+ + " PARTITIONED BY (dt string) TBLPROPERTIES ('"
+ + LensConfConstants.STORAGE_COST + "'='500')";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
// Craete again
QueryContext context = createContext(createTable, conf);
@@ -205,19 +209,19 @@ public class TestHiveDriver {
// Load some data into the table
String dataLoad = "LOAD DATA LOCAL INPATH '" + TEST_DATA_FILE + "' OVERWRITE INTO TABLE " + tableName
- + " partition (dt='today')";
+ + " partition (dt='today')";
context = createContext(dataLoad, conf);
resultSet = driver.execute(context);
assertNull(resultSet);
Assert.assertEquals(0, driver.getHiveHandleSize());
}
-
+
// Tests
+
/**
* Test insert overwrite conf.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testInsertOverwriteConf() throws Exception {
@@ -234,8 +238,7 @@ public class TestHiveDriver {
/**
* Test temptable.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testTemptable() throws Exception {
@@ -260,8 +263,7 @@ public class TestHiveDriver {
/**
* Test execute query.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testExecuteQuery() throws Exception {
@@ -278,9 +280,9 @@ public class TestHiveDriver {
resultSet = driver.execute(context);
validatePersistentResult(resultSet, TEST_DATA_FILE, context.getHDFSResultDir(), false);
conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT,
- "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
- + " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
- + " 'field.delim'=',' ) STORED AS TEXTFILE ");
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
+ + " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
+ + " 'field.delim'=',' ) STORED AS TEXTFILE ");
select = "SELECT ID, null, ID FROM test_execute";
context = createContext(select, conf);
resultSet = driver.execute(context);
@@ -291,12 +293,9 @@ public class TestHiveDriver {
/**
* Validate in memory result.
*
- * @param resultSet
- * the result set
- * @throws LensException
- * the lens exception
- * @throws IOException
- * Signals that an I/O exception has occurred.
+ * @param resultSet the result set
+ * @throws LensException the lens exception
+ * @throws IOException Signals that an I/O exception has occurred.
*/
private void validateInMemoryResult(LensResultSet resultSet) throws LensException, IOException {
validateInMemoryResult(resultSet, null);
@@ -305,14 +304,10 @@ public class TestHiveDriver {
/**
* Validate in memory result.
*
- * @param resultSet
- * the result set
- * @param outputTable
- * the output table
- * @throws LensException
- * the lens exception
- * @throws IOException
- * Signals that an I/O exception has occurred.
+ * @param resultSet the result set
+ * @param outputTable the output table
+ * @throws LensException the lens exception
+ * @throws IOException Signals that an I/O exception has occurred.
*/
private void validateInMemoryResult(LensResultSet resultSet, String outputTable) throws LensException, IOException {
assertNotNull(resultSet);
@@ -330,7 +325,7 @@ public class TestHiveDriver {
}
expectedCol += "ID";
assertTrue(columns.get(0).getName().toLowerCase().equals(expectedCol.toLowerCase())
- || columns.get(0).getName().toLowerCase().equals("ID".toLowerCase()));
+ || columns.get(0).getName().toLowerCase().equals("ID".toLowerCase()));
assertEquals(columns.get(0).getTypeName().toLowerCase(), "STRING".toLowerCase());
List<String> expectedRows = new ArrayList<String>();
@@ -357,7 +352,7 @@ public class TestHiveDriver {
/*
* (non-Javadoc)
- *
+ *
* @see
* org.apache.hadoop.hive.ql.HiveDriverRunHook#postDriverRun(org.apache.hadoop.hive.ql.HiveDriverRunHookContext)
*/
@@ -369,7 +364,7 @@ public class TestHiveDriver {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.hadoop.hive.ql.HiveDriverRunHook#preDriverRun(org.apache.hadoop.hive.ql.HiveDriverRunHookContext)
*/
@Override
@@ -380,11 +375,11 @@ public class TestHiveDriver {
}
// executeAsync
+
/**
* Test execute query async.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testExecuteQueryAsync() throws Exception {
@@ -422,9 +417,9 @@ public class TestHiveDriver {
Assert.assertEquals(0, driver.getHiveHandleSize());
conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT,
- "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
- + " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
- + " 'field.delim'=',' ) STORED AS TEXTFILE ");
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
+ + " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
+ + " 'field.delim'=',' ) STORED AS TEXTFILE ");
select = "SELECT ID, null, ID FROM test_execute_sync";
context = createContext(select, conf);
driver.executeAsync(context);
@@ -437,21 +432,15 @@ public class TestHiveDriver {
/**
* Validate execute async.
*
- * @param ctx
- * the ctx
- * @param finalState
- * the final state
- * @param isPersistent
- * the is persistent
- * @param formatNulls
- * the format nulls
- * @param driver
- * the driver
- * @throws Exception
- * the exception
+ * @param ctx the ctx
+ * @param finalState the final state
+ * @param isPersistent the is persistent
+ * @param formatNulls the format nulls
+ * @param driver the driver
+ * @throws Exception the exception
*/
protected void validateExecuteAsync(QueryContext ctx, DriverQueryState finalState, boolean isPersistent,
- boolean formatNulls, HiveDriver driver) throws Exception {
+ boolean formatNulls, HiveDriver driver) throws Exception {
waitForAsyncQuery(ctx, driver);
driver.updateStatus(ctx);
assertEquals(ctx.getDriverStatus().getState(), finalState, "Expected query to finish with" + finalState);
@@ -474,27 +463,21 @@ public class TestHiveDriver {
/**
* Validate execute async.
*
- * @param ctx
- * the ctx
- * @param finalState
- * the final state
- * @param isPersistent
- * the is persistent
- * @param formatNulls
- * the format nulls
- * @throws Exception
- * the exception
+ * @param ctx the ctx
+ * @param finalState the final state
+ * @param isPersistent the is persistent
+ * @param formatNulls the format nulls
+ * @throws Exception the exception
*/
protected void validateExecuteAsync(QueryContext ctx, DriverQueryState finalState, boolean isPersistent,
- boolean formatNulls) throws Exception {
+ boolean formatNulls) throws Exception {
validateExecuteAsync(ctx, finalState, isPersistent, formatNulls, driver);
}
/**
* Test cancel async query.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testCancelAsyncQuery() throws Exception {
@@ -519,19 +502,14 @@ public class TestHiveDriver {
/**
* Validate persistent result.
*
- * @param resultSet
- * the result set
- * @param dataFile
- * the data file
- * @param outptuDir
- * the outptu dir
- * @param formatNulls
- * the format nulls
- * @throws Exception
- * the exception
+ * @param resultSet the result set
+ * @param dataFile the data file
+ * @param outptuDir the outptu dir
+ * @param formatNulls the format nulls
+ * @throws Exception the exception
*/
private void validatePersistentResult(LensResultSet resultSet, String dataFile, Path outptuDir, boolean formatNulls)
- throws Exception {
+ throws Exception {
assertTrue(resultSet instanceof HivePersistentResultSet);
HivePersistentResultSet persistentResultSet = (HivePersistentResultSet) resultSet;
String path = persistentResultSet.getOutputPath();
@@ -583,15 +561,14 @@ public class TestHiveDriver {
/**
* Test persistent result set.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testPersistentResultSet() throws Exception {
createTestTable("test_persistent_result_set");
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, true);
- conf.set(LensConfConstants.RESULT_SET_PARENT_DIR, TEST_OUTPUT_DIR);
+ conf.set(LensConfConstants.RESULT_SET_PARENT_DIR, testOutputDir);
QueryContext ctx = createContext("SELECT ID FROM test_persistent_result_set", conf);
LensResultSet resultSet = driver.execute(ctx);
validatePersistentResult(resultSet, TEST_DATA_FILE, ctx.getHDFSResultDir(), false);
@@ -605,9 +582,9 @@ public class TestHiveDriver {
Assert.assertEquals(0, driver.getHiveHandleSize());
conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT,
- "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
- + " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
- + " 'field.delim'=',' ) STORED AS TEXTFILE ");
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
+ + " WITH SERDEPROPERTIES ('serialization.null.format'='-NA-',"
+ + " 'field.delim'=',' ) STORED AS TEXTFILE ");
ctx = createContext("SELECT ID, null, ID FROM test_persistent_result_set", conf);
resultSet = driver.execute(ctx);
Assert.assertEquals(0, driver.getHiveHandleSize());
@@ -626,18 +603,15 @@ public class TestHiveDriver {
/**
* Wait for async query.
*
- * @param ctx
- * the ctx
- * @param driver
- * the driver
- * @throws Exception
- * the exception
+ * @param ctx the ctx
+ * @param driver the driver
+ * @throws Exception the exception
*/
private void waitForAsyncQuery(QueryContext ctx, HiveDriver driver) throws Exception {
while (true) {
driver.updateStatus(ctx);
System.out.println("#W Waiting for query " + ctx.getQueryHandle() + " status: "
- + ctx.getDriverStatus().getState());
+ + ctx.getDriverStatus().getState());
assertNotNull(ctx.getDriverStatus());
if (ctx.getDriverStatus().isFinished()) {
assertTrue(ctx.getDriverStatus().getDriverFinishTime() > 0);
@@ -650,11 +624,11 @@ public class TestHiveDriver {
}
// explain
+
/**
* Test explain.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testExplain() throws Exception {
@@ -662,7 +636,7 @@ public class TestHiveDriver {
SessionState.setCurrentSessionState(ss);
DriverQueryPlan plan = driver.explain(createExplainContext("SELECT ID FROM test_explain", conf));
assertTrue(plan instanceof HiveQueryPlan);
- assertEquals(plan.getTableWeight(DATA_BASE + ".test_explain"), 500.0);
+ assertEquals(plan.getTableWeight(dataBase + ".test_explain"), 500.0);
Assert.assertEquals(0, driver.getHiveHandleSize());
// test execute prepare
@@ -710,8 +684,7 @@ public class TestHiveDriver {
/**
* Test explain partitioned table
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testExplainPartitionedTable() throws Exception {
@@ -724,18 +697,17 @@ public class TestHiveDriver {
assertNotNull(plan.getTablesQueried());
assertEquals(plan.getTablesQueried().size(), 1);
System.out.println("Tables:" + plan.getTablesQueried());
- assertEquals(plan.getTableWeight(DATA_BASE + ".test_part_table"), 500.0);
+ assertEquals(plan.getTableWeight(dataBase + ".test_part_table"), 500.0);
System.out.println("Parts:" + plan.getPartitions());
assertFalse(plan.getPartitions().isEmpty());
- assertTrue(plan.getPartitions().get(DATA_BASE + ".test_part_table").get(0).contains("today"));
- assertTrue(plan.getPartitions().get(DATA_BASE + ".test_part_table").get(0).contains("dt"));
+ assertTrue(plan.getPartitions().get(dataBase + ".test_part_table").get(0).contains("today"));
+ assertTrue(plan.getPartitions().get(dataBase + ".test_part_table").get(0).contains("dt"));
}
/**
* Test explain output.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testExplainOutput() throws Exception {
@@ -744,16 +716,16 @@ public class TestHiveDriver {
SessionState.setCurrentSessionState(ss);
DriverQueryPlan plan = driver.explain(createExplainContext("SELECT explain_test_1.ID, count(1) FROM "
- + " explain_test_1 join explain_test_2 on explain_test_1.ID = explain_test_2.ID"
- + " WHERE explain_test_1.ID = 'foo' or explain_test_2.ID = 'bar'" + " GROUP BY explain_test_1.ID", conf));
+ + " explain_test_1 join explain_test_2 on explain_test_1.ID = explain_test_2.ID"
+ + " WHERE explain_test_1.ID = 'foo' or explain_test_2.ID = 'bar'" + " GROUP BY explain_test_1.ID", conf));
Assert.assertEquals(0, driver.getHiveHandleSize());
assertTrue(plan instanceof HiveQueryPlan);
assertNotNull(plan.getTablesQueried());
assertEquals(plan.getTablesQueried().size(), 2);
assertNotNull(plan.getTableWeights());
- assertTrue(plan.getTableWeights().containsKey(DATA_BASE + ".explain_test_1"));
- assertTrue(plan.getTableWeights().containsKey(DATA_BASE + ".explain_test_2"));
+ assertTrue(plan.getTableWeights().containsKey(dataBase + ".explain_test_1"));
+ assertTrue(plan.getTableWeights().containsKey(dataBase + ".explain_test_2"));
assertEquals(plan.getNumJoins(), 1);
assertTrue(plan.getPlan() != null && !plan.getPlan().isEmpty());
driver.closeQuery(plan.getHandle());
@@ -762,8 +734,7 @@ public class TestHiveDriver {
/**
* Test explain output persistent.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testExplainOutputPersistent() throws Exception {
@@ -778,7 +749,7 @@ public class TestHiveDriver {
Assert.assertEquals(0, driver.getHiveHandleSize());
assertNotNull(plan2.getTablesQueried());
assertEquals(plan2.getTablesQueried().size(), 1);
- assertTrue(plan2.getTableWeights().containsKey(DATA_BASE + ".explain_test_1"));
+ assertTrue(plan2.getTableWeights().containsKey(dataBase + ".explain_test_1"));
assertEquals(plan2.getNumSels(), 1);
QueryContext ctx = createContext(pctx, conf);
LensResultSet resultSet = driver.execute(ctx);
@@ -791,6 +762,7 @@ public class TestHiveDriver {
/**
* Testing Duration Based Priority Logic by mocking everything except partitions.
+ *
* @throws IOException
* @throws LensException
*/
@@ -807,7 +779,7 @@ public class TestHiveDriver {
BufferedReader br = new BufferedReader(new InputStreamReader(
TestHiveDriver.class.getResourceAsStream("/priority_tests.data")));
String line;
- while((line = br.readLine()) != null) {
+ while ((line = br.readLine()) != null) {
String[] kv = line.split("\\s*:\\s*");
final List<String> partitions = Arrays.asList(kv[0].trim().split("\\s*,\\s*"));
@@ -818,17 +790,17 @@ public class TestHiveDriver {
}
};
AbstractQueryContext ctx = new MockQueryContext("driverQuery1", new LensConf(), conf,
- driverQuery1.keySet());
+ driverQuery1.keySet());
ctx.setDriverQueriesAndPlans(driverQuery1);
ctx.setSelectedDriver(mockDriver);
- ((MockDriver.MockQueryPlan)ctx.getDriverContext().getDriverQueryPlan(mockDriver)).setPartitions
- (new HashMap<String,
- List<String>>
- () {
+ ((MockDriver.MockQueryPlan) ctx.getDriverContext().getDriverQueryPlan(mockDriver)).setPartitions(
+ new HashMap<String, List<String>>() {
{
put("table1", partitions);
- }});
+ }
+ }
+ );
Assert.assertEquals(expected, driver.queryPriorityDecider.decidePriority(ctx));
Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 2fe7a06..5c70a89 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -26,19 +26,22 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hive.service.server.HiveServer2;
import org.apache.lens.api.LensException;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.DriverQueryPlan;
-import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
+import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.service.Service;
+import org.apache.hive.service.server.HiveServer2;
+
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -59,27 +62,27 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
static final int HS2_PORT = 12345;
/** The server. */
- public static HiveServer2 server;
+ private static HiveServer2 server;
/** The remote conf. */
+
private static HiveConf remoteConf = new HiveConf();
/**
* Setup test.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@BeforeClass
public static void setupTest() throws Exception {
createHS2Service();
}
+
/**
* Creates the h s2 service.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
public static void createHS2Service() throws Exception {
remoteConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, RemoteThriftConnection.class, ThriftConnection.class);
@@ -102,8 +105,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
/**
* Cleanup test.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@AfterClass
public static void cleanupTest() throws Exception {
@@ -113,8 +115,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
/**
* Stop h s2 service.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
public static void stopHS2Service() throws Exception {
try {
@@ -124,23 +125,29 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
}
}
+ public static Service.STATE getServerState() {
+ return server.getServiceState();
+ }
+
protected void createDriver() throws LensException {
- DATA_BASE = TestRemoteHiveDriver.class.getSimpleName().toLowerCase();
+ dataBase = TestRemoteHiveDriver.class.getSimpleName().toLowerCase();
conf = new HiveConf(remoteConf);
conf.addResource("hivedriver-site.xml");
driver = new HiveDriver();
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, false);
driver.configure(conf);
- drivers = new ArrayList<LensDriver>() {{ add
- (driver);}};
+ drivers = new ArrayList<LensDriver>() {
+ {
+ add(driver);
+ }
+ };
System.out.println("TestRemoteHiveDriver created");
}
/**
* Test multi thread client.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testMultiThreadClient() throws Exception {
@@ -151,7 +158,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
thConf.setLong(HiveDriver.HS2_CONNECTION_EXPIRY_DELAY, 10000);
final HiveDriver thrDriver = new HiveDriver();
thrDriver.configure(thConf);
- QueryContext ctx = createContext("USE " + DATA_BASE, conf);
+ QueryContext ctx = createContext("USE " + dataBase, conf);
thrDriver.execute(ctx);
// Launch a select query
@@ -225,8 +232,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
/**
* Test hive driver persistence.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testHiveDriverPersistence() throws Exception {
@@ -241,7 +247,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
driverConf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
driverConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
- QueryContext ctx = createContext("USE " + DATA_BASE, driverConf);
+ QueryContext ctx = createContext("USE " + dataBase, driverConf);
oldDriver.execute(ctx);
Assert.assertEquals(0, oldDriver.getHiveHandleSize());
@@ -296,11 +302,9 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
/**
* Persist context.
*
- * @param ctx
- * the ctx
+ * @param ctx the ctx
* @return the byte[]
- * @throws IOException
- * Signals that an I/O exception has occurred.
+ * @throws IOException Signals that an I/O exception has occurred.
*/
private byte[] persistContext(QueryContext ctx) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -324,15 +328,11 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
/**
* Read context.
*
- * @param bytes
- * the bytes
- * @param driver
- * the driver
+ * @param bytes the bytes
+ * @param driver the driver
* @return the query context
- * @throws IOException
- * Signals that an I/O exception has occurred.
- * @throws ClassNotFoundException
- * the class not found exception
+ * @throws IOException Signals that an I/O exception has occurred.
+ * @throws ClassNotFoundException the class not found exception
*/
private QueryContext readContext(byte[] bytes, LensDriver driver) throws IOException, ClassNotFoundException {
ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
@@ -356,19 +356,16 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
/**
* Creates the partitioned table.
*
- * @param tableName
- * the table name
- * @param partitions
- * the partitions
- * @throws Exception
- * the exception
+ * @param tableName the table name
+ * @param partitions the partitions
+ * @throws Exception the exception
*/
private void createPartitionedTable(String tableName, int partitions) throws Exception {
conf.setBoolean(LensConfConstants.QUERY_ADD_INSERT_OVEWRITE, false);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
QueryContext ctx = createContext("CREATE EXTERNAL TABLE IF NOT EXISTS " + tableName
- + " (ID STRING) PARTITIONED BY (DT STRING, ET STRING)", conf);
+ + " (ID STRING) PARTITIONED BY (DT STRING, ET STRING)", conf);
driver.execute(ctx);
Assert.assertEquals(0, driver.getHiveHandleSize());
@@ -390,7 +387,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
System.out.println("@@ Adding partition " + i);
QueryContext partCtx = createContext("ALTER TABLE " + tableName + " ADD IF NOT EXISTS PARTITION (DT='p" + i
- + "', ET='1') LOCATION '" + partDir.getPath() + "'", conf);
+ + "', ET='1') LOCATION '" + partDir.getPath() + "'", conf);
driver.execute(partCtx);
}
}
@@ -398,8 +395,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
/**
* Test partition in query plan.
*
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
@Test
public void testPartitionInQueryPlan() throws Exception {
@@ -409,9 +405,9 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
// Query should select 5 partitions of table 1 and 1 partitions of table 2
String explainQuery = "SELECT table_1.ID "
- + "FROM table_1 LEFT OUTER JOIN table_2 ON table_1.ID = table_2.ID AND table_2.DT='p0' "
- + "WHERE table_1.DT='p0' OR table_1.DT='p1' OR table_1.DT='p2' OR table_1.DT='p3' OR table_1.DT='p4' "
- + "AND table_1.ET='1'";
+ + "FROM table_1 LEFT OUTER JOIN table_2 ON table_1.ID = table_2.ID AND table_2.DT='p0' "
+ + "WHERE table_1.DT='p0' OR table_1.DT='p1' OR table_1.DT='p2' OR table_1.DT='p3' OR table_1.DT='p4' "
+ + "AND table_1.ET='1'";
SessionState.setCurrentSessionState(ss);
DriverQueryPlan plan = driver.explain(createExplainContext(explainQuery, conf));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/resources/hive-site.xml b/lens-driver-hive/src/test/resources/hive-site.xml
index 116bf11..a35dc11 100644
--- a/lens-driver-hive/src/test/resources/hive-site.xml
+++ b/lens-driver-hive/src/test/resources/hive-site.xml
@@ -22,36 +22,38 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
-<!-- This configuration is for HiveServer in tests -->
-<property>
- <name>hive.metastore.local</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.metastore.warehouse.dir</name>
- <value>/tmp/hive/warehouse</value>
-</property>
-
-<property>
- <name>mapreduce.framework.name</name>
- <value>local</value>
-</property>
-
-<property>
- <name>mapred.job.tracker</name>
- <value>local</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
- <description>JDBC connect string for a JDBC metastore</description>
-</property>
-
-<property>
- <name>hive.server2.query.log.dir</name>
- <value>target/query_logs</value>
-</property>
+ <!-- This configuration is for HiveServer in tests -->
+ <property>
+
+ <name>hive.metastore.local</name>
+
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>/tmp/hive/warehouse</value>
+ </property>
+
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>local</value>
+ </property>
+
+ <property>
+ <name>mapred.job.tracker</name>
+ <value>local</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>hive.server2.query.log.dir</name>
+ <value>target/query_logs</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/src/test/resources/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/resources/hivedriver-site.xml b/lens-driver-hive/src/test/resources/hivedriver-site.xml
index 30e64b1..3f37b7a 100644
--- a/lens-driver-hive/src/test/resources/hivedriver-site.xml
+++ b/lens-driver-hive/src/test/resources/hivedriver-site.xml
@@ -23,25 +23,27 @@
<configuration>
-<property>
- <name>hive.metastore.local</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.metastore.warehouse.dir</name>
- <value>/tmp/hive/warehouse</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
- <description>JDBC connect string for a JDBC metastore</description>
-</property>
-
-<property>
- <name>lens.query.result.parent.dir</name>
- <value>target/lens-results</value>
-</property>
+ <property>
+
+ <name>hive.metastore.local</name>
+
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.warehouse.dir</name>
+ <value>/tmp/hive/warehouse</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=target/metastore_db;create=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+ </property>
+
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/lens-results</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/testng.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/testng.xml b/lens-driver-hive/testng.xml
index b98d252..2ccba52 100644
--- a/lens-driver-hive/testng.xml
+++ b/lens-driver-hive/testng.xml
@@ -20,17 +20,17 @@
-->
<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
-<suite name="lens-driver-hive" verbose="1" >
+<suite name="lens-driver-hive" verbose="1">
- <test name="TestHiveDriver">
- <classes>
- <class name="org.apache.lens.driver.hive.TestHiveDriver"/>
- </classes>
- </test>
+ <test name="TestHiveDriver">
+ <classes>
+ <class name="org.apache.lens.driver.hive.TestHiveDriver"/>
+ </classes>
+ </test>
- <test name="TestRemoteHiveDriver">
- <classes>
- <class name="org.apache.lens.driver.hive.TestRemoteHiveDriver"/>
- </classes>
- </test>
+ <test name="TestRemoteHiveDriver">
+ <classes>
+ <class name="org.apache.lens.driver.hive.TestRemoteHiveDriver"/>
+ </classes>
+ </test>
</suite>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-hive/testutils/hadoop
----------------------------------------------------------------------
diff --git a/lens-driver-hive/testutils/hadoop b/lens-driver-hive/testutils/hadoop
index c1658c7..c950eb1 100755
--- a/lens-driver-hive/testutils/hadoop
+++ b/lens-driver-hive/testutils/hadoop
@@ -29,7 +29,7 @@ bin=`cd "$bin"; pwd`
export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m
+JAVA_HEAP_MAX=-Xmx1000m
# check envvars which might override default args
if [ "$HADOOP_HEAPSIZE" != "" ]; then
@@ -65,7 +65,7 @@ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi
+fi
HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE"
# Disable ipv6 as it can cause issues
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d0baff6b/lens-driver-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml
index 360f694..4e6f322 100644
--- a/lens-driver-jdbc/pom.xml
+++ b/lens-driver-jdbc/pom.xml
@@ -82,5 +82,36 @@
<artifactId>hadoop-client</artifactId>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${lens.checkstyle.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>checkstyle-check</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <consoleOutput>true</consoleOutput>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <configLocation>checkstyle.xml</configLocation>
+ <headerLocation>checkstyle-java-header.txt</headerLocation>
+ <failOnViolation>true</failOnViolation>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
-</project>
\ No newline at end of file
+</project>