You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/02/04 00:51:17 UTC
[14/50] [abbrv] kylin git commit: Minor,
fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver
Minor, fix NotClassFound issue in org.apache.hadoop.hive.ql.Driver
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/26bee662
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/26bee662
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/26bee662
Branch: refs/heads/sync
Commit: 26bee6625e7a09373a9b2c39db57af0c1a5a8460
Parents: ebc53b6
Author: Yifan Zhang <ev...@gmail.com>
Authored: Mon Jan 1 16:33:26 2018 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jan 26 22:54:58 2018 +0800
----------------------------------------------------------------------
.../kylin/source/hive/BeelineHiveClient.java | 5 +--
.../apache/kylin/source/hive/CLIHiveClient.java | 39 +++++++-------------
.../kylin/source/hive/HiveMetadataExplorer.java | 29 ++++++++++-----
.../apache/kylin/source/hive/IHiveClient.java | 6 +--
.../apache/kylin/source/jdbc/JdbcExplorer.java | 9 ++++-
5 files changed, 43 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
index ee693c5..314402c 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java
@@ -28,7 +28,6 @@ import java.sql.Statement;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.kylin.common.util.DBUtils;
import com.google.common.base.Preconditions;
@@ -117,12 +116,12 @@ public class BeelineHiveClient implements IHiveClient {
}
@Override
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String hql) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String[] hqls) throws IOException {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
index e8a93bd..bc9f17e 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/CLIHiveClient.java
@@ -22,17 +22,15 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
-import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.HiveCmdBuilder;
+import org.apache.kylin.common.util.Pair;
import com.google.common.collect.Lists;
@@ -43,7 +41,6 @@ import com.google.common.collect.Lists;
*/
public class CLIHiveClient implements IHiveClient {
protected HiveConf hiveConf = null;
- protected Driver driver = null;
protected HiveMetaStoreClient metaStoreClient = null;
public CLIHiveClient() {
@@ -52,22 +49,25 @@ public class CLIHiveClient implements IHiveClient {
/**
* only used by Deploy Util
+ * @throws IOException
*/
@Override
- public void executeHQL(String hql) throws CommandNeedRetryException, IOException {
- CommandProcessorResponse response = getDriver().run(hql);
- int retCode = response.getResponseCode();
- if (retCode != 0) {
- String err = response.getErrorMessage();
- throw new IOException("Failed to execute hql [" + hql + "], error message is: " + err);
+ public void executeHQL(String hql) throws IOException {
+ final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
+ hiveCmdBuilder.addStatement(hql);
+ Pair<Integer, String> response = KylinConfig.getInstanceFromEnv().getCliCommandExecutor()
+ .execute(hiveCmdBuilder.toString());
+ if (response.getFirst() != 0) {
+ throw new IllegalArgumentException("Failed to execute hql [" + hql + "], error message is: " + response.getSecond());
}
+
}
/**
* only used by Deploy Util
*/
@Override
- public void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException {
+ public void executeHQL(String[] hqls) throws IOException {
for (String sql : hqls)
executeHQL(sql);
}
@@ -159,17 +159,4 @@ public class CLIHiveClient implements IHiveClient {
}
return result;
}
-
- /**
- * Get the hive ql driver to execute ddl or dml
- * @return
- */
- private Driver getDriver() {
- if (driver == null) {
- driver = new Driver(hiveConf);
- SessionState.start(new CliSessionState(hiveConf));
- }
-
- return driver;
- }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index d80c546..cb3eb02 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -32,11 +32,15 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.apache.kylin.source.ISampleDataDeployer;
import org.apache.kylin.source.ISourceMetadataExplorer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDataDeployer {
+ private static final Logger logger = LoggerFactory.getLogger(HiveClientFactory.class);
+
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
-
+
@Override
public List<String> listDatabases() throws Exception {
return hiveClient.getHiveDbNames();
@@ -90,7 +94,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
tableExtDesc.setUuid(UUID.randomUUID().toString());
tableExtDesc.setLastModified(0);
tableExtDesc.init(prj);
-
+
tableExtDesc.addDataSourceProp("location", hiveTableMeta.sdLocation);
tableExtDesc.addDataSourceProp("owner", hiveTableMeta.owner);
tableExtDesc.addDataSourceProp("last_access_time", String.valueOf(hiveTableMeta.lastAccessTime));
@@ -114,10 +118,10 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
hiveClient.executeHQL(generateCreateSchemaSql(database));
}
- private String generateCreateSchemaSql(String schemaName){
+ private String generateCreateSchemaSql(String schemaName) {
return String.format("CREATE DATABASE IF NOT EXISTS %s", schemaName);
}
-
+
@Override
public void createSampleTable(TableDesc table) throws Exception {
hiveClient.executeHQL(generateCreateTableSql(table));
@@ -146,7 +150,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
return new String[] { dropsql, dropsql2, ddl.toString() };
}
-
+
@Override
public void loadSampleData(String tableName, String tmpDataDir) throws Exception {
hiveClient.executeHQL(generateLoadDataSql(tableName, tmpDataDir));
@@ -155,12 +159,12 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
private String generateLoadDataSql(String tableName, String tableFileDir) {
return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName + ".csv' OVERWRITE INTO TABLE " + tableName;
}
-
+
@Override
public void createWrapperView(String origTableName, String viewName) throws Exception {
hiveClient.executeHQL(generateCreateViewSql(viewName, origTableName));
}
-
+
private String[] generateCreateViewSql(String viewName, String tableName) {
String dropView = "DROP VIEW IF EXISTS " + viewName;
@@ -170,7 +174,7 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
return new String[] { dropView, dropTable, createSql };
}
-
+
private static String getHiveDataType(String javaDataType) {
String hiveDataType = javaDataType.toLowerCase().startsWith("varchar") ? "string" : javaDataType;
hiveDataType = javaDataType.toLowerCase().startsWith("integer") ? "int" : hiveDataType;
@@ -192,16 +196,21 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
String evalViewSql = "CREATE VIEW " + tmpDatabase + "." + tmpView + " as " + query;
try {
- hiveClient.executeHQL(new String[] { dropViewSql, evalViewSql });
+ logger.debug("Removing duplicate view {}", tmpView);
+ hiveClient.executeHQL(dropViewSql);
+ logger.debug("Creating view {} for query: {}", tmpView, query);
+ hiveClient.executeHQL(evalViewSql);
+ logger.debug("Evaluating query columns' metadata");
HiveTableMeta hiveTableMeta = hiveClient.getHiveTableMeta(tmpDatabase, tmpView);
return extractColumnFromMeta(hiveTableMeta);
} catch (Exception e) {
throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
} finally {
try {
+ logger.debug("Cleaning up.");
hiveClient.executeHQL(dropViewSql);
} catch (Exception e) {
- throw new RuntimeException("Cannot temp view of query: " + query, e);
+ logger.warn("Cannot drop temp view of query: {}", query, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
index ca5312d..ec2bf7f 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/IHiveClient.java
@@ -21,13 +21,11 @@ package org.apache.kylin.source.hive;
import java.io.IOException;
import java.util.List;
-import org.apache.hadoop.hive.ql.CommandNeedRetryException;
-
public interface IHiveClient {
- void executeHQL(String hql) throws CommandNeedRetryException, IOException;
+ void executeHQL(String hql) throws IOException;
- void executeHQL(String[] hqls) throws CommandNeedRetryException, IOException;
+ void executeHQL(String[] hqls) throws IOException;
HiveTableMeta getHiveTableMeta(String database, String tableName) throws Exception;
http://git-wip-us.apache.org/repos/asf/kylin/blob/26bee662/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
index 81e42bb..d96a68d 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/jdbc/JdbcExplorer.java
@@ -243,7 +243,11 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
String evalViewSql = "CREATE VIEW " + tmpView + " as " + query;
try {
- executeSQL(new String[] { dropViewSql, evalViewSql });
+ logger.debug("Removing duplicate view {}", tmpView);
+ executeSQL(dropViewSql);
+ logger.debug("Creating view {} for query: {}", tmpView, query);
+ executeSQL(evalViewSql);
+ logger.debug("Evaluating query columns' metadata");
Connection con = SqlUtil.getConnection(dbconf);
DatabaseMetaData dbmd = con.getMetaData();
ResultSet rs = dbmd.getColumns(null, tmpDatabase, tmpView, null);
@@ -255,9 +259,10 @@ public class JdbcExplorer implements ISourceMetadataExplorer, ISampleDataDeploye
throw new RuntimeException("Cannot evalutate metadata of query: " + query, e);
} finally {
try {
+ logger.debug("Cleaning up.");
executeSQL(dropViewSql);
} catch (Exception e) {
- throw new RuntimeException("Cannot temp view of query: " + query, e);
+ logger.warn("Cannot drop temp view of query: {}", query, e);
}
}
}