You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by li...@apache.org on 2018/04/26 08:12:56 UTC
[33/50] [abbrv] incubator-griffin git commit: test thrift metastore
client
test thrift metastore client
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/5a7793d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/5a7793d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/5a7793d1
Branch: refs/heads/griffin-0.2.0-incubating-rc4
Commit: 5a7793d1d957f07dda9ad9dd3e8c8e85ac26ddd4
Parents: 41e986b
Author: Lionel Liu <bh...@163.com>
Authored: Tue Apr 24 08:41:14 2018 +0800
Committer: Lionel Liu <bh...@163.com>
Committed: Tue Apr 24 08:41:14 2018 +0800
----------------------------------------------------------------------
service/pom.xml | 148 +++++++++++++++++--
.../core/metastore/hive/HiveMetaStoreProxy.java | 96 +++++++++++-
.../hive/HiveMetaStoreServiceImpl.java | 37 ++++-
3 files changed, 253 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/5a7793d1/service/pom.xml
----------------------------------------------------------------------
diff --git a/service/pom.xml b/service/pom.xml
index 9c8f3bf..8105bb3 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -36,7 +36,7 @@ under the License.
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hadoop.version>2.7.1</hadoop.version>
- <hive.version>2.2.0</hive.version>
+ <hive.version>2.3.3</hive.version>
<scala.version>2.10</scala.version>
<spring.boot.version>1.5.1.RELEASE</spring.boot.version>
<confluent.version>3.2.0</confluent.version>
@@ -70,10 +70,38 @@ under the License.
</dependencyManagement>
<dependencies>
+ <!--<dependency>-->
+ <!--<groupId>javax.servlet</groupId>-->
+ <!--<artifactId>javax.servlet-api</artifactId>-->
+ <!--<version>3.1.0</version>-->
+ <!--</dependency>-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <!--<exclusion>-->
+ <!--<groupId>org.springframework.boot</groupId>-->
+ <!--<artifactId>spring-boot-starter-tomcat</artifactId>-->
+ <!--</exclusion>-->
+ <exclusion>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-logging</artifactId>
+ </exclusion>
+ <!--<exclusion>-->
+ <!--<groupId>ch.qos.logback</groupId>-->
+ <!--<artifactId>logback-classic</artifactId>-->
+ <!--</exclusion>-->
+ <!--<exclusion>-->
+ <!--<groupId>org.slf4j</groupId>-->
+ <!--<artifactId>log4j-over-slf4j</artifactId>-->
+ <!--</exclusion>-->
+ </exclusions>
</dependency>
+ <!--<dependency>-->
+ <!--<groupId>org.springframework.boot</groupId>-->
+ <!--<artifactId>spring-boot-starter-tomcat</artifactId>-->
+ <!--<version>1.1.2.RELEASE</version>-->
+ <!--</dependency>-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
@@ -131,10 +159,34 @@ under the License.
<version>${hadoop.version}</version>
<!--<scope>provided</scope>-->
<exclusions>
- <!--<exclusion>-->
- <!--<groupId>javax.servlet</groupId>-->
- <!--<artifactId>servlet-api</artifactId>-->
- <!--</exclusion>-->
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${hive.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.eclipse.jetty.aggregate</groupId>
+ <artifactId>jetty-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
@@ -147,17 +199,58 @@ under the License.
<version>${hive.version}</version>
<exclusions>
<exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j-web</artifactId>
+ <groupId>org.apache.logging.log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+
+ <exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>jetty-all</artifactId>
</exclusion>
- <!--<exclusion>-->
- <!--<groupId>org.apache.hive</groupId>-->
- <!--<artifactId>hive-shims</artifactId>-->
- <!--</exclusion>-->
+ <exclusion>
+ <groupId>org.eclipse.jetty.orbit</groupId>
+ <artifactId>javax.servlet</artifactId>
+ </exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <!--<!–<exclusion>–>-->
+ <!--<!–<groupId>org.apache.hive</groupId>–>-->
+ <!--<!–<artifactId>hive-shims</artifactId>–>-->
+ <!--<!–</exclusion>–>-->
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <!--<exclusion>-->
+ <!--<groupId>tomcat</groupId>-->
+ <!--<artifactId>jasper-compiler</artifactId>-->
+ <!--</exclusion>-->
+ <!--<exclusion>-->
+ <!--<groupId>tomcat</groupId>-->
+ <!--<artifactId>jasper-runtime</artifactId>-->
+ <!--</exclusion>-->
+ <!--<exclusion>-->
+ <!--<groupId>javax.servlet</groupId>-->
+ <!--<artifactId>servlet-api</artifactId>-->
+ <!--</exclusion>-->
</exclusions>
</dependency>
@@ -166,12 +259,12 @@ under the License.
<groupId>io.confluent</groupId>
<artifactId>kafka-schema-registry-client</artifactId>
<version>${confluent.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
+ <!--<exclusions>-->
+ <!--<exclusion>-->
+ <!--<groupId>org.slf4j</groupId>-->
+ <!--<artifactId>slf4j-log4j12</artifactId>-->
+ <!--</exclusion>-->
+ <!--</exclusions>-->
</dependency>
<!--schedule-->
@@ -284,6 +377,31 @@ under the License.
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <!--<exclude>classworlds:classworlds</exclude>-->
+ <!--<exclude>junit:junit</exclude>-->
+ <!--<exclude>jmock:*</exclude>-->
+ <!--<exclude>*:xml-apis</exclude>-->
+ <!--<exclude>org.apache.maven:lib:tests</exclude>-->
+ <!--<exclude>log4j:log4j:jar:</exclude>-->
+ </excludes>
+ </artifactSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/5a7793d1/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java
index f632f14..8291f82 100644
--- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java
+++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreProxy.java
@@ -29,6 +29,10 @@ import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import javax.annotation.PreDestroy;
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.security.CodeSource;
@Component
public class HiveMetaStoreProxy {
@@ -49,19 +53,101 @@ public class HiveMetaStoreProxy {
@Value("${hive.hmshandler.retry.interval}")
private String interval;
- private HiveMetaStoreClient client = null;
+// private HiveMetaStoreClient client = null;
+ private ThriftMetastoreClient client = null;
+
+ private static final Logger l4j = LoggerFactory.getLogger(HiveConf.class);
+
+ private static URL checkConfigFile(File f) {
+ try {
+ return (f.exists() && f.isFile()) ? f.toURI().toURL() : null;
+ } catch (Throwable e) {
+ if (l4j.isInfoEnabled()) {
+ l4j.info("Error looking for config " + f, e);
+ }
+ System.err.println("Error looking for config " + f + ": " + e.getMessage());
+ return null;
+ }
+ }
+
+ private static URL findConfigFile(ClassLoader classLoader, String name, boolean doLog) {
+ URL result = classLoader.getResource(name);
+ LOGGER.warn("result: {}", result);
+ if (result == null) {
+ String confPath = System.getenv("HIVE_CONF_DIR");
+ LOGGER.warn("confPath: {}", confPath);
+ result = checkConfigFile(new File(confPath, name));
+ LOGGER.warn("result: {}", result);
+ if (result == null) {
+ String homePath = System.getenv("HIVE_HOME");
+ LOGGER.warn("homePath: {}", homePath);
+ String nameInConf = "conf" + File.pathSeparator + name;
+ LOGGER.warn("nameInConf: {}", nameInConf);
+ result = checkConfigFile(new File(homePath, nameInConf));
+ LOGGER.warn("result: {}", result);
+ if (result == null) {
+ URI jarUri = null;
+ try {
+ java.security.ProtectionDomain domain = HiveConf.class.getProtectionDomain();
+ LOGGER.warn("domain: {}", domain);
+ CodeSource codeSource = domain.getCodeSource();
+ LOGGER.warn("codeSource: {}", codeSource);
+ URL location = codeSource.getLocation();
+ LOGGER.warn("location: {}", location);
+ jarUri = location.toURI();
+
+// jarUri = HiveConf.class.getProtectionDomain().getCodeSource().getLocation().toURI();
+ LOGGER.warn("jarUri: {}", jarUri);
+ } catch (Throwable e) {
+ if (l4j.isInfoEnabled()) {
+ l4j.info("Cannot get jar URI", e);
+ }
+ System.err.println("Cannot get jar URI: " + e.getMessage());
+ }
+ File f1 = new File(jarUri);
+ LOGGER.warn("f1: {}", f1);
+ File f2 = f1.getParentFile();
+ LOGGER.warn("f2: {}", f2);
+ File f3 = new File(f2, nameInConf);
+ LOGGER.warn("f3: {}", f3);
+ result = checkConfigFile(f3);
+ }
+ }
+ }
+ if (doLog && l4j.isInfoEnabled()) {
+ l4j.info("Found configuration file " + result);
+ }
+ return result;
+ }
+
+// @Bean
+ public ThriftMetastoreClient initHiveMetastoreClient() {
+ LOGGER.warn("begin client");
+ LOGGER.warn(File.pathSeparator);
+ LOGGER.warn(uris);
+
+// ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ ClassLoader classLoader = HiveConf.class.getClassLoader();
+ LOGGER.warn("classLoader: {}", classLoader);
+ if (classLoader == null) {
+ classLoader = HiveConf.class.getClassLoader();
+ LOGGER.warn("classLoader: {}", classLoader);
+ }
+ URL url = findConfigFile(classLoader, "hive-site.xml", true);
+ LOGGER.warn("url: {}", url);
- @Bean
- public HiveMetaStoreClient initHiveMetastoreClient() {
HiveConf hiveConf = new HiveConf();
+ LOGGER.warn("hive conf success");
hiveConf.set("hive.metastore.local", "false");
hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris);
hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts);
hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval);
try {
- client = new HiveMetaStoreClient(hiveConf);
- } catch (MetaException e) {
+ LOGGER.warn("begin hive ms client");
+ client = new ThriftMetastoreClient(hiveConf);
+ LOGGER.warn("hive ms client success");
+ } catch (Exception e) {
LOGGER.error("Failed to connect hive metastore. {}", e.getMessage());
client = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/5a7793d1/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
index 8e265e0..674defd 100644
--- a/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/metastore/hive/HiveMetaStoreServiceImpl.java
@@ -48,8 +48,18 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
private static final Logger LOGGER = LoggerFactory.getLogger(HiveMetaStoreService.class);
+// @Autowired
+// private HiveMetaStoreClient client = null;
+ private ThriftMetastoreClient client = null;
+
@Autowired
- private HiveMetaStoreClient client;
+ private HiveMetaStoreProxy proxy;
+
+ public void initClient() {
+ if (client == null) {
+ client = proxy.initHiveMetastoreClient();
+ }
+ }
@Value("${hive.metastore.dbname}")
private String defaultDbName;
@@ -64,14 +74,16 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
@Override
@Cacheable
public Iterable<String> getAllDatabases() {
+ initClient();
Iterable<String> results = null;
try {
if (client == null) {
LOGGER.warn("Hive client is null. Please check your hive config.");
return new ArrayList<>();
}
- results = client.getAllDatabases();
- } catch (MetaException e) {
+ results = client.getClient().get_all_databases();
+// results = client.getAllDatabases();
+ } catch (Exception e) {
reconnect();
LOGGER.error("Can not get databases : {}", e.getMessage());
}
@@ -82,13 +94,15 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
@Override
@Cacheable
public Iterable<String> getAllTableNames(String dbName) {
+ initClient();
Iterable<String> results = null;
try {
if (client == null) {
LOGGER.warn("Hive client is null. Please check your hive config.");
return new ArrayList<>();
}
- results = client.getAllTables(getUseDbName(dbName));
+ results = client.getClient().get_all_tables(dbName);
+// results = client.getAllTables(getUseDbName(dbName));
} catch (Exception e) {
reconnect();
LOGGER.error("Exception fetching tables info: {}", e.getMessage());
@@ -107,6 +121,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
@Override
@Cacheable
public Map<String, List<Table>> getAllTable() {
+ initClient();
Map<String, List<Table>> results = new HashMap<>();
Iterable<String> dbs;
// if hive.metastore.uris in application.properties configs wrong, client will be injected failure and will be null.
@@ -128,13 +143,15 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
@Override
@Cacheable
public Table getTable(String dbName, String tableName) {
+ initClient();
Table result = null;
try {
if (client == null) {
LOGGER.warn("Hive client is null. Please check your hive config.");
return null;
}
- result = client.getTable(getUseDbName(dbName), tableName);
+ result = client.getClient().get_table(getUseDbName(dbName), tableName);
+// result = client.getTable(getUseDbName(dbName), tableName);
} catch (Exception e) {
reconnect();
LOGGER.error("Exception fetching table info : {}. {}", tableName, e.getMessage());
@@ -152,6 +169,7 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
private List<Table> getTables(String db) {
+ initClient();
String useDbName = getUseDbName(db);
List<Table> allTables = new ArrayList<>();
try {
@@ -159,9 +177,11 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
LOGGER.warn("Hive client is null. Please check your hive config.");
return allTables;
}
- Iterable<String> tables = client.getAllTables(useDbName);
+ Iterable<String> tables = client.getClient().get_all_tables(useDbName);
+// Iterable<String> tables = client.getAllTables(useDbName);
for (String table : tables) {
- Table tmp = client.getTable(db, table);
+ Table tmp = client.getClient().get_table(db, table);
+// Table tmp = client.getTable(db, table);
allTables.add(tmp);
}
} catch (Exception e) {
@@ -180,12 +200,13 @@ public class HiveMetaStoreServiceImpl implements HiveMetaStoreService {
}
private void reconnect() {
+ initClient();
if (singleThreadExecutor.getActiveCount() == 0) {
System.out.println("execute create thread.");
singleThreadExecutor.execute(() -> {
try {
client.reconnect();
- } catch (MetaException e) {
+ } catch (Exception e) {
LOGGER.error("reconnect to hive failed.");
}
});