You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2015/12/11 19:25:15 UTC
[3/3] drill git commit: DRILL-4127: Reduce Hive metastore client API
call in HiveSchema.
DRILL-4127: Reduce Hive metastore client API call in HiveSchema.
1) Use lazy loading of tableNames in HiveSchema, in stead of pre-loading all table names under each HiveSchema.
2) Do not call get_all_databases for subSchema to check existence if the name comes from getSubSchemaNames() directly.
review comments.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b8b18ebf
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b8b18ebf
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b8b18ebf
Branch: refs/heads/master
Commit: b8b18ebf4b9dbf2027d1f5e9666504dbe11901cb
Parents: 539cbba
Author: Jinfeng Ni <jn...@apache.org>
Authored: Wed Nov 18 20:18:51 2015 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Fri Dec 11 09:49:55 2015 -0800
----------------------------------------------------------------------
.../store/hive/schema/HiveDatabaseSchema.java | 24 ++++++++++++++++----
.../store/hive/schema/HiveSchemaFactory.java | 22 ++++++++----------
2 files changed, 28 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/b8b18ebf/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
index 48c034e..6f43639 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveDatabaseSchema.java
@@ -17,30 +17,36 @@
*/
package org.apache.drill.exec.store.hive.schema;
-import java.util.List;
import java.util.Set;
import org.apache.calcite.schema.Table;
import org.apache.drill.exec.store.AbstractSchema;
+import org.apache.drill.exec.store.SchemaConfig;
+import org.apache.drill.exec.store.hive.DrillHiveMetaStoreClient;
import org.apache.drill.exec.store.hive.HiveStoragePluginConfig;
import org.apache.drill.exec.store.hive.schema.HiveSchemaFactory.HiveSchema;
import com.google.common.collect.Sets;
+import org.apache.thrift.TException;
public class HiveDatabaseSchema extends AbstractSchema{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveDatabaseSchema.class);
private final HiveSchema hiveSchema;
- private final Set<String> tables;
+ private Set<String> tables;
+ private final DrillHiveMetaStoreClient mClient;
+ private final SchemaConfig schemaConfig;
public HiveDatabaseSchema( //
- List<String> tableList, //
HiveSchema hiveSchema, //
- String name) {
+ String name,
+ DrillHiveMetaStoreClient mClient,
+ SchemaConfig schemaConfig) {
super(hiveSchema.getSchemaPath(), name);
this.hiveSchema = hiveSchema;
- this.tables = Sets.newHashSet(tableList);
+ this.mClient = mClient;
+ this.schemaConfig = schemaConfig;
}
@Override
@@ -50,6 +56,14 @@ public class HiveDatabaseSchema extends AbstractSchema{
@Override
public Set<String> getTableNames() {
+ if (tables == null) {
+ try {
+ tables = Sets.newHashSet(mClient.getTableNames(this.name, schemaConfig.getIgnoreAuthErrors()));
+ } catch (final TException e) {
+ logger.warn("Failure while attempting to access HiveDatabase '{}'.", this.name, e.getCause());
+ tables = Sets.newHashSet(); // empty set.
+ }
+ }
return tables;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b8b18ebf/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
index 73e7bf7..16aca54 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/HiveSchemaFactory.java
@@ -120,15 +120,13 @@ public class HiveSchemaFactory implements SchemaFactory {
@Override
public AbstractSchema getSubSchema(String name) {
- List<String> tables;
try {
- List<String> dbs = mClient.getDatabases();
+ List<String> dbs = mClient.getDatabases(schemaConfig.getIgnoreAuthErrors());
if (!dbs.contains(name)) {
logger.debug("Database '{}' doesn't exists in Hive storage '{}'", name, schemaName);
return null;
}
- tables = mClient.getTableNames(name);
- HiveDatabaseSchema schema = new HiveDatabaseSchema(tables, this, name);
+ HiveDatabaseSchema schema = getSubSchemaKnownExists(name);
if (name.equals("default")) {
this.defaultSchema = schema;
}
@@ -137,12 +135,17 @@ public class HiveSchemaFactory implements SchemaFactory {
logger.warn("Failure while attempting to access HiveDatabase '{}'.", name, e.getCause());
return null;
}
+ }
+ /** Help method to get subschema when we know it exists (already checks the existence) */
+ private HiveDatabaseSchema getSubSchemaKnownExists(String name) {
+ HiveDatabaseSchema schema = new HiveDatabaseSchema(this, name, mClient, schemaConfig);
+ return schema;
}
void setHolder(SchemaPlus plusOfThis) {
for (String s : getSubSchemaNames()) {
- plusOfThis.add(s, getSubSchema(s));
+ plusOfThis.add(s, getSubSchemaKnownExists(s));
}
}
@@ -154,7 +157,7 @@ public class HiveSchemaFactory implements SchemaFactory {
@Override
public Set<String> getSubSchemaNames() {
try {
- List<String> dbs = mClient.getDatabases();
+ List<String> dbs = mClient.getDatabases(schemaConfig.getIgnoreAuthErrors());
return Sets.newHashSet(dbs);
} catch (final TException e) {
logger.warn("Failure while getting Hive database list.", e);
@@ -215,13 +218,6 @@ public class HiveSchemaFactory implements SchemaFactory {
public String getTypeName() {
return HiveStoragePluginConfig.NAME;
}
-
- @Override
- public void close() throws Exception {
- if (mClient != null) {
- mClient.close();
- }
- }
}
}