You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2017/11/14 02:36:29 UTC
hive git commit: HIVE-15436: Enhancing metastore APIs to retrieve
only materialized views (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 4d32b8fc3 -> 69e38e8b2
HIVE-15436: Enhancing metastore APIs to retrieve only materialized views (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/69e38e8b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/69e38e8b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/69e38e8b
Branch: refs/heads/master
Commit: 69e38e8b288ad2170b9aa13b2d7c1d7752782500
Parents: 4d32b8f
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Mon Nov 13 09:36:40 2017 -0800
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Mon Nov 13 18:28:20 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/metadata/Hive.java | 30 +++++++++++++++--
.../metadata/HiveMaterializedViewsRegistry.java | 30 ++++++++---------
.../hive/metastore/MetaStoreDirectSql.java | 34 ++++++++++++++++++++
.../hadoop/hive/metastore/ObjectStore.java | 29 ++++++++++++++++-
4 files changed, 102 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/69e38e8b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index cceea01..3e9fff1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1402,12 +1402,36 @@ public class Hive {
/**
* Get all tables for the specified database.
* @param dbName
- * @return List of table names
+ * @return List of all tables
* @throws HiveException
*/
public List<Table> getAllTableObjects(String dbName) throws HiveException {
+ return getTableObjects(dbName, ".*", null);
+ }
+
+ /**
+ * Get all materialized view names for the specified database.
+ * @param dbName
+ * @return List of materialized view table names
+ * @throws HiveException
+ */
+ public List<String> getAllMaterializedViews(String dbName) throws HiveException {
+ return getTablesByType(dbName, ".*", TableType.MATERIALIZED_VIEW);
+ }
+
+ /**
+ * Get all materialized views for the specified database.
+ * @param dbName
+ * @return List of materialized view table objects
+ * @throws HiveException
+ */
+ public List<Table> getAllMaterializedViewObjects(String dbName) throws HiveException {
+ return getTableObjects(dbName, ".*", TableType.MATERIALIZED_VIEW);
+ }
+
+ private List<Table> getTableObjects(String dbName, String pattern, TableType tableType) throws HiveException {
try {
- return Lists.transform(getMSC().getTableObjectsByName(dbName, getMSC().getAllTables(dbName)),
+ return Lists.transform(getMSC().getTableObjectsByName(dbName, getTablesByType(dbName, pattern, tableType)),
new com.google.common.base.Function<org.apache.hadoop.hive.metastore.api.Table, Table>() {
@Override
public Table apply(org.apache.hadoop.hive.metastore.api.Table table) {
@@ -1510,7 +1534,7 @@ public class Hive {
List<RelOptMaterialization> result = new ArrayList<>();
for (String dbName : getMSC().getAllDatabases()) {
// From metastore (for security)
- List<String> tables = getMSC().getAllTables(dbName);
+ List<String> tables = getAllMaterializedViews(dbName);
// Cached views (includes all)
Collection<RelOptMaterialization> cachedViews =
HiveMaterializedViewsRegistry.get().getRewritingMaterializedViews(dbName);
http://git-wip-us.apache.org/repos/asf/hive/blob/69e38e8b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index ba46ff7..51b6ef5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -121,32 +121,28 @@ public final class HiveMaterializedViewsRegistry {
* it does.
*/
public void init(final Hive db) {
- try {
- List<Table> tables = new ArrayList<Table>();
- for (String dbName : db.getAllDatabases()) {
- // TODO: We should enhance metastore API such that it returns only
- // materialized views instead of all tables
- tables.addAll(db.getAllTableObjects(dbName));
- }
- pool.submit(new Loader(tables));
- } catch (HiveException e) {
- LOG.error("Problem connecting to the metastore when initializing the view registry");
- }
+ pool.submit(new Loader(db));
}
private class Loader implements Runnable {
- private final List<Table> tables;
+ private final Hive db;
- private Loader(List<Table> tables) {
- this.tables = tables;
+ private Loader(Hive db) {
+ this.db = db;
}
@Override
public void run() {
- for (Table table : tables) {
- if (table.isMaterializedView()) {
- addMaterializedView(table);
+ try {
+ List<Table> materializedViews = new ArrayList<Table>();
+ for (String dbName : db.getAllDatabases()) {
+ materializedViews.addAll(db.getAllMaterializedViewObjects(dbName));
+ }
+ for (Table mv : materializedViews) {
+ addMaterializedView(mv);
}
+ } catch (HiveException e) {
+ LOG.error("Problem connecting to the metastore when initializing the view registry");
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/69e38e8b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index d9155c4..db60ff2 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -383,6 +383,39 @@ class MetaStoreDirectSql {
}
/**
+ * Get table names by using direct SQL queries.
+ *
+ * @param dbName Metastore database namme
+ * @param tableType Table type, or null if we want to get all tables
+ * @return list of table names
+ */
+ public List<String> getTables(String db_name, TableType tableType) throws MetaException {
+ List<String> ret = new ArrayList<String>();
+ String queryText = "SELECT " + TBLS + ".\"TBL_NAME\","
+ + " FROM " + TBLS + " "
+ + " INNER JOIN " + DBS + " ON " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" "
+ + " WHERE " + DBS + ".\"NAME\" = ? "
+ + (tableType == null ? "" : "AND " + TBLS + ".\"TBL_TYPE\" = ? ") ;
+
+ List<String> pms = new ArrayList<String>();
+ pms.add(db_name);
+ if (tableType != null) {
+ pms.add(tableType.toString());
+ }
+
+ Query<?> queryParams = pm.newQuery("javax.jdo.query.SQL", queryText);
+ List<Object[]> sqlResult = ensureList(executeWithArray(
+ queryParams, pms.toArray(), queryText));
+
+ if (!sqlResult.isEmpty()) {
+ for (Object[] line : sqlResult) {
+ ret.add(extractSqlString(line[0]));
+ }
+ }
+ return ret;
+ }
+
+ /**
* Gets partitions by using direct SQL queries.
* Note that batching is not needed for this method - list of names implies the batch size;
* @param dbName Metastore db name.
@@ -2250,4 +2283,5 @@ class MetaStoreDirectSql {
}
return ret;
}
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/69e38e8b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 575ba05..de7dc2d 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -188,7 +188,6 @@ import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
-import org.apache.hadoop.util.StringUtils;
import org.apache.thrift.TException;
import org.datanucleus.AbstractNucleusContext;
import org.datanucleus.ClassLoaderResolver;
@@ -1296,6 +1295,34 @@ public class ObjectStore implements RawStore, Configurable {
@Override
public List<String> getTables(String dbName, String pattern, TableType tableType) throws MetaException {
+ try {
+ // We only support pattern matching via jdo since pattern matching in Java
+ // might be different than the one used by the metastore backends
+ return getTablesInternal(dbName, pattern, tableType, (pattern == null || pattern.equals(".*")), true);
+ } catch (NoSuchObjectException e) {
+ throw new MetaException(ExceptionUtils.getStackTrace(e));
+ }
+ }
+
+ protected List<String> getTablesInternal(String dbName, String pattern, TableType tableType,
+ boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException {
+ final String db_name = normalizeIdentifier(dbName);
+ return new GetListHelper<String>(dbName, null, allowSql, allowJdo) {
+ @Override
+ protected List<String> getSqlResult(GetHelper<List<String>> ctx)
+ throws MetaException {
+ return directSql.getTables(db_name, tableType);
+ }
+
+ @Override
+ protected List<String> getJdoResult(GetHelper<List<String>> ctx)
+ throws MetaException, NoSuchObjectException {
+ return getTablesInternalViaJdo(db_name, pattern, tableType);
+ }
+ }.run(false);
+ }
+
+ private List<String> getTablesInternalViaJdo(String dbName, String pattern, TableType tableType) throws MetaException {
boolean commited = false;
Query query = null;
List<String> tbls = null;