You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/11/12 05:00:04 UTC
[40/55] [abbrv] hive git commit: HIVE-7575 GetTables thrift call is
very slow (Navis via Aihua Xu, reviewed by Szehon Ho, Aihua Xu)
http://git-wip-us.apache.org/repos/asf/hive/blob/b678ed85/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index a100e9f..9a1d159 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
@@ -220,6 +221,12 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
}
@Override
+ public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
+ throws MetaException {
+ return objectStore.getTableMeta(dbNames, tableNames, tableTypes);
+ }
+
+ @Override
public List<Table> getTableObjectsByName(String dbName, List<String> tableNames)
throws MetaException, UnknownDBException {
return objectStore.getTableObjectsByName(dbName, tableNames);
http://git-wip-us.apache.org/repos/asf/hive/blob/b678ed85/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index f6100e6..8dde0af 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.api.PrivilegeBag;
import org.apache.hadoop.hive.metastore.api.Role;
import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.metastore.api.Type;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
@@ -224,6 +225,12 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
+ public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
+ throws MetaException {
+ return Collections.emptyList();
+ }
+
+ @Override
public List<Table> getTableObjectsByName(String dbname, List<String> tableNames)
throws MetaException, UnknownDBException {
http://git-wip-us.apache.org/repos/asf/hive/blob/b678ed85/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
index 7af9d85..581a919 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -161,11 +162,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
Matcher matcher = pattern.matcher("");
Set<String> combinedTableNames = new HashSet<String>();
for (String tableName : tables.keySet()) {
- if (matcher == null) {
- matcher = pattern.matcher(tableName);
- } else {
- matcher.reset(tableName);
- }
+ matcher.reset(tableName);
if (matcher.matches()) {
combinedTableNames.add(tableName);
}
@@ -177,6 +174,55 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
Collections.sort(tableNames);
return tableNames;
}
+
+ @Override
+ public List<TableMeta> getTableMeta(String dbPatterns, String tablePatterns, List<String> tableTypes)
+ throws MetaException {
+ List<TableMeta> tableMetas = super.getTableMeta(dbPatterns, tablePatterns, tableTypes);
+ Map<String, Map<String, Table>> tmpTables = getTempTables();
+ if (tmpTables.isEmpty()) {
+ return tableMetas;
+ }
+
+ List<Matcher> dbPatternList = new ArrayList<>();
+ for (String element : dbPatterns.split("\\|")) {
+ dbPatternList.add(Pattern.compile(element.replaceAll("\\*", ".*")).matcher(""));
+ }
+ List<Matcher> tblPatternList = new ArrayList<>();
+ for (String element : tablePatterns.split("\\|")) {
+ tblPatternList.add(Pattern.compile(element.replaceAll("\\*", ".*")).matcher(""));
+ }
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, Map<String, Table>> outer : tmpTables.entrySet()) {
+ if (!matchesAny(outer.getKey(), dbPatternList)) {
+ continue;
+ }
+ for (Map.Entry<String, Table> inner : outer.getValue().entrySet()) {
+ Table table = inner.getValue();
+ String tableName = table.getTableName();
+ String typeString = table.getTableType().name();
+ if (tableTypes != null && !tableTypes.contains(typeString)) {
+ continue;
+ }
+ if (!matchesAny(inner.getKey(), tblPatternList)) {
+ continue;
+ }
+ TableMeta tableMeta = new TableMeta(table.getDbName(), tableName, typeString);
+ tableMeta.setComments(table.getProperty("comment"));
+ tableMetas.add(tableMeta);
+ }
+ }
+ return tableMetas;
+ }
+
+ private boolean matchesAny(String string, List<Matcher> matchers) {
+ for (Matcher matcher : matchers) {
+ if (matcher.reset(string).matches()) {
+ return true;
+ }
+ }
+ return matchers.isEmpty();
+ }
@Override
public List<org.apache.hadoop.hive.metastore.api.Table> getTableObjectsByName(String dbName,
@@ -508,12 +554,16 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
}
public static Map<String, Table> getTempTablesForDatabase(String dbName) {
+ return getTempTables().get(dbName);
+ }
+
+ public static Map<String, Map<String, Table>> getTempTables() {
SessionState ss = SessionState.get();
if (ss == null) {
LOG.debug("No current SessionState, skipping temp tables");
- return null;
+ return Collections.emptyMap();
}
- return ss.getTempTables().get(dbName);
+ return ss.getTempTables();
}
private Map<String, ColumnStatisticsObj> getTempTableColumnStatsForTable(String dbName,
http://git-wip-us.apache.org/repos/asf/hive/blob/b678ed85/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
index 296280f..65bbc1c 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java
@@ -22,14 +22,11 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.metadata.TableIterable;
+import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObjectUtils;
-import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.OperationState;
@@ -48,7 +45,7 @@ public class GetTablesOperation extends MetadataOperation {
private final String catalogName;
private final String schemaName;
private final String tableName;
- private final List<String> tableTypes = new ArrayList<String>();
+ private final List<String> tableTypeList;
private final RowSet rowSet;
private final TableTypeMapping tableTypeMapping;
@@ -58,7 +55,14 @@ public class GetTablesOperation extends MetadataOperation {
.addStringColumn("TABLE_SCHEM", "Schema name.")
.addStringColumn("TABLE_NAME", "Table name.")
.addStringColumn("TABLE_TYPE", "The table type, e.g. \"TABLE\", \"VIEW\", etc.")
- .addStringColumn("REMARKS", "Comments about the table.");
+ .addStringColumn("REMARKS", "Comments about the table.")
+ .addStringColumn("TYPE_CAT", "The types catalog.")
+ .addStringColumn("TYPE_SCHEM", "The types schema.")
+ .addStringColumn("TYPE_NAME", "Type name.")
+ .addStringColumn("SELF_REFERENCING_COL_NAME",
+ "Name of the designated \"identifier\" column of a typed table.")
+ .addStringColumn("REF_GENERATION",
+ "Specifies how values in SELF_REFERENCING_COL_NAME are created.");
protected GetTablesOperation(HiveSession parentSession,
String catalogName, String schemaName, String tableName,
@@ -72,7 +76,12 @@ public class GetTablesOperation extends MetadataOperation {
tableTypeMapping =
TableTypeMappingFactory.getTableTypeMapping(tableMappingStr);
if (tableTypes != null) {
- this.tableTypes.addAll(tableTypes);
+ tableTypeList = new ArrayList<String>();
+ for (String tableType : tableTypes) {
+ tableTypeList.add(tableTypeMapping.mapToHiveType(tableType.trim()));
+ }
+ } else {
+ tableTypeList = null;
}
this.rowSet = RowSetFactory.create(RESULT_SET_SCHEMA, getProtocolVersion());
}
@@ -91,23 +100,17 @@ public class GetTablesOperation extends MetadataOperation {
}
String tablePattern = convertIdentifierPattern(tableName, true);
- int maxBatchSize = SessionState.get().getConf().getIntVar(ConfVars.METASTORE_BATCH_RETRIEVE_MAX);
- for (String dbName : metastoreClient.getDatabases(schemaPattern)) {
- List<String> tableNames = metastoreClient.getTables(dbName, tablePattern);
- for (Table table : new TableIterable(metastoreClient, dbName, tableNames, maxBatchSize)) {
- Object[] rowData = new Object[] {
+ for (TableMeta tableMeta :
+ metastoreClient.getTableMeta(schemaPattern, tablePattern, tableTypeList)) {
+ rowSet.addRow(new Object[] {
DEFAULT_HIVE_CATALOG,
- table.getDbName(),
- table.getTableName(),
- tableTypeMapping.mapToClientType(table.getTableType()),
- table.getParameters().get("comment")
- };
- if (tableTypes.isEmpty() || tableTypes.contains(
- tableTypeMapping.mapToClientType(table.getTableType()))) {
- rowSet.addRow(rowData);
- }
- }
+ tableMeta.getDbName(),
+ tableMeta.getTableName(),
+ tableTypeMapping.mapToClientType(tableMeta.getTableType()),
+ tableMeta.getComments(),
+ null, null, null, null, null
+ });
}
setState(OperationState.FINISHED);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/b678ed85/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java b/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java
index 4595ef5..285b4f9 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java
@@ -18,7 +18,6 @@
package org.apache.hive.service.cli.operation;
-import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -95,16 +94,30 @@ public abstract class MetadataOperation extends Operation {
* other hand is done locally inside the hive code and that requires the regex wildchar
* format '.*' This is driven by the datanucleusFormat flag.
*/
- private String convertPattern(final String pattern, boolean datanucleusFormat) {
+ private String convertPattern(String pattern, boolean datanucleusFormat) {
String wStr;
if (datanucleusFormat) {
wStr = "*";
} else {
wStr = ".*";
}
- return pattern
- .replaceAll("([^\\\\])%", "$1" + wStr).replaceAll("\\\\%", "%").replaceAll("^%", wStr)
- .replaceAll("([^\\\\])_", "$1.").replaceAll("\\\\_", "_").replaceAll("^_", ".");
+ pattern = replaceAll(pattern, "([^\\\\])%", "$1" + wStr);
+ pattern = replaceAll(pattern, "\\\\%", "%");
+ pattern = replaceAll(pattern, "^%", wStr);
+ pattern = replaceAll(pattern, "([^\\\\])_", "$1.");
+ pattern = replaceAll(pattern, "\\\\_", "_");
+ pattern = replaceAll(pattern, "^_", ".");
+ return pattern;
+ }
+
+ private String replaceAll(String input, final String pattern, final String replace) {
+ while (true) {
+ String replaced = input.replaceAll(pattern, replace);
+ if (replaced.equals(input)) {
+ return replaced;
+ }
+ input = replaced;
+ }
}
protected boolean isAuthV2Enabled(){