You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/10/15 23:01:44 UTC
hive git commit: HIVE-12053 : Stats performance regression caused by
HIVE-11786 (Chaoyu Tang, reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/master 6462daf0b -> 4afcc31ba
HIVE-12053 : Stats performance regression caused by HIVE-11786 (Chaoyu Tang, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4afcc31b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4afcc31b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4afcc31b
Branch: refs/heads/master
Commit: 4afcc31baa1dab5357da2505b4093a5e697777b5
Parents: 6462daf
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 15 14:01:37 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 15 14:01:37 2015 -0700
----------------------------------------------------------------------
.../hive/metastore/MetaStoreDirectSql.java | 62 ++++++--------------
.../hadoop/hive/metastore/ObjectStore.java | 33 +++++------
.../hive/metastore/StatObjectConverter.java | 20 +++----
.../metastore/txn/CompactionTxnHandler.java | 50 ++--------------
.../model/MPartitionColumnStatistics.java | 12 ++++
.../metastore/model/MTableColumnStatistics.java | 8 +++
.../hive/metastore/VerifyingObjectStore.java | 28 +--------
7 files changed, 71 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4afcc31b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 3455a92..95b1ccc 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1122,9 +1122,9 @@ class MetaStoreDirectSql {
doDbSpecificInitializationsBeforeQuery();
boolean doTrace = LOG.isDebugEnabled();
long start = doTrace ? System.nanoTime() : 0;
- String queryText = "select " + STATS_COLLIST + " from " + STATS_TABLE_JOINED_TBLS
- + "where " + STATS_DB_NAME + " = ? and " + STATS_TABLE_NAME + " = ? "
- + "and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")";
+ String queryText = "select " + STATS_COLLIST + " from \"TAB_COL_STATS\" "
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in ("
+ + makeParams(colNames.size()) + ")";
Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
Object[] params = new Object[colNames.size() + 2];
params[0] = dbName;
@@ -1214,11 +1214,11 @@ class MetaStoreDirectSql {
assert !colNames.isEmpty() && !partNames.isEmpty();
long partsFound = 0;
boolean doTrace = LOG.isDebugEnabled();
- String queryText = "select count(\"COLUMN_NAME\") from " + STATS_PART_JOINED_TBLS
- + "where " + STATS_DB_NAME + " = ? and " + STATS_TABLE_NAME + " = ? "
- + "and \"PART_COL_STATS\".\"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ") "
- + "and " + STATS_PARTITION_NAME + " in (" + makeParams(partNames.size()) + ") "
- + "group by " + STATS_PARTITION_NAME;
+ String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\""
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
+ + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ + " group by \"PARTITION_NAME\"";
long start = doTrace ? System.nanoTime() : 0;
Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
Object qResult = executeWithArray(query, prepareParams(
@@ -1263,7 +1263,7 @@ class MetaStoreDirectSql {
+ "avg((\"LONG_HIGH_VALUE\"-\"LONG_LOW_VALUE\")/cast(\"NUM_DISTINCTS\" as decimal)),"
+ "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\"),"
+ "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\"),"
- + "sum(\"NUM_DISTINCTS\")" + " from " + PART_COL_STATS_VW
+ + "sum(\"NUM_DISTINCTS\")" + " from \"PART_COL_STATS\""
+ " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? ";
String queryText = null;
long start = 0;
@@ -1302,7 +1302,7 @@ class MetaStoreDirectSql {
// We need to extrapolate this partition based on the other partitions
List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(colNames.size());
queryText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", count(\"PARTITION_NAME\") "
- + " from " + PART_COL_STATS_VW + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ + " from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")"
+ " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ " group by \"COLUMN_NAME\", \"COLUMN_TYPE\"";
@@ -1367,7 +1367,7 @@ class MetaStoreDirectSql {
// get sum for all columns to reduce the number of queries
Map<String, Map<Integer, Object>> sumMap = new HashMap<String, Map<Integer, Object>>();
queryText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\"), sum(\"NUM_DISTINCTS\")"
- + " from " + PART_COL_STATS_VW
+ + " from \"PART_COL_STATS\""
+ " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "
+ " and \"COLUMN_NAME\" in ("
+ makeParams(extraColumnNameTypeParts.size())
@@ -1444,13 +1444,13 @@ class MetaStoreDirectSql {
// left/right borders
if (!decimal) {
queryText = "select \"" + colStatName
- + "\",\"PARTITION_NAME\" from " + PART_COL_STATS_VW
+ + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\""
+ " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+ " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ " order by \"" + colStatName + "\"";
} else {
queryText = "select \"" + colStatName
- + "\",\"PARTITION_NAME\" from " + PART_COL_STATS_VW
+ + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\""
+ " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?"
+ " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")"
+ " order by cast(\"" + colStatName + "\" as decimal)";
@@ -1482,7 +1482,7 @@ class MetaStoreDirectSql {
+ "avg((\"LONG_HIGH_VALUE\"-\"LONG_LOW_VALUE\")/cast(\"NUM_DISTINCTS\" as decimal)),"
+ "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\"),"
+ "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\")"
- + " from " + PART_COL_STATS_VW + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?"
+ + " from \"PART_COL_STATS\"" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?"
+ " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in ("
+ makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\"";
start = doTrace ? System.nanoTime() : 0;
@@ -1558,11 +1558,10 @@ class MetaStoreDirectSql {
boolean doTrace = LOG.isDebugEnabled();
doDbSpecificInitializationsBeforeQuery();
long start = doTrace ? System.nanoTime() : 0;
- String queryText = "select " + STATS_PARTITION_NAME + ", " + STATS_COLLIST + " from "
- + STATS_PART_JOINED_TBLS + " where " + STATS_DB_NAME + " = ? and " + STATS_TABLE_NAME + " = ? "
- + "and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ") "
- + "and " + STATS_PARTITION_NAME + " in (" + makeParams(partNames.size()) + ") "
- + "order by " + STATS_PARTITION_NAME + " asc";
+ String queryText = "select \"PARTITION_NAME\", " + STATS_COLLIST + " from \"PART_COL_STATS\""
+ + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in ("
+ + makeParams(colNames.size()) + ") AND \"PARTITION_NAME\" in ("
+ + makeParams(partNames.size()) + ") order by \"PARTITION_NAME\"";
Query query = pm.newQuery("javax.jdo.query.SQL", queryText);
Object qResult = executeWithArray(query, prepareParams(
@@ -1604,31 +1603,6 @@ class MetaStoreDirectSql {
+ "\"BIG_DECIMAL_HIGH_VALUE\", \"NUM_NULLS\", \"NUM_DISTINCTS\", \"AVG_COL_LEN\", "
+ "\"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\" ";
- private static final String STATS_PART_JOINED_TBLS = "\"PART_COL_STATS\" "
- + "JOIN \"PARTITIONS\" ON \"PART_COL_STATS\".\"PART_ID\" = \"PARTITIONS\".\"PART_ID\" "
- + "JOIN \"TBLS\" ON \"PARTITIONS\".\"TBL_ID\" = \"TBLS\".\"TBL_ID\" "
- + "JOIN \"DBS\" ON \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" ";
-
- private static final String STATS_TABLE_JOINED_TBLS = "\"TAB_COL_STATS\" "
- + "JOIN \"TBLS\" ON \"TAB_COL_STATS\".\"TBL_ID\" = \"TBLS\".\"TBL_ID\" "
- + "JOIN \"DBS\" ON \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" ";
-
- private static final String PART_COL_STATS_VW = "(SELECT \"DBS\".\"NAME\" \"DB_NAME\", "
- + "\"TBLS\".\"TBL_NAME\" \"TABLE_NAME\", \"PARTITIONS\".\"PART_NAME\" \"PARTITION_NAME\", "
- + "\"PCS\".\"COLUMN_NAME\", \"PCS\".\"COLUMN_TYPE\", \"PCS\".\"LONG_LOW_VALUE\", "
- + "\"PCS\".\"LONG_HIGH_VALUE\", \"PCS\".\"DOUBLE_HIGH_VALUE\", \"PCS\".\"DOUBLE_LOW_VALUE\", "
- + "\"PCS\".\"BIG_DECIMAL_LOW_VALUE\", \"PCS\".\"BIG_DECIMAL_HIGH_VALUE\", \"PCS\".\"NUM_NULLS\", "
- + "\"PCS\".\"NUM_DISTINCTS\", \"PCS\".\"AVG_COL_LEN\",\"PCS\".\"MAX_COL_LEN\", "
- + "\"PCS\".\"NUM_TRUES\", \"PCS\".\"NUM_FALSES\",\"PCS\".\"LAST_ANALYZED\" "
- + "FROM \"PART_COL_STATS\" \"PCS\" JOIN \"PARTITIONS\" "
- + "ON (\"PCS\".\"PART_ID\" = \"PARTITIONS\".\"PART_ID\") "
- + "JOIN \"TBLS\" ON (\"PARTITIONS\".\"TBL_ID\" = \"TBLS\".\"TBL_ID\") "
- + "JOIN \"DBS\" ON (\"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\")) VW ";
-
- private static final String STATS_DB_NAME = "\"DBS\".\"NAME\" ";
- private static final String STATS_TABLE_NAME = "\"TBLS\".\"TBL_NAME\" ";
- private static final String STATS_PARTITION_NAME = "\"PARTITIONS\".\"PART_NAME\" ";
-
private ColumnStatistics makeColumnStats(
List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException {
ColumnStatistics result = new ColumnStatistics();
http://git-wip-us.apache.org/repos/asf/hive/blob/4afcc31b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 136eff5..a02f179 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -6266,8 +6266,8 @@ public class ObjectStore implements RawStore, Configurable {
private void writeMTableColumnStatistics(Table table, MTableColumnStatistics mStatsObj)
throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
- String tableName = mStatsObj.getTable().getTableName();
- String dbName = mStatsObj.getTable().getDatabase().getName();
+ String dbName = mStatsObj.getDbName();
+ String tableName = mStatsObj.getTableName();
String colName = mStatsObj.getColName();
QueryWrapper queryWrapper = new QueryWrapper();
@@ -6293,9 +6293,9 @@ public class ObjectStore implements RawStore, Configurable {
private void writeMPartitionColumnStatistics(Table table, Partition partition,
MPartitionColumnStatistics mStatsObj) throws NoSuchObjectException,
MetaException, InvalidObjectException, InvalidInputException {
- String partName = mStatsObj.getPartition().getPartitionName();
- String tableName = mStatsObj.getPartition().getTable().getTableName();
- String dbName = mStatsObj.getPartition().getTable().getDatabase().getName();
+ String dbName = mStatsObj.getDbName();
+ String tableName = mStatsObj.getTableName();
+ String partName = mStatsObj.getPartitionName();
String colName = mStatsObj.getColName();
LOG.info("Updating partition level column statistics for db=" + dbName + " tableName=" +
@@ -6401,7 +6401,7 @@ public class ObjectStore implements RawStore, Configurable {
List<MTableColumnStatistics> result = null;
validateTableCols(table, colNames);
Query query = queryWrapper.query = pm.newQuery(MTableColumnStatistics.class);
- String filter = "table.tableName == t1 && table.database.name == t2 && (";
+ String filter = "tableName == t1 && dbName == t2 && (";
String paramStr = "java.lang.String t1, java.lang.String t2";
Object[] params = new Object[colNames.size() + 2];
params[0] = table.getTableName();
@@ -6527,7 +6527,7 @@ public class ObjectStore implements RawStore, Configurable {
for (int i = 0; i <= mStats.size(); ++i) {
boolean isLast = i == mStats.size();
MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
- String partName = isLast ? null : (String)mStatsObj.getPartition().getPartitionName();
+ String partName = isLast ? null : (String)mStatsObj.getPartitionName();
if (isLast || !partName.equals(lastPartName)) {
if (i != 0) {
result.add(new ColumnStatistics(csd, curList));
@@ -6593,14 +6593,14 @@ public class ObjectStore implements RawStore, Configurable {
validateTableCols(table, colNames);
Query query = queryWrapper.query = pm.newQuery(MPartitionColumnStatistics.class);
String paramStr = "java.lang.String t1, java.lang.String t2";
- String filter = "partition.table.tableName == t1 && partition.table.database.name == t2 && (";
+ String filter = "tableName == t1 && dbName == t2 && (";
Object[] params = new Object[colNames.size() + partNames.size() + 2];
int i = 0;
params[i++] = table.getTableName();
params[i++] = table.getDbName();
int firstI = i;
for (String s : partNames) {
- filter += ((i == firstI) ? "" : " || ") + "partition.partitionName == p" + i;
+ filter += ((i == firstI) ? "" : " || ") + "partitionName == p" + i;
paramStr += ", java.lang.String p" + i;
params[i++] = s;
}
@@ -6614,7 +6614,7 @@ public class ObjectStore implements RawStore, Configurable {
filter += ")";
query.setFilter(filter);
query.declareParameters(paramStr);
- query.setOrdering("partition.partitionName ascending");
+ query.setOrdering("partitionName ascending");
@SuppressWarnings("unchecked")
List<MPartitionColumnStatistics> result =
(List<MPartitionColumnStatistics>) query.executeWithArray(params);
@@ -6639,7 +6639,7 @@ public class ObjectStore implements RawStore, Configurable {
String dbName, String tableName, List<String> partNames) throws MetaException {
ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(
dbName, tableName, partNames, MPartitionColumnStatistics.class,
- "partition.table.tableName", "partition.table.database.name", "partition.partitionName");
+ "tableName", "dbName", "partition.partitionName");
queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond());
}
@@ -6674,14 +6674,13 @@ public class ObjectStore implements RawStore, Configurable {
String parameters;
if (colName != null) {
filter =
- "partition.partitionName == t1 && partition.table.database.name == t2 && "
- + "partition.table.tableName == t3 && colName == t4";
+ "partition.partitionName == t1 && dbName == t2 && tableName == t3 && "
+ + "colName == t4";
parameters =
"java.lang.String t1, java.lang.String t2, "
+ "java.lang.String t3, java.lang.String t4";
} else {
- filter = "partition.partitionName == t1 && partition.table.database.name == t2 && "
- + " partition.table.tableName == t3";
+ filter = "partition.partitionName == t1 && dbName == t2 && tableName == t3";
parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3";
}
query.setFilter(filter);
@@ -6752,10 +6751,10 @@ public class ObjectStore implements RawStore, Configurable {
String filter;
String parameters;
if (colName != null) {
- filter = "table.tableName == t1 && table.database.name == t2 && colName == t3";
+ filter = "table.tableName == t1 && dbName == t2 && colName == t3";
parameters = "java.lang.String t1, java.lang.String t2, java.lang.String t3";
} else {
- filter = "table.tableName == t1 && table.database.name == t2";
+ filter = "table.tableName == t1 && dbName == t2";
parameters = "java.lang.String t1, java.lang.String t2";
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4afcc31b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
index dc56a8f..b3ceff1 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java
@@ -58,8 +58,8 @@ public class StatObjectConverter {
MTableColumnStatistics mColStats = new MTableColumnStatistics();
mColStats.setTable(table);
- mColStats.setDbName("Deprecated");
- mColStats.setTableName("Deprecated");
+ mColStats.setDbName(statsDesc.getDbName());
+ mColStats.setTableName(statsDesc.getTableName());
mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
mColStats.setColName(statsObj.getColName());
mColStats.setColType(statsObj.getColType());
@@ -289,8 +289,8 @@ public class StatObjectConverter {
MTableColumnStatistics mStatsObj) {
ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
statsDesc.setIsTblLevel(true);
- statsDesc.setTableName(mStatsObj.getTable().getTableName());
- statsDesc.setDbName(mStatsObj.getTable().getDatabase().getName());
+ statsDesc.setDbName(mStatsObj.getDbName());
+ statsDesc.setTableName(mStatsObj.getTableName());
statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
return statsDesc;
}
@@ -304,9 +304,9 @@ public class StatObjectConverter {
MPartitionColumnStatistics mColStats = new MPartitionColumnStatistics();
mColStats.setPartition(partition);
- mColStats.setDbName("Deprecated");
- mColStats.setTableName("Deprecated");
- mColStats.setPartitionName("Deprecated");
+ mColStats.setDbName(statsDesc.getDbName());
+ mColStats.setTableName(statsDesc.getTableName());
+ mColStats.setPartitionName(statsDesc.getPartName());
mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed());
mColStats.setColName(statsObj.getColName());
mColStats.setColType(statsObj.getColType());
@@ -442,9 +442,9 @@ public class StatObjectConverter {
MPartitionColumnStatistics mStatsObj) {
ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
statsDesc.setIsTblLevel(false);
- statsDesc.setPartName(mStatsObj.getPartition().getPartitionName());
- statsDesc.setTableName(mStatsObj.getPartition().getTable().getTableName());
- statsDesc.setDbName(mStatsObj.getPartition().getTable().getDatabase().getName());
+ statsDesc.setDbName(mStatsObj.getDbName());
+ statsDesc.setTableName(mStatsObj.getTableName());
+ statsDesc.setPartName(mStatsObj.getPartitionName());
statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
return statsDesc;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4afcc31b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
index 7d0a76a..44ee5c6 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -554,18 +554,15 @@ public class CompactionTxnHandler extends TxnHandler {
StringBuilder bldr = new StringBuilder();
bldr.append("SELECT ").append(quote).append("COLUMN_NAME").append(quote)
.append(" FROM ")
- .append((ci.partName == null ?
- getTableColStatsJoinedTables(quote) : getPartitionColStatsJoinedTables(quote)))
+ .append(quote).append((ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS"))
+ .append(quote)
.append(" WHERE ")
- .append(quote).append("DBS").append(quote).append(".").append(quote).append("NAME").append(quote)
- .append(" = '").append(ci.dbname)
- .append("' AND ")
- .append(quote).append("TBLS").append(quote).append(".").append(quote).append("TBL_NAME").append(quote)
+ .append(quote).append("DB_NAME").append(quote).append(" = '").append(ci.dbname)
+ .append("' AND ").append(quote).append("TABLE_NAME").append(quote)
.append(" = '").append(ci.tableName).append("'");
if (ci.partName != null) {
- bldr.append(" AND ")
- .append(quote).append("PARTITIONS").append(quote).append(".").append(quote).append("PART_NAME").append(quote)
- .append(" = '").append(ci.partName).append("'");
+ bldr.append(" AND ").append(quote).append("PARTITION_NAME").append(quote).append(" = '")
+ .append(ci.partName).append("'");
}
String s = bldr.toString();
@@ -615,41 +612,6 @@ public class CompactionTxnHandler extends TxnHandler {
}
return new ValidCompactorTxnList(exceptions, minOpenTxn, highWater);
}
-
- private String getTableColStatsJoinedTables(String quote) {
- return (new StringBuffer(quote)).append("TAB_COL_STATS").append(quote)
- .append(" JOIN ").append(quote).append("TBLS").append(quote)
- .append(" ON ").append(quote).append("TAB_COL_STATS").append(quote)
- .append(".").append(quote).append("TBL_ID").append(quote)
- .append(" = ").append(quote).append("TBLS").append(quote)
- .append(".").append(quote).append("TBL_ID").append(quote)
- .append(" JOIN ").append(quote).append("DBS").append(quote)
- .append(" ON ").append(quote).append("TBLS").append(quote)
- .append(".").append(quote).append("DB_ID").append(quote)
- .append(" = ").append(quote).append("DBS").append(quote)
- .append(".").append(quote).append("DB_ID").append(quote).toString();
- }
-
- private String getPartitionColStatsJoinedTables(String quote) {
- //actually we do not have to get the quote from database since double quoted identifier
- //should work on all favors of db so far Hive supports.
- return (new StringBuffer(quote)).append("PART_COL_STATS").append(quote)
- .append(" JOIN ").append(quote).append("PARTITIONS").append(quote)
- .append(" ON ").append(quote).append("PART_COL_STATS").append(quote)
- .append(".").append(quote).append("PART_ID").append(quote)
- .append(" = ").append(quote).append("PARTITIONS").append(quote)
- .append(".").append(quote).append("PART_ID").append(quote)
- .append(" JOIN ").append(quote).append("TBLS").append(quote)
- .append(" ON ").append(quote).append("PARTITIONS").append(quote)
- .append(".").append(quote).append("TBL_ID").append(quote)
- .append(" = ").append(quote).append("TBLS").append(quote)
- .append(".").append(quote).append("TBL_ID").append(quote)
- .append(" JOIN ").append(quote).append("DBS").append(quote)
- .append(" ON ").append(quote).append("TBLS").append(quote)
- .append(".").append(quote).append("DB_ID").append(quote)
- .append(" = ").append(quote).append("DBS").append(quote)
- .append(".").append(quote).append("DB_ID").append(quote).toString();
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4afcc31b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
----------------------------------------------------------------------
diff --git a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
index 70608a9..2967a60 100644
--- a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
+++ b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionColumnStatistics.java
@@ -56,6 +56,10 @@ public class MPartitionColumnStatistics {
public MPartitionColumnStatistics() {}
+ public String getTableName() {
+ return tableName;
+ }
+
public void setTableName(String tableName) {
this.tableName = tableName;
}
@@ -124,6 +128,10 @@ public class MPartitionColumnStatistics {
this.lastAnalyzed = lastAnalyzed;
}
+ public String getDbName() {
+ return dbName;
+ }
+
public void setDbName(String dbName) {
this.dbName = dbName;
}
@@ -136,6 +144,10 @@ public class MPartitionColumnStatistics {
this.partition = partition;
}
+ public String getPartitionName() {
+ return partitionName;
+ }
+
public void setPartitionName(String partitionName) {
this.partitionName = partitionName;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4afcc31b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
----------------------------------------------------------------------
diff --git a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
index d8dcf5b..132f7a1 100644
--- a/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
+++ b/metastore/src/model/org/apache/hadoop/hive/metastore/model/MTableColumnStatistics.java
@@ -62,6 +62,10 @@ public class MTableColumnStatistics {
this.table = table;
}
+ public String getTableName() {
+ return tableName;
+ }
+
public void setTableName(String tableName) {
this.tableName = tableName;
}
@@ -138,6 +142,10 @@ public class MTableColumnStatistics {
this.lastAnalyzed = lastAnalyzed;
}
+ public String getDbName() {
+ return dbName;
+ }
+
public void setDbName(String dbName) {
this.dbName = dbName;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/4afcc31b/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java
index 8d3819a..7e46523 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java
@@ -25,8 +25,6 @@ import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -38,7 +36,6 @@ import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
-import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -107,21 +104,10 @@ class VerifyingObjectStore extends ObjectStore {
dbName, tableName, colNames, true, false);
ColumnStatistics jdoResult = getTableColumnStatisticsInternal(
dbName, tableName, colNames, false, true);
- if (sqlResult != null && jdoResult != null) {
- Collections.sort(sqlResult.getStatsObj(), new ColumnStatsComparator());
- Collections.sort(jdoResult.getStatsObj(), new ColumnStatsComparator());
- }
verifyObjects(sqlResult, jdoResult, ColumnStatistics.class);
return sqlResult;
}
- private static class ColumnStatsComparator implements Comparator<ColumnStatisticsObj> {
- @Override
- public int compare(ColumnStatisticsObj obj1, ColumnStatisticsObj obj2) {
- return obj1.getColName().compareTo(obj2.getColName());
- }
- }
-
@Override
public List<ColumnStatistics> getPartitionColumnStatistics(String dbName,
String tableName, List<String> partNames, List<String> colNames)
@@ -130,19 +116,7 @@ class VerifyingObjectStore extends ObjectStore {
dbName, tableName, partNames, colNames, true, false);
List<ColumnStatistics> jdoResult = getPartitionColumnStatisticsInternal(
dbName, tableName, partNames, colNames, false, true);
-
- if (sqlResult.size() != jdoResult.size()) {
- String msg = "Lists are not the same size: SQL " + sqlResult.size()
- + ", ORM " + jdoResult.size();
- LOG.error(msg);
- throw new MetaException(msg);
- }
-
- for (int i = 0; i < jdoResult.size(); i++) {
- Collections.sort(sqlResult.get(i).getStatsObj(), new ColumnStatsComparator());
- Collections.sort(jdoResult.get(i).getStatsObj(), new ColumnStatsComparator());
- verifyObjects(sqlResult.get(i), jdoResult.get(i), ColumnStatistics.class);
- }
+ verifyLists(sqlResult, jdoResult, ColumnStatistics.class);
return sqlResult;
}