You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2023/05/22 06:35:59 UTC
[hive] branch master updated: HIVE-25366: Reduce number of Table calls in updatePartitonColStatsInternal (Dmitriy Fingerman, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6aff0d006e8 HIVE-25366: Reduce number of Table calls in updatePartitonColStatsInternal (Dmitriy Fingerman, reviewed by Denys Kuzmenko)
6aff0d006e8 is described below
commit 6aff0d006e81d26cc8dc68dec546eb9650a8e941
Author: Dmitriy Fingerman <dm...@gmail.com>
AuthorDate: Mon May 22 02:35:50 2023 -0400
HIVE-25366: Reduce number of Table calls in updatePartitonColStatsInternal (Dmitriy Fingerman, reviewed by Denys Kuzmenko)
Closes #4319
---
.../hcatalog/listener/DummyRawStoreFailEvent.java | 13 +++++++++
.../apache/hadoop/hive/metastore/AlterHandler.java | 5 ++--
.../apache/hadoop/hive/metastore/HMSHandler.java | 26 +++++++++--------
.../hadoop/hive/metastore/HiveAlterHandler.java | 14 +++++----
.../apache/hadoop/hive/metastore/ObjectStore.java | 19 +++++++++----
.../org/apache/hadoop/hive/metastore/RawStore.java | 11 ++++++--
.../hadoop/hive/metastore/cache/CachedStore.java | 33 ++++++++++++++++++----
.../metastore/DummyRawStoreControlledCommit.java | 14 +++++++++
.../metastore/DummyRawStoreForJdoConnection.java | 16 +++++++++--
.../hadoop/hive/metastore/TestObjectStore.java | 4 ++-
.../hadoop/hive/metastore/TestOldSchema.java | 4 ++-
.../hive/metastore/cache/TestCachedStore.java | 4 ++-
12 files changed, 127 insertions(+), 36 deletions(-)
diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
index 464555da4d6..a242284250e 100644
--- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
+++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java
@@ -117,6 +117,7 @@ import org.apache.hadoop.hive.metastore.api.WMNullablePool;
import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
import org.apache.hadoop.hive.metastore.api.ReplicationMetricList;
import org.apache.hadoop.hive.metastore.api.GetReplicationMetricsRequest;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo;
import org.apache.thrift.TException;
@@ -884,6 +885,13 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
return objectStore.updateTableColumnStatistics(statsObj, validWriteIds, writeId);
}
+ @Override
+ public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics statsObj,
+ List<String> partVals, String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ return objectStore.updatePartitionColumnStatistics(table, mTable, statsObj, partVals, validWriteIds, writeId);
+ }
+
@Override
public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics statsObj,
List<String> partVals, String validWriteIds, long writeId)
@@ -1592,6 +1600,11 @@ public class DummyRawStoreFailEvent implements RawStore, Configurable {
objectStore.dropPackage(request);
}
+ @Override
+ public MTable ensureGetMTable(String s, String s1, String s2) throws NoSuchObjectException {
+ return objectStore.ensureGetMTable(s, s1, s2);
+ }
+
@Override
public Map<String, Map<String, String>> updatePartitionColumnStatisticsInBatch(
Map<String, ColumnStatistics> partColStatsMap,
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java
index d903a912b91..6edbef39397 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -118,7 +119,7 @@ public interface AlterHandler extends Configurable {
Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
final String name, final List<String> part_vals, final Partition new_part,
EnvironmentContext environmentContext)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException;
+ throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException, NoSuchObjectException;
/**
* handles alter partition
@@ -147,7 +148,7 @@ public interface AlterHandler extends Configurable {
final String dbname, final String name, final List<String> part_vals,
final Partition new_part, EnvironmentContext environmentContext,
IHMSHandler handler, String validWriteIds)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException;
+ throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException, NoSuchObjectException;
/**
* @deprecated As of release 3.0.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String,
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 9033358a7cd..66303f12ef8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType;
import org.apache.hadoop.hive.metastore.metrics.Metrics;
import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.metastore.metrics.PerfLogger;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.txn.*;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
@@ -4106,7 +4107,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
+ TableName.getQualified(catName, dbName, tblName) +
" does not exist");
}
-
+ MTable mTable = getMS().ensureGetMTable(catName, dbName, tblName);
db = ms.getDatabase(catName, dbName);
if (!parts.isEmpty()) {
@@ -4187,7 +4188,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
new long[0], new BitSet(), writeId);
validWriteIds = validWriteIdList.toString();
}
- updatePartitonColStatsInternal(tbl, partColStats, validWriteIds, writeId);
+ updatePartitonColStatsInternal(tbl, mTable, partColStats, validWriteIds, writeId);
}
success = ms.commitTransaction();
@@ -7021,7 +7022,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
colStats.setStatsObj(colStats.getStatsObj());
}
- private boolean updatePartitonColStatsInternal(Table tbl, ColumnStatistics colStats,
+ private boolean updatePartitonColStatsInternal(Table tbl, MTable mTable, ColumnStatistics colStats,
String validWriteIds, long writeId)
throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException {
normalizeColStatsInput(colStats);
@@ -7037,13 +7038,12 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
List<String> partVals;
boolean committed = false;
getMS().openTransaction();
-
+
try {
- if (tbl == null) {
- tbl = getTable(catName, dbName, tableName);
- }
+ tbl = Optional.ofNullable(tbl).orElse(getTable(catName, dbName, tableName));
+ mTable = Optional.ofNullable(mTable).orElse(getMS().ensureGetMTable(catName, dbName, tableName));
partVals = getPartValsFromName(tbl, csd.getPartName());
- parameters = getMS().updatePartitionColumnStatistics(colStats, partVals, validWriteIds, writeId);
+ parameters = getMS().updatePartitionColumnStatistics(tbl, mTable, colStats, partVals, validWriteIds, writeId);
if (parameters != null) {
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
@@ -7139,7 +7139,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
@Override
public boolean update_partition_column_statistics(ColumnStatistics colStats) throws TException {
// Deprecated API.
- return updatePartitonColStatsInternal(null, colStats, null, -1);
+ return updatePartitonColStatsInternal(null, null, colStats, null, -1);
}
@@ -7155,7 +7155,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
throw new InvalidInputException("Merge is not supported for non-aggregate stats");
}
ColumnStatistics colStats = req.getColStatsIterator().next();
- boolean ret = updatePartitonColStatsInternal(null, colStats,
+ boolean ret = updatePartitonColStatsInternal(null, null, colStats,
req.getValidWriteIdList(), req.getWriteId());
return new SetPartitionsStatsResponse(ret);
}
@@ -9169,6 +9169,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
colNames, newStatsMap, request);
} else { // No merge.
Table t = getTable(catName, dbName, tableName);
+ MTable mTable = getMS().ensureGetMTable(catName, dbName, tableName);
// We don't short-circuit on errors here anymore. That can leave acid stats invalid.
if (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL)) {
ret = updatePartitionColStatsInBatch(t, newStatsMap,
@@ -9176,7 +9177,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
} else {
for (Map.Entry<String, ColumnStatistics> entry : newStatsMap.entrySet()) {
// We don't short-circuit on errors here anymore. That can leave acid stats invalid.
- ret = updatePartitonColStatsInternal(t, entry.getValue(),
+ ret = updatePartitonColStatsInternal(t, mTable, entry.getValue(),
request.getValidWriteIdList(), request.getWriteId()) && ret;
}
}
@@ -9214,6 +9215,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
}
Table t = getTable(catName, dbName, tableName);
+ MTable mTable = getMS().ensureGetMTable(catName, dbName, tableName);
Map<String, ColumnStatistics> statsMap = new HashMap<>();
boolean useDirectSql = MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL);
for (Map.Entry<String, ColumnStatistics> entry : newStatsMap.entrySet()) {
@@ -9239,7 +9241,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
if (useDirectSql) {
statsMap.put(csNew.getStatsDesc().getPartName(), csNew);
} else {
- result = updatePartitonColStatsInternal(t, csNew,
+ result = updatePartitonColStatsInternal(t, mTable, csNew,
request.getValidWriteIdList(), request.getWriteId()) && result;
}
} else if (isInvalidTxnStats) {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index fc413504ec1..4f03e5d5b58 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
@@ -390,12 +391,14 @@ public class HiveAlterHandler implements AlterHandler {
}
}
Deadline.checkTimeout();
+ Table table = msdb.getTable(catName, newDbName, newTblName);
+ MTable mTable = msdb.ensureGetMTable(catName, newDbName, newTblName);
for (Entry<Partition, ColumnStatistics> partColStats : columnStatsNeedUpdated.entries()) {
ColumnStatistics newPartColStats = partColStats.getValue();
newPartColStats.getStatsDesc().setDbName(newDbName);
newPartColStats.getStatsDesc().setTableName(newTblName);
- msdb.updatePartitionColumnStatistics(newPartColStats, partColStats.getKey().getValues(),
- writeIdList, newt.getWriteId());
+ msdb.updatePartitionColumnStatistics(table, mTable, newPartColStats,
+ partColStats.getKey().getValues(), writeIdList, newt.getWriteId());
}
} else {
msdb.alterTable(catName, dbname, name, newt, writeIdList);
@@ -554,7 +557,7 @@ public class HiveAlterHandler implements AlterHandler {
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
final String name, final List<String> part_vals, final Partition new_part,
EnvironmentContext environmentContext)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
+ throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException, NoSuchObjectException {
return alterPartition(msdb, wh, MetaStoreUtils.getDefaultCatalog(conf), dbname, name, part_vals, new_part,
environmentContext, null, null);
}
@@ -563,7 +566,7 @@ public class HiveAlterHandler implements AlterHandler {
public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, String dbname,
String name, List<String> part_vals, final Partition new_part,
EnvironmentContext environmentContext, IHMSHandler handler, String validWriteIds)
- throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException {
+ throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException, NoSuchObjectException {
boolean success = false;
Partition oldPart;
List<TransactionalMetaStoreEventListener> transactionalListeners = null;
@@ -648,6 +651,7 @@ public class HiveAlterHandler implements AlterHandler {
throw new InvalidObjectException(
"Unable to alter partition because table or database does not exist.");
}
+ MTable mTable = msdb.ensureGetMTable(catName, dbname, name);
try {
oldPart = msdb.getPartition(catName, dbname, name, part_vals);
} catch (NoSuchObjectException e) {
@@ -766,7 +770,7 @@ public class HiveAlterHandler implements AlterHandler {
for (ColumnStatistics cs : multiColumnStats) {
cs.getStatsDesc().setPartName(newPartName);
try {
- msdb.updatePartitionColumnStatistics(cs, new_part.getValues(),
+ msdb.updatePartitionColumnStatistics(tbl, mTable, cs, new_part.getValues(),
validWriteIds, new_part.getWriteId());
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 1fbe24eec37..d6f6e165ace 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2064,7 +2064,7 @@ public class ObjectStore implements RawStore, Configurable {
Query query = null;
try {
openTransaction();
- catName = normalizeIdentifier(catName);
+ catName = normalizeIdentifier(Optional.ofNullable(catName).orElse(getDefaultCatalog(conf)));
db = normalizeIdentifier(db);
table = normalizeIdentifier(table);
query = pm.newQuery(MTable.class,
@@ -4874,7 +4874,8 @@ public class ObjectStore implements RawStore, Configurable {
* @param tblName Table name.
* @return Table object.
*/
- private MTable ensureGetMTable(String catName, String dbName, String tblName)
+ @Override
+ public MTable ensureGetMTable(String catName, String dbName, String tblName)
throws NoSuchObjectException {
MTable mtable = getMTable(catName, dbName, tblName);
if (mtable == null) {
@@ -9924,7 +9925,7 @@ public class ObjectStore implements RawStore, Configurable {
}
@Override
- public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats,
+ public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics colStats,
List<String> partVals, String validWriteIds, long writeId)
throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
boolean committed = false;
@@ -9934,8 +9935,6 @@ public class ObjectStore implements RawStore, Configurable {
List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
String catName = statsDesc.isSetCatName() ? statsDesc.getCatName() : getDefaultCatalog(conf);
- MTable mTable = ensureGetMTable(catName, statsDesc.getDbName(), statsDesc.getTableName());
- Table table = convertToTable(mTable);
Partition partition = convertToPart(getMPartition(
catName, statsDesc.getDbName(), statsDesc.getTableName(), partVals, mTable), false);
List<String> colNames = new ArrayList<>();
@@ -9994,6 +9993,16 @@ public class ObjectStore implements RawStore, Configurable {
}
}
+ @Override
+ public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats,
+ List<String> partVals, String validWriteIds, long writeId)
+ throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException {
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ Table table = getTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName());
+ MTable mTable = ensureGetMTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName());
+ return updatePartitionColumnStatistics(table, mTable, colStats, partVals, validWriteIds, writeId);
+ }
+
@Override
public Map<String, Map<String, String>> updatePartitionColumnStatisticsInBatch(
Map<String, ColumnStatistics> partColStatsMap,
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index b0f2df9c998..69073707f25 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -113,6 +113,7 @@ import org.apache.hadoop.hive.metastore.api.WMResourcePlan;
import org.apache.hadoop.hive.metastore.api.WMTrigger;
import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo;
import org.apache.thrift.TException;
@@ -1151,8 +1152,13 @@ public interface RawStore extends Configurable {
* @throws InvalidInputException unable to record the stats for the table
*/
Map<String, String> updatePartitionColumnStatistics(ColumnStatistics statsObj,
- List<String> partVals, String validWriteIds, long writeId)
- throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException;
+ List<String> partVals, String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException;
+
+ Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable,
+ ColumnStatistics statsObj, List<String> partVals,
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException;
/**
* Returns the relevant column statistics for a given column in a given table in a given database
@@ -2201,4 +2207,5 @@ public interface RawStore extends Configurable {
Package findPackage(GetPackageRequest request);
List<String> listPackages(ListPackageRequest request);
void dropPackage(DropPackageRequest request);
+ public MTable ensureGetMTable(String catName, String dbName, String tblName) throws NoSuchObjectException;
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 1d475605b3b..38deb9f8a5c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.Stack;
import java.util.concurrent.Executors;
@@ -64,6 +65,7 @@ import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFa
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.messaging.*;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
@@ -1276,7 +1278,7 @@ public class CachedStore implements RawStore, Configurable {
@Override public Table getTable(String catName, String dbName, String tblName, String validWriteIds, long tableId)
throws MetaException {
- catName = normalizeIdentifier(catName);
+ catName = normalizeIdentifier(Optional.ofNullable(catName).orElse(getDefaultCatalog(conf)));
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) {
@@ -2226,10 +2228,26 @@ public class CachedStore implements RawStore, Configurable {
}
@Override public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats, List<String> partVals,
- String validWriteIds, long writeId)
- throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
- Map<String, String> newParams =
- rawStore.updatePartitionColumnStatistics(colStats, partVals, validWriteIds, writeId);
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ return updatePartitionColumnStatisticsInternal(null, null, colStats, partVals, validWriteIds, writeId);
+ }
+
+ @Override public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable,
+ ColumnStatistics colStats, List<String> partVals,
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ return updatePartitionColumnStatisticsInternal(table, mTable, colStats, partVals, validWriteIds, writeId);
+ }
+
+ private Map<String, String> updatePartitionColumnStatisticsInternal(Table table, MTable mTable,
+ ColumnStatistics colStats, List<String> partVals,
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
+ table = Optional.ofNullable(table).orElse(getTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()));
+ mTable = Optional.ofNullable(mTable).orElse(ensureGetMTable(statsDesc.getCatName(), statsDesc.getDbName(), statsDesc.getTableName()));
+ Map<String, String> newParams = rawStore.updatePartitionColumnStatistics(table, mTable, colStats, partVals, validWriteIds, writeId);
// in case of event based cache update, cache is updated during commit txn
if (newParams != null && !canUseEvents) {
updatePartitionColumnStatisticsInCache(colStats, newParams, partVals);
@@ -3340,6 +3358,11 @@ public class CachedStore implements RawStore, Configurable {
rawStore.dropPackage(request);
}
+ @Override
+ public MTable ensureGetMTable(String catName, String dbName, String tblName) throws NoSuchObjectException {
+ return rawStore.ensureGetMTable(catName, dbName, tblName);
+ }
+
private boolean shouldGetConstraintFromRawStore(String catName, String dbName, String tblName) {
return !shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())
|| !sharedCache.isTableConstraintValid(catName, dbName, tblName);
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
index bc19b961e91..46a638fe468 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java
@@ -113,6 +113,7 @@ import org.apache.hadoop.hive.metastore.api.WMTrigger;
import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse;
import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo;
import org.apache.thrift.TException;
@@ -827,6 +828,14 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
return objectStore.updateTableColumnStatistics(statsObj, validWriteIds, writeId);
}
+ @Override
+ public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable, ColumnStatistics statsObj,
+ List<String> partVals, String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException,
+ InvalidInputException {
+ return objectStore.updatePartitionColumnStatistics(table, mTable, statsObj, partVals, validWriteIds, writeId);
+ }
+
@Override
public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics statsObj,
List<String> partVals, String validWriteIds, long writeId)
@@ -1532,6 +1541,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable {
objectStore.dropPackage(request);
}
+ @Override
+ public MTable ensureGetMTable(String catName, String dbName, String tblName) throws NoSuchObjectException {
+ return objectStore.ensureGetMTable(catName, dbName, catName);
+ }
+
@Override
public Map<String, Map<String, String>> updatePartitionColumnStatisticsInBatch(
Map<String, ColumnStatistics> partColStatsMap,
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
index 52ae4b8b12c..db443e12811 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java
@@ -128,6 +128,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.thrift.TException;
@@ -898,12 +899,18 @@ public class DummyRawStoreForJdoConnection implements RawStore {
}
@Override
- public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics statsObj,List<String> partVals,
- String validWriteIds, long writeId)
+ public Map<String, String> updatePartitionColumnStatistics(Table table, MTable mTable,
+ ColumnStatistics statsObj, List<String> partVals, String validWriteIds, long writeId)
throws NoSuchObjectException, MetaException, InvalidObjectException {
return null;
}
+ @Override public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics statsObj, List<String> partVals,
+ String validWriteIds, long writeId)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException {
+ return null;
+ }
+
@Override
public void verifySchema() throws MetaException {
}
@@ -1532,6 +1539,11 @@ public class DummyRawStoreForJdoConnection implements RawStore {
public void dropPackage(DropPackageRequest request) {
}
+ @Override
+ public MTable ensureGetMTable(String catName, String dbName, String tblName) throws NoSuchObjectException {
+ return null;
+ }
+
@Override
public Map<String, Map<String, String>> updatePartitionColumnStatisticsInBatch(
Map<String, ColumnStatistics> partColStatsMap,
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
index bc9e04a227b..1a5d8715174 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.hive.metastore.metrics.Metrics;
import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.metastore.model.MNotificationLog;
import org.apache.hadoop.hive.metastore.model.MNotificationNextId;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.junit.Assert;
@@ -819,6 +820,7 @@ public class TestObjectStore {
try (AutoCloseable c = deadline()) {
objectStore.createTable(tbl1);
}
+ MTable mTable1 = objectStore.ensureGetMTable(tbl1.getCatName(), tbl1.getDbName(), tbl1.getTableName());
PrivilegeBag privilegeBag = new PrivilegeBag();
// Create partitions for the partitioned table
for(int i=0; i < 3; i++) {
@@ -877,7 +879,7 @@ public class TestObjectStore {
statsObjList.add(partStats);
try (AutoCloseable c = deadline()) {
- objectStore.updatePartitionColumnStatistics(stats, part.getValues(), null, -1);
+ objectStore.updatePartitionColumnStatistics(tbl1, mTable1, stats, part.getValues(), null, -1);
}
}
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java
index 14110c6dbe1..6928aa55a96 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.junit.After;
import org.junit.Assert;
@@ -148,6 +149,7 @@ public class TestOldSchema {
Table table = new Table(tableName, dbName, "me", (int) now, (int) now, 0, sd, partCols,
Collections.emptyMap(), null, null, null);
store.createTable(table);
+ MTable mTable = store.ensureGetMTable(table.getCatName(), table.getDbName(), table.getTableName());
Deadline.startTimer("getPartition");
for (int i = 0; i < 10; i++) {
@@ -178,7 +180,7 @@ public class TestOldSchema {
obj.setStatsData(data);
cs.addToStatsObj(obj);
cs.setEngine(ENGINE);
- store.updatePartitionColumnStatistics(cs, partVal, null, -1);
+ store.updatePartitionColumnStatistics(table, mTable, cs, partVal, null, -1);
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
index fbd5c25f937..4b5e1813f0b 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
import org.apache.hadoop.hive.metastore.columnstats.ColStatsBuilder;
import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.model.MTable;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.junit.After;
import org.junit.Assert;
@@ -345,6 +346,7 @@ import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
Table salesTable =
createTable(tpcdsdb.getName(), "store_sales", createStorageDescriptor(columns), partitionsColumns);
objectStore.createTable(salesTable);
+ MTable salesMTable = objectStore.ensureGetMTable(salesTable.getCatName(), salesTable.getDbName(), salesTable.getTableName());
Map<String, ColumnStatisticsData> partitionStats = new HashMap<>();
ColumnStatisticsData data1 = new ColStatsBuilder<>(long.class).numNulls(100).numDVs(50)
@@ -363,7 +365,7 @@ import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
partNames.add(pName);
ColumnStatistics stats = createColumnStatistics(pStat.getValue(), salesTable, soldDateCol, pName);
- objectStore.updatePartitionColumnStatistics(stats, partitionValue, null, -1);
+ objectStore.updatePartitionColumnStatistics(salesTable, salesMTable, stats, partitionValue, null, -1);
}
List<ColumnStatistics> rawStats = objectStore