You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/10/22 21:07:07 UTC
hive git commit: HIVE-20765 : fetch partitions for txn stats
validation in get_aggr_stats with one call (Sergey Shelukhin,
reviewed by Ashutosh Chauhan, and Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master b4302bb7a -> cbe3228c2
HIVE-20765 : fetch partitions for txn stats validation in get_aggr_stats with one call (Sergey Shelukhin, reviewed by Ashutosh Chauhan, and Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cbe3228c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cbe3228c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cbe3228c
Branch: refs/heads/master
Commit: cbe3228c29e2db6ae5cffcb94cdcf79063859ffe
Parents: b4302bb
Author: sergey <se...@apache.org>
Authored: Mon Oct 22 14:06:48 2018 -0700
Committer: sergey <se...@apache.org>
Committed: Mon Oct 22 14:06:48 2018 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/ObjectStore.java | 22 +++++++++++++-------
.../metastore/PartitionProjectionEvaluator.java | 8 +++++--
2 files changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index b98b4b4..ddd64e7 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -9052,21 +9052,27 @@ public class ObjectStore implements RawStore, Configurable {
return null;
}
- MTable table = getMTable(catName, dbName, tblName);
+ Table table = getTable(catName, dbName, tblName);
boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters());
if (isTxn && !areTxnStatsSupported) {
return null;
}
+ GetPartitionsFilterSpec fs = new GetPartitionsFilterSpec();
+ fs.setFilterMode(PartitionFilterMode.BY_NAMES);
+ fs.setFilters(partNames);
+ GetPartitionsProjectionSpec ps = new GetPartitionsProjectionSpec();
+ ps.setIncludeParamKeyPattern(StatsSetupConst.COLUMN_STATS_ACCURATE + '%');
+ ps.setFieldList(Lists.newArrayList("writeId", "parameters", "values"));
+ List<Partition> parts = getPartitionSpecsByFilterAndProjection(table, ps, fs);
// Loop through the given "partNames" list
// checking isolation-level-compliance of each partition column stats.
- for (String partName : partNames) {
- MPartition mpart = getMPartition(
- catName, dbName, tblName, Warehouse.getPartValuesFromPartName(partName));
- if (!isCurrentStatsValidForTheQuery(mpart, writeIdList, false)) {
- LOG.debug("The current metastore transactional partition column statistics " +
- "for " + dbName + "." + tblName + "." + mpart.getPartitionName() + " is not valid " +
- "for the current query.");
+ for (Partition part : parts) {
+
+ if (!isCurrentStatsValidForTheQuery(part, part.getWriteId(), writeIdList, false)) {
+ String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+ LOG.debug("The current metastore transactional partition column statistics for " + dbName
+ + "." + tblName + "." + partName + " is not valid for the current query");
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/cbe3228c/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
index e918a33..c3a0093 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionProjectionEvaluator.java
@@ -26,6 +26,7 @@ import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableBiMap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -38,6 +39,7 @@ import org.slf4j.LoggerFactory;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -369,8 +371,8 @@ public class PartitionProjectionEvaluator {
int numColumns = buildQueryForSingleValuedFields(partitionIds, queryTextBuilder);
String queryText = queryTextBuilder.toString();
- try (Query query = pm.newQuery("javax.jdo.query.SQL", queryText)) {
-
+ Query<?> query = pm.newQuery("javax.jdo.query.SQL", queryText);
+ try {
long start = LOG.isDebugEnabled() ? System.nanoTime() : 0;
List<Object> sqlResult = MetastoreDirectSqlUtils.executeWithArray(query, null, queryText);
long queryTime = LOG.isDebugEnabled() ? System.nanoTime() : 0;
@@ -456,6 +458,8 @@ public class PartitionProjectionEvaluator {
} catch (Exception e) {
LOG.error("Exception received while getting partitions using projected fields", e);
throw new MetaException(e.getMessage());
+ } finally {
+ query.closeAll();
}
}