You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ab...@apache.org on 2019/06/18 14:01:27 UTC
[hive] 02/02: HIVE-21842: Code cleanup in
org.apache.hadoop.hive.ql.stats.StatsUtils.collectStatistics (Ivan Suller
via Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit 1f20cc34a8a589cea734f7f0b4e2e1431e65f516
Author: Ivan Suller <is...@cloudera.com>
AuthorDate: Tue Jun 18 15:54:46 2019 +0200
HIVE-21842: Code cleanup in org.apache.hadoop.hive.ql.stats.StatsUtils.collectStatistics (Ivan Suller via Laszlo Bodor)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 81 ++++++++--------------
1 file changed, 28 insertions(+), 53 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index e184b9d..e0f401b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -263,7 +263,7 @@ public class StatsUtils {
// long nr = getNumRows(conf, schema, neededColumns, table, ds);
long ds = basicStats.getDataSize();
long nr = basicStats.getNumRows();
- List<ColStatistics> colStats = Lists.newArrayList();
+ List<ColStatistics> colStats = Collections.emptyList();
long numErasureCodedFiles = getErasureCodedFiles(table);
@@ -303,11 +303,6 @@ public class StatsUtils {
}
BasicStats bbs = BasicStats.buildFrom(partStats);
- List<Long> rowCounts = Lists.newArrayList();
- for (BasicStats basicStats : partStats) {
- rowCounts.add(basicStats.getNumRows());
- }
-
long nr = bbs.getNumRows();
long ds = bbs.getDataSize();
@@ -316,7 +311,7 @@ public class StatsUtils {
long numErasureCodedFiles = getSumIgnoreNegatives(erasureCodedFiles);
if (nr == 0) {
- nr=1;
+ nr = 1;
}
stats = new Statistics(nr, ds, numErasureCodedFiles);
stats.setBasicStatsState(bbs.getState());
@@ -331,51 +326,12 @@ public class StatsUtils {
List<String> partitionCols = getPartitionColumns(schema, neededColumns, referencedColumns);
// We will retrieve stats from the metastore only for columns that are not cached
- List<String> neededColsToRetrieve;
- List<String> partitionColsToRetrieve;
List<ColStatistics> columnStats = new ArrayList<>();
- if (colStatsCache != null) {
- neededColsToRetrieve = new ArrayList<String>(neededColumns.size());
- for (String colName : neededColumns) {
- ColStatistics colStats = colStatsCache.getColStats().get(colName);
- if (colStats == null) {
- neededColsToRetrieve.add(colName);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stats for column " + colName +
- " in table " + table.getCompleteName() + " could not be retrieved from cache");
- }
- } else {
- columnStats.add(colStats);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stats for column " + colName +
- " in table " + table.getCompleteName() + " retrieved from cache");
- }
- }
- }
- partitionColsToRetrieve = new ArrayList<>(partitionCols.size());
- for (String colName : partitionCols) {
- ColStatistics colStats = colStatsCache.getColStats().get(colName);
- if (colStats == null) {
- partitionColsToRetrieve.add(colName);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stats for column " + colName +
- " in table " + table.getCompleteName() + " could not be retrieved from cache");
- }
- } else {
- columnStats.add(colStats);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stats for column " + colName +
- " in table " + table.getCompleteName() + " retrieved from cache");
- }
- }
- }
- } else {
- neededColsToRetrieve = neededColumns;
- partitionColsToRetrieve = partitionCols;
- }
+ List<String> neededColsToRetrieve = extractColumnStates(table, neededColumns, colStatsCache, columnStats);
+ List<String> partitionColsToRetrieve = extractColumnStates(table, partitionCols, colStatsCache, columnStats);
// List of partitions
- List<String> partNames = new ArrayList<String>(partList.getNotDeniedPartns().size());
+ List<String> partNames = new ArrayList<>(partList.getNotDeniedPartns().size());
for (Partition part : partList.getNotDeniedPartns()) {
partNames.add(part.getName());
}
@@ -384,15 +340,14 @@ public class StatsUtils {
// We check the sizes of neededColumns and partNames here. If either
// size is 0, aggrStats is null after several retries. Thus, we can
// skip the step to connect to the metastore.
- if (fetchColStats && neededColsToRetrieve.size() > 0 && partNames.size() > 0) {
+ if (fetchColStats && !neededColsToRetrieve.isEmpty() && !partNames.isEmpty()) {
aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(),
neededColsToRetrieve, partNames, false);
}
boolean statsRetrieved = aggrStats != null &&
aggrStats.getColStats() != null && aggrStats.getColStatsSize() != 0;
- if (neededColumns.size() == 0 ||
- (neededColsToRetrieve.size() > 0 && !statsRetrieved)) {
+ if (neededColumns.isEmpty() || (!neededColsToRetrieve.isEmpty() && !statsRetrieved)) {
estimateStatsForMissingCols(neededColsToRetrieve, columnStats, table, conf, nr, schema);
// There are some partitions with no state (or we didn't fetch any state).
// Update the stats with empty list to reflect that in the
@@ -439,7 +394,7 @@ public class StatsUtils {
}
}
- if(rowCounts.size() == 0 ) {
+ if (partStats.isEmpty()) {
// all partitions are filtered by partition pruning
stats.setBasicStatsState(State.COMPLETE);
}
@@ -459,6 +414,26 @@ public class StatsUtils {
return stats;
}
+ private static List<String> extractColumnStates(Table table, List<String> columns,
+ ColumnStatsList colStatsCache, List<ColStatistics> columnStats) {
+ if (colStatsCache == null) {
+ return columns;
+ }
+ List<String> neededColsToRetrieve = new ArrayList<>(columns.size());
+ for (String colName : columns) {
+ ColStatistics colStats = colStatsCache.getColStats().get(colName);
+ if (colStats == null) {
+ neededColsToRetrieve.add(colName);
+ LOG.debug("Stats for column {} in table {} could not be retrieved from cache", colName,
+ table.getCompleteName());
+ } else {
+ columnStats.add(colStats);
+ LOG.debug("Stats for column {} in table {} retrieved from cache", colName, table.getCompleteName());
+ }
+ }
+ return neededColsToRetrieve;
+ }
+
/**
* Based on the provided column statistics and number of rows, this method infers if the column