You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2018/01/18 17:55:53 UTC
[33/70] [abbrv] hive git commit: HIVE-18161: Remove hive.stats.atomic
(Bertalan Kondrat via Zoltan Haindrich)
HIVE-18161: Remove hive.stats.atomic (Bertalan Kondrat via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/13c1bf9e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/13c1bf9e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/13c1bf9e
Branch: refs/heads/standalone-metastore
Commit: 13c1bf9e903c502dbfe3f2fb10959ac1a64bc0ad
Parents: 5d4559b
Author: Bertalan Kondrat <kb...@gmail.com>
Authored: Tue Jan 16 11:49:55 2018 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jan 16 11:49:55 2018 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 --
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 --
.../hadoop/hive/ql/stats/BasicStatsTask.java | 30 ++------------------
3 files changed, 3 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 631c836..854bbdf 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1770,8 +1770,6 @@ public class HiveConf extends Configuration {
"The Java class (implementing the StatsPublisher interface) that is used by default if hive.stats.dbclass is custom type."),
HIVE_STATS_DEFAULT_AGGREGATOR("hive.stats.default.aggregator", "",
"The Java class (implementing the StatsAggregator interface) that is used by default if hive.stats.dbclass is custom type."),
- HIVE_STATS_ATOMIC("hive.stats.atomic", false,
- "whether to update metastore stats only if all stats are available"),
CLIENT_STATS_COUNTERS("hive.client.stats.counters", "",
"Subset of counters that should be of interest for hive.client.stats.publishers (when one wants to limit their publishing). \n" +
"Non-display names should be used"),
http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 476c261..a7183ce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -538,9 +538,6 @@ public enum ErrorMsg {
STATSAGGREGATOR_SOURCETASK_NULL(30014, "SourceTask of StatsTask should not be null"),
STATSAGGREGATOR_CONNECTION_ERROR(30015,
"Stats aggregator of type {0} cannot be connected to", true),
- STATSAGGREGATOR_MISSED_SOMESTATS(30016,
- "Stats type {0} is missing from stats aggregator. If you don't want the query " +
- "to fail because of this, set hive.stats.atomic=false", true),
STATS_SKIPPING_BY_ERROR(30017, "Skipping stats aggregation by error {0}", true),
INVALID_FILE_FORMAT_IN_LOAD(30019, "The file that you are trying to load does not match the" +
http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
index 21e6984..4e85ae1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
@@ -113,13 +113,11 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
private Partish partish;
private FileStatus[] partfileStatus;
private BasicStatsWork work;
- private boolean atomic;
private boolean followedColStats1;
public BasicStatsProcessor(Partish partish, BasicStatsWork work, HiveConf conf, boolean followedColStats2) {
this.partish = partish;
this.work = work;
- atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC);
followedColStats1 = followedColStats2;
}
@@ -141,12 +139,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
if (!work.isExplicitAnalyze() && !followedColStats1) {
StatsSetupConst.clearColumnStatsState(parameters);
}
- // non-partitioned tables:
- // XXX: I don't aggree with this logic
- // FIXME: deprecate atomic? what's its purpose?
- if (!existStats(parameters) && atomic) {
- return null;
- }
+
if(partfileStatus == null){
LOG.warn("Partition/partfiles is null for: " + partish.getPartition().getSpec());
return null;
@@ -164,7 +157,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
if (StatsSetupConst.areBasicStatsUptoDate(parameters)) {
if (statsAggregator != null) {
String prefix = getAggregationPrefix(p.getTable(), p.getPartition());
- updateStats(statsAggregator, parameters, prefix, atomic);
+ updateStats(statsAggregator, parameters, prefix);
}
}
@@ -172,22 +165,9 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
}
public void collectFileStatus(Warehouse wh) throws MetaException {
- Map<String, String> parameters = partish.getPartParameters();
- if (!existStats(parameters) && atomic) {
- return;
- }
partfileStatus = wh.getFileStatusesForSD(partish.getPartSd());
}
- @Deprecated
- private boolean existStats(Map<String, String> parameters) {
- return parameters.containsKey(StatsSetupConst.ROW_COUNT)
- || parameters.containsKey(StatsSetupConst.NUM_FILES)
- || parameters.containsKey(StatsSetupConst.TOTAL_SIZE)
- || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE)
- || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS);
- }
-
private void updateQuickStats(Map<String, String> parameters, FileStatus[] partfileStatus) throws MetaException {
MetaStoreUtils.populateQuickStats(partfileStatus, parameters);
}
@@ -211,7 +191,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
return prefix;
}
- private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String aggKey, boolean atomic) throws HiveException {
+ private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String aggKey) throws HiveException {
for (String statType : StatsSetupConst.statsRequireCompute) {
String value = statsAggregator.aggregateStats(aggKey, statType);
@@ -225,10 +205,6 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
}
}
parameters.put(statType, String.valueOf(longValue));
- } else {
- if (atomic) {
- throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType);
- }
}
}
}