You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2018/01/16 11:13:30 UTC
[1/3] hive git commit: HIVE-18314: qtests: semijoin_hint.q breaks
hybridgrace_hashjoin_2.q (Laszlo Bodor via Zoltan Haindrich)
Repository: hive
Updated Branches:
refs/heads/master 6938fcabf -> 13c1bf9e9
HIVE-18314: qtests: semijoin_hint.q breaks hybridgrace_hashjoin_2.q (Laszlo Bodor via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/19c2b722
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/19c2b722
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/19c2b722
Branch: refs/heads/master
Commit: 19c2b7225e1c072cba5c1a93329f70059e7e063d
Parents: 6938fca
Author: Laszlo Bodor <bo...@gmail.com>
Authored: Tue Jan 16 11:19:47 2018 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jan 16 11:19:47 2018 +0100
----------------------------------------------------------------------
ql/src/test/queries/clientpositive/semijoin_hint.q | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/19c2b722/ql/src/test/queries/clientpositive/semijoin_hint.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/semijoin_hint.q b/ql/src/test/queries/clientpositive/semijoin_hint.q
index 2a15344..0586dbc 100644
--- a/ql/src/test/queries/clientpositive/semijoin_hint.q
+++ b/ql/src/test/queries/clientpositive/semijoin_hint.q
@@ -13,6 +13,7 @@ set hive.stats.autogather=true;
set hive.tez.bigtable.minsize.semijoin.reduction=1;
set hive.tez.min.bloom.filter.entries=1;
set hive.tez.dynamic.semijoin.reduction.threshold=-999999999999;
+set hive.metastore.aggregate.stats.cache.enabled=false;
-- Create Tables
create table alltypesorc_int ( cint int, cstring string ) stored as ORC;
@@ -96,4 +97,3 @@ explain select /*+ semi(k, str, v, 5000)*/ count(*) from srcpart_date k join src
-- This should NOT create a semijoin
explain select /*+ semi(k, str, v, 5000)*/ count(*) from srcpart_date k join srcpart_small v on (k.value = v.key1);
-
[2/3] hive git commit: HIVE-18309: qtests: smb_mapjoin_19.q breaks
bucketsortoptimize_insert_2.q (Laszlo Bodor via Zoltan Haindrich)
Posted by kg...@apache.org.
HIVE-18309: qtests: smb_mapjoin_19.q breaks bucketsortoptimize_insert_2.q (Laszlo Bodor via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5d4559bb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5d4559bb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5d4559bb
Branch: refs/heads/master
Commit: 5d4559bbdb5fb36450b67841d0da4fa2fecdeeea
Parents: 19c2b72
Author: Laszlo Bodor <bo...@gmail.com>
Authored: Tue Jan 16 11:40:42 2018 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jan 16 11:40:42 2018 +0100
----------------------------------------------------------------------
ql/src/test/queries/clientpositive/smb_mapjoin_19.q | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5d4559bb/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_19.q b/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
index 4695e5a..01972c9 100644
--- a/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
+++ b/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
@@ -4,7 +4,9 @@ set hive.optimize.bucketmapjoin.sortedmerge = true;
set hive.exec.reducers.max = 1;
set hive.merge.mapfiles=false;
-set hive.merge.mapredfiles=false;
+set hive.merge.mapredfiles=false;
+
+set hive.metastore.aggregate.stats.cache.enabled=false;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1 (key INT, value STRING) PARTITIONED BY (ds STRING)
[3/3] hive git commit: HIVE-18161: Remove hive.stats.atomic (Bertalan
Kondrat via Zoltan Haindrich)
Posted by kg...@apache.org.
HIVE-18161: Remove hive.stats.atomic (Bertalan Kondrat via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/13c1bf9e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/13c1bf9e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/13c1bf9e
Branch: refs/heads/master
Commit: 13c1bf9e903c502dbfe3f2fb10959ac1a64bc0ad
Parents: 5d4559b
Author: Bertalan Kondrat <kb...@gmail.com>
Authored: Tue Jan 16 11:49:55 2018 +0100
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Tue Jan 16 11:49:55 2018 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 --
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 --
.../hadoop/hive/ql/stats/BasicStatsTask.java | 30 ++------------------
3 files changed, 3 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 631c836..854bbdf 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1770,8 +1770,6 @@ public class HiveConf extends Configuration {
"The Java class (implementing the StatsPublisher interface) that is used by default if hive.stats.dbclass is custom type."),
HIVE_STATS_DEFAULT_AGGREGATOR("hive.stats.default.aggregator", "",
"The Java class (implementing the StatsAggregator interface) that is used by default if hive.stats.dbclass is custom type."),
- HIVE_STATS_ATOMIC("hive.stats.atomic", false,
- "whether to update metastore stats only if all stats are available"),
CLIENT_STATS_COUNTERS("hive.client.stats.counters", "",
"Subset of counters that should be of interest for hive.client.stats.publishers (when one wants to limit their publishing). \n" +
"Non-display names should be used"),
http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 476c261..a7183ce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -538,9 +538,6 @@ public enum ErrorMsg {
STATSAGGREGATOR_SOURCETASK_NULL(30014, "SourceTask of StatsTask should not be null"),
STATSAGGREGATOR_CONNECTION_ERROR(30015,
"Stats aggregator of type {0} cannot be connected to", true),
- STATSAGGREGATOR_MISSED_SOMESTATS(30016,
- "Stats type {0} is missing from stats aggregator. If you don't want the query " +
- "to fail because of this, set hive.stats.atomic=false", true),
STATS_SKIPPING_BY_ERROR(30017, "Skipping stats aggregation by error {0}", true),
INVALID_FILE_FORMAT_IN_LOAD(30019, "The file that you are trying to load does not match the" +
http://git-wip-us.apache.org/repos/asf/hive/blob/13c1bf9e/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
index 21e6984..4e85ae1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java
@@ -113,13 +113,11 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
private Partish partish;
private FileStatus[] partfileStatus;
private BasicStatsWork work;
- private boolean atomic;
private boolean followedColStats1;
public BasicStatsProcessor(Partish partish, BasicStatsWork work, HiveConf conf, boolean followedColStats2) {
this.partish = partish;
this.work = work;
- atomic = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_ATOMIC);
followedColStats1 = followedColStats2;
}
@@ -141,12 +139,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
if (!work.isExplicitAnalyze() && !followedColStats1) {
StatsSetupConst.clearColumnStatsState(parameters);
}
- // non-partitioned tables:
- // XXX: I don't aggree with this logic
- // FIXME: deprecate atomic? what's its purpose?
- if (!existStats(parameters) && atomic) {
- return null;
- }
+
if(partfileStatus == null){
LOG.warn("Partition/partfiles is null for: " + partish.getPartition().getSpec());
return null;
@@ -164,7 +157,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
if (StatsSetupConst.areBasicStatsUptoDate(parameters)) {
if (statsAggregator != null) {
String prefix = getAggregationPrefix(p.getTable(), p.getPartition());
- updateStats(statsAggregator, parameters, prefix, atomic);
+ updateStats(statsAggregator, parameters, prefix);
}
}
@@ -172,22 +165,9 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
}
public void collectFileStatus(Warehouse wh) throws MetaException {
- Map<String, String> parameters = partish.getPartParameters();
- if (!existStats(parameters) && atomic) {
- return;
- }
partfileStatus = wh.getFileStatusesForSD(partish.getPartSd());
}
- @Deprecated
- private boolean existStats(Map<String, String> parameters) {
- return parameters.containsKey(StatsSetupConst.ROW_COUNT)
- || parameters.containsKey(StatsSetupConst.NUM_FILES)
- || parameters.containsKey(StatsSetupConst.TOTAL_SIZE)
- || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE)
- || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS);
- }
-
private void updateQuickStats(Map<String, String> parameters, FileStatus[] partfileStatus) throws MetaException {
MetaStoreUtils.populateQuickStats(partfileStatus, parameters);
}
@@ -211,7 +191,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
return prefix;
}
- private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String aggKey, boolean atomic) throws HiveException {
+ private void updateStats(StatsAggregator statsAggregator, Map<String, String> parameters, String aggKey) throws HiveException {
for (String statType : StatsSetupConst.statsRequireCompute) {
String value = statsAggregator.aggregateStats(aggKey, statType);
@@ -225,10 +205,6 @@ public class BasicStatsTask implements Serializable, IStatsProcessor {
}
}
parameters.put(statType, String.valueOf(longValue));
- } else {
- if (atomic) {
- throw new HiveException(ErrorMsg.STATSAGGREGATOR_MISSED_SOMESTATS, statType);
- }
}
}
}