You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2016/11/30 06:01:50 UTC
[1/2] hive git commit: HIVE-15240: Updating/Altering stats in
metastore can be expensive in S3 (Rajesh Balamohan reviewed by Sergey
Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 4f2d1f509 -> 97c3fb396
HIVE-15240: Updating/Altering stats in metastore can be expensive in S3 (Rajesh Balamohan reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b91787db
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b91787db
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b91787db
Branch: refs/heads/master
Commit: b91787dbb71e48d5182de047a87c11f7cde59296
Parents: 4f2d1f5
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Tue Nov 29 21:53:48 2016 -0800
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Tue Nov 29 21:53:48 2016 -0800
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveAlterHandler.java | 14 +++++++--
.../hadoop/hive/metastore/MetaStoreUtils.java | 33 ++++++++++++++------
2 files changed, 36 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b91787db/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index be7ed32..86565a4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -399,7 +399,12 @@ public class HiveAlterHandler implements AlterHandler {
msdb.openTransaction();
oldPart = msdb.getPartition(dbname, name, new_part.getValues());
if (MetaStoreUtils.requireCalStats(hiveConf, oldPart, new_part, tbl, environmentContext)) {
- MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
+ // if stats are same, no need to update
+ if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) {
+ MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters());
+ } else {
+ MetaStoreUtils.updatePartitionStatsFast(new_part, wh, false, true, environmentContext);
+ }
}
updatePartColumnStats(msdb, dbname, name, new_part.getValues(), new_part);
@@ -627,7 +632,12 @@ public class HiveAlterHandler implements AlterHandler {
partValsList.add(tmpPart.getValues());
if (MetaStoreUtils.requireCalStats(hiveConf, oldTmpPart, tmpPart, tbl, environmentContext)) {
- MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext);
+ // Check if stats are same, no need to update
+ if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) {
+ MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters());
+ } else {
+ MetaStoreUtils.updatePartitionStatsFast(tmpPart, wh, false, true, environmentContext);
+ }
}
updatePartColumnStats(msdb, dbname, name, oldTmpPart.getValues(), tmpPart);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b91787db/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 9a4c61a..9ea6ac4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -300,16 +300,24 @@ public class MetaStoreUtils {
}
// requires to calculate stats if new and old have different fast stats
+ return !isFastStatsSame(oldPart, newPart);
+ }
+
+ static boolean isFastStatsSame(Partition oldPart, Partition newPart) {
+ // requires to calculate stats if new and old have different fast stats
if ((oldPart != null) && (oldPart.getParameters() != null)) {
for (String stat : StatsSetupConst.fastStats) {
if (oldPart.getParameters().containsKey(stat)) {
Long oldStat = Long.parseLong(oldPart.getParameters().get(stat));
Long newStat = Long.parseLong(newPart.getParameters().get(stat));
if (!oldStat.equals(newStat)) {
- return true;
+ return false;
}
+ } else {
+ return false;
}
}
+ return true;
}
return false;
}
@@ -367,14 +375,7 @@ public class MetaStoreUtils {
FileStatus[] fileStatus = wh.getFileStatusesForLocation(part.getLocation());
populateQuickStats(fileStatus, params);
LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE));
- if (environmentContext != null
- && environmentContext.isSetProperties()
- && StatsSetupConst.TASK.equals(environmentContext.getProperties().get(
- StatsSetupConst.STATS_GENERATED))) {
- StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE);
- } else {
- StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE);
- }
+ updateBasicState(environmentContext, params);
}
part.setParameters(params);
updated = true;
@@ -382,6 +383,20 @@ public class MetaStoreUtils {
return updated;
}
+ static void updateBasicState(EnvironmentContext environmentContext, Map<String,String> params) {
+ if (params == null) {
+ return;
+ }
+ if (environmentContext != null
+ && environmentContext.isSetProperties()
+ && StatsSetupConst.TASK.equals(environmentContext.getProperties().get(
+ StatsSetupConst.STATS_GENERATED))) {
+ StatsSetupConst.setBasicStatsState(params, StatsSetupConst.TRUE);
+ } else {
+ StatsSetupConst.setBasicStatsState(params, StatsSetupConst.FALSE);
+ }
+ }
+
/**
* getDeserializer
*
[2/2] hive git commit: HIVE-15250: Reuse partitions info generated in
MoveTask to its subscribers (StatsTask) (Rajesh Balamohan reviewed by Sergey
Shelukhin)
Posted by pr...@apache.org.
HIVE-15250: Reuse partitions info generated in MoveTask to its subscribers (StatsTask) (Rajesh Balamohan reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/97c3fb39
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/97c3fb39
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/97c3fb39
Branch: refs/heads/master
Commit: 97c3fb396b9d8e590f8b8fb3c8179be155137085
Parents: b91787d
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Tue Nov 29 21:54:46 2016 -0800
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Tue Nov 29 21:54:46 2016 -0800
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java | 9 +++++----
.../java/org/apache/hadoop/hive/ql/exec/StatsTask.java | 11 ++++-------
2 files changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/97c3fb39/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
index 349f115..e1381be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
@@ -423,10 +423,6 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx);
- // publish DP columns to its subscribers
- if (dps != null && dps.size() > 0) {
- pushFeed(FeedType.DYNAMIC_PARTITIONS, dps);
- }
console.printInfo(System.getProperty("line.separator"));
long startTime = System.currentTimeMillis();
// load the list of DP partitions and return the list of partition specs
@@ -448,6 +444,11 @@ public class MoveTask extends Task<MoveWork> implements Serializable {
SessionState.get().getTxnMgr().getCurrentTxnId(), hasFollowingStatsTask(),
work.getLoadTableWork().getWriteType());
+ // publish DP columns to its subscribers
+ if (dps != null && dps.size() > 0) {
+ pushFeed(FeedType.DYNAMIC_PARTITIONS, dp.values());
+ }
+
String loadTime = "\t Time taken to load dynamic partitions: " +
(System.currentTimeMillis() - startTime)/1000.0 + " seconds";
console.printInfo(loadTime);
http://git-wip-us.apache.org/repos/asf/hive/blob/97c3fb39/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
index aa5d914..c22d69b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -78,7 +79,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
private static transient final Logger LOG = LoggerFactory.getLogger(StatsTask.class);
private Table table;
- private List<LinkedHashMap<String, String>> dpPartSpecs;
+ private Collection<Partition> dpPartSpecs;
public StatsTask() {
super();
@@ -89,8 +90,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
protected void receiveFeed(FeedType feedType, Object feedValue) {
// this method should be called by MoveTask when there are dynamic partitions generated
if (feedType == FeedType.DYNAMIC_PARTITIONS) {
- assert feedValue instanceof List<?>;
- dpPartSpecs = (List<LinkedHashMap<String, String>>) feedValue;
+ dpPartSpecs = (Collection<Partition>) feedValue;
}
}
@@ -496,10 +496,7 @@ public class StatsTask extends Task<StatsWork> implements Serializable {
// If no dynamic partitions are generated, dpPartSpecs may not be initialized
if (dpPartSpecs != null) {
// load the list of DP partitions and return the list of partition specs
- for (LinkedHashMap<String, String> partSpec : dpPartSpecs) {
- Partition partn = db.getPartition(table, partSpec, false);
- list.add(partn);
- }
+ list.addAll(dpPartSpecs);
}
} else { // static partition
Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false);