You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2020/08/10 15:19:20 UTC
[hive] branch master updated: HIVE-23901: Overhead of Logger in
ColumnStatsMerger damage the performance
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 7293956 HIVE-23901: Overhead of Logger in ColumnStatsMerger damage the performance
7293956 is described below
commit 7293956fc2f13ccbe72825b67ce1b53dce536359
Author: YU-WEN LAI <hs...@gmail.com>
AuthorDate: Wed Jul 22 18:14:50 2020 -0700
HIVE-23901: Overhead of Logger in ColumnStatsMerger damage the performance
---
.../metastore/columnstats/merge/BinaryColumnStatsMerger.java | 8 +++++++-
.../metastore/columnstats/merge/BooleanColumnStatsMerger.java | 8 +++++++-
.../hive/metastore/columnstats/merge/ColumnStatsMerger.java | 11 +----------
.../metastore/columnstats/merge/DateColumnStatsMerger.java | 10 ++++++++--
.../metastore/columnstats/merge/DecimalColumnStatsMerger.java | 10 ++++++++--
.../metastore/columnstats/merge/DoubleColumnStatsMerger.java | 10 ++++++++--
.../metastore/columnstats/merge/LongColumnStatsMerger.java | 11 +++++++++--
.../metastore/columnstats/merge/StringColumnStatsMerger.java | 11 +++++++++--
.../columnstats/merge/TimestampColumnStatsMerger.java | 11 +++++++++--
9 files changed, 66 insertions(+), 24 deletions(-)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
index 52377b4..1756db8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java
@@ -21,11 +21,17 @@ package org.apache.hadoop.hive.metastore.columnstats.merge;
import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BinaryColumnStatsMerger extends ColumnStatsMerger {
+ private static final Logger LOG = LoggerFactory.getLogger(BinaryColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats();
BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats();
aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
index 0685334..e33573c 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java
@@ -21,11 +21,17 @@ package org.apache.hadoop.hive.metastore.columnstats.merge;
import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BooleanColumnStatsMerger extends ColumnStatsMerger {
+ private static final Logger LOG = LoggerFactory.getLogger(BooleanColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats();
BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats();
aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues());
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
index 8204770..3bbdee9 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java
@@ -20,16 +20,7 @@
package org.apache.hadoop.hive.metastore.columnstats.merge;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public abstract class ColumnStatsMerger {
- protected final Logger log = LoggerFactory.getLogger(getClass());
-
- public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
- log.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
- doMerge(aggregateColStats, newColStats);
- }
-
- protected abstract void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats);
+ public abstract void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats);
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
index 84962aa..714249f 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java
@@ -28,11 +28,17 @@ import org.apache.hadoop.hive.metastore.api.Date;
import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector;
import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DateColumnStatsMerger extends ColumnStatsMerger {
+ private static final Logger LOG = LoggerFactory.getLogger(DateColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
DateColumnStatsDataInspector aggregateData = dateInspectorFromStats(aggregateColStats);
DateColumnStatsDataInspector newData = dateInspectorFromStats(newColStats);
@@ -53,7 +59,7 @@ public class DateColumnStatsMerger extends ColumnStatsMerger {
} else {
ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
}
- log.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
+ LOG.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
aggregateData.getNumDVs(), newData.getNumDVs(), ndv);
aggregateData.setNumDVs(ndv);
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
index eaa67c6..3c0d455 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java
@@ -29,11 +29,17 @@ import com.google.common.base.MoreObjects;
import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.decimalInspectorFromStats;
import org.apache.commons.lang3.ObjectUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DecimalColumnStatsMerger extends ColumnStatsMerger {
+ private static final Logger LOG = LoggerFactory.getLogger(DecimalColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
DecimalColumnStatsDataInspector aggregateData = decimalInspectorFromStats(aggregateColStats);
DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(newColStats);
@@ -55,7 +61,7 @@ public class DecimalColumnStatsMerger extends ColumnStatsMerger {
} else {
ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
}
- log.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
+ LOG.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
aggregateData.getNumDVs(), newData.getNumDVs(), ndv);
aggregateData.setNumDVs(ndv);
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
index 6f41232..284e1c3 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java
@@ -22,13 +22,19 @@ package org.apache.hadoop.hive.metastore.columnstats.merge;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.doubleInspectorFromStats;
public class DoubleColumnStatsMerger extends ColumnStatsMerger {
+ private static final Logger LOG = LoggerFactory.getLogger(DoubleColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
DoubleColumnStatsDataInspector aggregateData = doubleInspectorFromStats(aggregateColStats);
DoubleColumnStatsDataInspector newData = doubleInspectorFromStats(newColStats);
setLowValue(aggregateData, newData);
@@ -47,7 +53,7 @@ public class DoubleColumnStatsMerger extends ColumnStatsMerger {
} else {
ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
}
- log.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+ LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of "
+ aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv);
aggregateData.setNumDVs(ndv);
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
index 5a56c59..17613cf 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java
@@ -22,12 +22,19 @@ package org.apache.hadoop.hive.metastore.columnstats.merge;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.longInspectorFromStats;
public class LongColumnStatsMerger extends ColumnStatsMerger {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
LongColumnStatsDataInspector aggregateData = longInspectorFromStats(aggregateColStats);
LongColumnStatsDataInspector newData = longInspectorFromStats(newColStats);
setLowValue(aggregateData, newData);
@@ -46,7 +53,7 @@ public class LongColumnStatsMerger extends ColumnStatsMerger {
} else {
ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
}
- log.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
+ LOG.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
aggregateData.getNumDVs(), newData.getNumDVs(), ndv);
aggregateData.setNumDVs(ndv);
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
index 1ef9e1c..7bd5b72 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java
@@ -22,12 +22,19 @@ package org.apache.hadoop.hive.metastore.columnstats.merge;
import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.stringInspectorFromStats;
public class StringColumnStatsMerger extends ColumnStatsMerger {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StringColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
StringColumnStatsDataInspector aggregateData = stringInspectorFromStats(aggregateColStats);
StringColumnStatsDataInspector newData = stringInspectorFromStats(newColStats);
aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen()));
@@ -46,7 +53,7 @@ public class StringColumnStatsMerger extends ColumnStatsMerger {
} else {
ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
}
- log.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
+ LOG.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
aggregateData.getNumDVs(), newData.getNumDVs(), ndv);
aggregateData.setNumDVs(ndv);
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/TimestampColumnStatsMerger.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/TimestampColumnStatsMerger.java
index 748f738..7dea431 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/TimestampColumnStatsMerger.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/TimestampColumnStatsMerger.java
@@ -27,10 +27,17 @@ import org.apache.hadoop.hive.metastore.api.Timestamp;
import org.apache.hadoop.hive.metastore.columnstats.cache.TimestampColumnStatsDataInspector;
import com.google.common.base.MoreObjects;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TimestampColumnStatsMerger extends ColumnStatsMerger {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TimestampColumnStatsMerger.class);
+
@Override
- protected void doMerge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) {
+ LOG.debug("Merging statistics: [aggregateColStats:{}, newColStats: {}]", aggregateColStats, newColStats);
+
TimestampColumnStatsDataInspector aggregateData = timestampInspectorFromStats(aggregateColStats);
TimestampColumnStatsDataInspector newData = timestampInspectorFromStats(newColStats);
@@ -51,7 +58,7 @@ public class TimestampColumnStatsMerger extends ColumnStatsMerger {
} else {
ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs());
}
- log.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
+ LOG.debug("Use bitvector to merge column {}'s ndvs of {} and {} to be {}", aggregateColStats.getColName(),
aggregateData.getNumDVs(), newData.getNumDVs(), ndv);
aggregateData.setNumDVs(ndv);
}