You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2017/10/11 21:34:29 UTC
hive git commit: HIVE-17669: Cache to optimize SearchArgument
deserialization (Mithun Radhakrishnan reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master 63bda8c70 -> f1050a6e8
HIVE-17669: Cache to optimize SearchArgument deserialization (Mithun Radhakrishnan reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f1050a6e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f1050a6e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f1050a6e
Branch: refs/heads/master
Commit: f1050a6e8462864a3d8c9696943cf5b4d12d9515
Parents: 63bda8c
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Wed Oct 11 14:29:57 2017 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Wed Oct 11 14:29:57 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../hive/ql/io/sarg/ConvertAstToSearchArg.java | 59 ++++++++++++++++++--
2 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f1050a6e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 942629b..9084785 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1393,6 +1393,9 @@ public class HiveConf extends Configuration {
"references for the cached object. Setting this to true can help avoid out of memory\n" +
"issues under memory pressure (in some cases) at the cost of slight unpredictability in\n" +
"overall query performance."),
+ HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB("hive.io.sarg.cache.max.weight.mb", 10,
+ "The max weight allowed for the SearchArgument Cache. By default, the cache allows a max-weight of 10MB, " +
+ "after which entries will be evicted."),
HIVE_LAZYSIMPLE_EXTENDED_BOOLEAN_LITERAL("hive.lazysimple.extended_boolean_literal", false,
"LazySimpleSerde uses this property to determine if it treats 'T', 't', 'F', 'f',\n" +
http://git-wip-us.apache.org/repos/asf/hive/blob/f1050a6e/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
index 997334b..b9bf801 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
@@ -21,13 +21,16 @@ package org.apache.hadoop.hive.ql.io.sarg;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.List;
+import java.util.concurrent.ExecutionException;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.apache.hadoop.hive.ql.io.sarg.LiteralDelegate;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -436,14 +439,60 @@ public class ConvertAstToSearchArg {
}
}
-
public static final String SARG_PUSHDOWN = "sarg.pushdown";
+ private static volatile Cache<String, SearchArgument> sargsCache = null;
+
+ private static synchronized Cache<String, SearchArgument> initializeAndGetSargsCache(Configuration conf) {
+ if (sargsCache == null) {
+ sargsCache = CacheBuilder.newBuilder()
+ .weigher((String key, SearchArgument value) -> key.length())
+ .maximumWeight(
+ HiveConf.getIntVar(conf,
+ HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) * 1024 *1024
+ )
+ .build(); // Can't use CacheLoader because SearchArguments may be built either from Kryo strings,
+ // or from expressions.
+ }
+ return sargsCache;
+ }
+
+ private static Cache<String, SearchArgument> getSargsCache(Configuration conf) {
+ return sargsCache == null? initializeAndGetSargsCache(conf) : sargsCache;
+ }
+
+ private static boolean isSargsCacheEnabled(Configuration conf) {
+ return HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) > 0;
+ }
+
+ private static SearchArgument getSearchArgumentFromString(Configuration conf, String sargString) {
+
+ try {
+ return isSargsCacheEnabled(conf)? getSargsCache(conf).get(sargString, () -> create(sargString))
+ : create(sargString);
+ }
+ catch (ExecutionException exception) {
+ throw new RuntimeException(exception);
+ }
+ }
+
+ private static SearchArgument getSearchArgumentFromExpression(Configuration conf, String sargString) {
+
+ try {
+ return isSargsCacheEnabled(conf)?
+ getSargsCache(conf).get(sargString,
+ () -> create(conf, SerializationUtilities.deserializeExpression(sargString)))
+ : create(conf, SerializationUtilities.deserializeExpression(sargString));
+ }
+ catch (ExecutionException exception) {
+ throw new RuntimeException(exception);
+ }
+ }
+
public static SearchArgument create(Configuration conf, ExprNodeGenericFuncDesc expression) {
return new ConvertAstToSearchArg(conf, expression).buildSearchArgument();
}
-
private final static ThreadLocal<Kryo> kryo = new ThreadLocal<Kryo>() {
protected Kryo initialValue() { return new Kryo(); }
};
@@ -459,9 +508,9 @@ public class ConvertAstToSearchArg {
public static SearchArgument createFromConf(Configuration conf) {
String sargString;
if ((sargString = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR)) != null) {
- return create(conf, SerializationUtilities.deserializeExpression(sargString));
+ return getSearchArgumentFromExpression(conf, sargString);
} else if ((sargString = conf.get(SARG_PUSHDOWN)) != null) {
- return create(sargString);
+ return getSearchArgumentFromString(conf, sargString);
}
return null;
}