You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mi...@apache.org on 2017/10/11 23:04:45 UTC
hive git commit: HIVE-17669: Cache to optimize SearchArgument
deserialization (Mithun Radhakrishnan, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/branch-2.2 1f16c0721 -> b8cff2d22
HIVE-17669: Cache to optimize SearchArgument deserialization (Mithun Radhakrishnan, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b8cff2d2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b8cff2d2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b8cff2d2
Branch: refs/heads/branch-2.2
Commit: b8cff2d221c2c5e6f59a9452e2a5fbac1ea2a6ba
Parents: 1f16c07
Author: Mithun RK <mi...@apache.org>
Authored: Tue Oct 3 13:33:48 2017 -0700
Committer: Mithun Radhakrishnan <mi...@apache.org>
Committed: Wed Oct 11 16:03:07 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +
.../hive/ql/io/sarg/ConvertAstToSearchArg.java | 76 ++++++++++++++++++--
2 files changed, 74 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b8cff2d2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 06efb88..9ee4ca0 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1371,6 +1371,9 @@ public class HiveConf extends Configuration {
"references for the cached object. Setting this to true can help avoid out of memory\n" +
"issues under memory pressure (in some cases) at the cost of slight unpredictability in\n" +
"overall query performance."),
+ HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB("hive.io.sarg.cache.max.weight.mb", 10,
+ "The max weight allowed for the SearchArgument Cache. By default, the cache allows a max-weight of 10MB, " +
+ "after which entries will be evicted."),
HIVE_ORC_SKIP_CORRUPT_DATA("hive.exec.orc.skip.corrupt.data", false,
"If ORC reader encounters corrupt data, this value will be used to determine\n" +
"whether to skip the corrupt data or throw exception. The default behavior is to throw exception."),
http://git-wip-us.apache.org/repos/asf/hive/blob/b8cff2d2/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
index a3d3278..f9e4c85 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java
@@ -21,13 +21,18 @@ package org.apache.hadoop.hive.ql.io.sarg;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.apache.hadoop.hive.ql.io.sarg.LiteralDelegate;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -437,14 +442,75 @@ public class ConvertAstToSearchArg {
}
}
-
public static final String SARG_PUSHDOWN = "sarg.pushdown";
+ private static volatile Cache<String, SearchArgument> sargsCache = null;
+
+ private static synchronized Cache<String, SearchArgument> initializeAndGetSargsCache(Configuration conf) {
+ if (sargsCache == null) {
+ sargsCache = CacheBuilder.newBuilder()
+ .weigher(new Weigher<String, SearchArgument>() {
+ @Override
+ public int weigh(String key, SearchArgument value) {
+ return key.length();
+ }
+ })
+ .maximumWeight(
+ HiveConf.getIntVar(conf,
+ HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) * 1024 *1024
+ )
+ .build(); // Can't use CacheLoader because SearchArguments may be built either from Kryo strings,
+ // or from expressions.
+ }
+ return sargsCache;
+ }
+
+ private static Cache<String, SearchArgument> getSargsCache(Configuration conf) {
+ return sargsCache == null? initializeAndGetSargsCache(conf) : sargsCache;
+ }
+
+ private static boolean isSargsCacheEnabled(Configuration conf) {
+ return HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_IO_SARG_CACHE_MAX_WEIGHT_MB) > 0;
+ }
+
+ private static SearchArgument getSearchArgumentFromString(Configuration conf, final String sargString) {
+
+ try {
+ return isSargsCacheEnabled(conf)?
+ getSargsCache(conf).get(sargString, new Callable<SearchArgument>() {
+ @Override
+ public SearchArgument call() {
+ return create(sargString);
+ }
+ })
+ : create(sargString);
+ }
+ catch (ExecutionException exception) {
+ throw new RuntimeException(exception);
+ }
+ }
+
+ private static SearchArgument getSearchArgumentFromExpression(final Configuration conf, final String sargString) {
+
+ try {
+ return isSargsCacheEnabled(conf)?
+ getSargsCache(conf).get(sargString, new Callable<SearchArgument>() {
+ @Override
+ public SearchArgument call() {
+ return create(conf, SerializationUtilities.deserializeExpression(sargString));
+ }
+ })
+ : create(conf, SerializationUtilities.deserializeExpression(sargString));
+ }
+ catch (ExecutionException exception) {
+ throw new RuntimeException(exception);
+ }
+ }
+
public static SearchArgument create(Configuration conf, ExprNodeGenericFuncDesc expression) {
return new ConvertAstToSearchArg(conf, expression).buildSearchArgument();
}
-
private final static ThreadLocal<Kryo> kryo = new ThreadLocal<Kryo>() {
protected Kryo initialValue() { return new Kryo(); }
};
@@ -460,9 +526,9 @@ public class ConvertAstToSearchArg {
public static SearchArgument createFromConf(Configuration conf) {
String sargString;
if ((sargString = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR)) != null) {
- return create(conf, SerializationUtilities.deserializeExpression(sargString));
+ return getSearchArgumentFromExpression(conf, sargString);
} else if ((sargString = conf.get(SARG_PUSHDOWN)) != null) {
- return create(sargString);
+ return getSearchArgumentFromString(conf, sargString);
}
return null;
}