You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2015/08/11 00:40:20 UTC
[1/2] hive git commit: HIVE-8518: Compile time skew join optimization
returns duplicated results (Rui via Xuefu) merged from trunk r1633422
Repository: hive
Updated Branches:
refs/heads/branch-1.0 f81a9b39d -> deb35763d
HIVE-8518: Compile time skew join optimization returns duplicated results (Rui via Xuefu) merged from trunk r1633422
git-svn-id: https://svn.apache.org/repos/asf/hive/branches/spark@1633508 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/846dd922
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/846dd922
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/846dd922
Branch: refs/heads/branch-1.0
Commit: 846dd922060f488bf29367744ecc4bcfa6efa160
Parents: f81a9b3
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon Aug 10 15:31:23 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Aug 10 15:31:23 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/optimizer/SkewJoinOptimizer.java | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/846dd922/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
index 54afbfe..e87c41b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SkewJoinOptimizer.java
@@ -171,15 +171,17 @@ public class SkewJoinOptimizer implements Transform {
joinOpClone = (JoinOperator)currOpClone;
}
+ List<TableScanOperator> tableScanCloneOpsForJoin =
+ new ArrayList<TableScanOperator>();
+ if (!getTableScanOpsForJoin(joinOpClone, tableScanCloneOpsForJoin)) {
+ LOG.debug("Operator tree not properly cloned!");
+ return null;
+ }
+
// Put the filter "skewed column = skewed keys" in op
// and "skewed columns != skewed keys" in selectOpClone
insertSkewFilter(tableScanOpsForJoin, skewedValues, true);
- List<TableScanOperator> tableScanCloneOpsForJoin =
- new ArrayList<TableScanOperator>();
- assert
- getTableScanOpsForJoin(joinOpClone, tableScanCloneOpsForJoin);
-
insertSkewFilter(tableScanCloneOpsForJoin, skewedValues, false);
// Update the topOps appropriately
[2/2] hive git commit: HIVE-8680 - Set Max Message for Binary Thrift
endpoints (Brock reviewed by Szehon)
Posted by jd...@apache.org.
HIVE-8680 - Set Max Message for Binary Thrift endpoints (Brock reviewed by Szehon)
git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1636192 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/deb35763
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/deb35763
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/deb35763
Branch: refs/heads/branch-1.0
Commit: deb35763d08071f1cbd901ba15b321a2b5853934
Parents: 846dd92
Author: Brock Noland <br...@apache.org>
Authored: Sun Nov 2 21:08:27 2014 +0000
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon Aug 10 15:36:48 2015 -0700
----------------------------------------------------------------------
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 4 ++++
.../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 2 ++
.../apache/hive/service/cli/thrift/ThriftBinaryCLIService.java | 5 ++++-
3 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/deb35763/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 23ab4df..2b8ccc5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -385,6 +385,8 @@ public class HiveConf extends Configuration {
"the connection URL, before the next metastore query that accesses the\n" +
"datastore. Once reloaded, this value is reset to false. Used for\n" +
"testing only."),
+ METASTORESERVERMAXMESSAGESIZE("hive.metastore.server.max.message.size", 100*1024*1024,
+ "Maximum message size in bytes a HMS will accept."),
METASTORESERVERMINTHREADS("hive.metastore.server.min.threads", 200,
"Minimum number of worker threads in the Thrift server's pool."),
METASTORESERVERMAXTHREADS("hive.metastore.server.max.threads", 100000,
@@ -1628,6 +1630,8 @@ public class HiveConf extends Configuration {
"Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'http'."),
HIVE_SERVER2_THRIFT_HTTP_PATH("hive.server2.thrift.http.path", "cliservice",
"Path component of URL endpoint when in HTTP mode."),
+ HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE("hive.server2.thrift.max.message.size", 100*1024*1024,
+ "Maximum message size in bytes a HS2 server will accept."),
HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS("hive.server2.thrift.http.min.worker.threads", 5,
"Minimum number of worker threads when in HTTP mode."),
HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS("hive.server2.thrift.http.max.worker.threads", 500,
http://git-wip-us.apache.org/repos/asf/hive/blob/deb35763/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 8d647e3..cbfd036 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -5697,6 +5697,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// Server will create new threads up to max as necessary. After an idle
// period, it will destory threads to keep the number of threads in the
// pool to min.
+ int maxMessageSize = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXMESSAGESIZE);
int minWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMINTHREADS);
int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS);
boolean tcpKeepAlive = conf.getBoolVar(HiveConf.ConfVars.METASTORE_TCP_KEEP_ALIVE);
@@ -5750,6 +5751,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
.processor(processor)
.transportFactory(transFactory)
.protocolFactory(new TBinaryProtocol.Factory())
+ .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize))
.minWorkerThreads(minWorkerThreads)
.maxWorkerThreads(maxWorkerThreads);
http://git-wip-us.apache.org/repos/asf/hive/blob/deb35763/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index 47208bf..299c734 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -81,9 +81,12 @@ public class ThriftBinaryCLIService extends ThriftCLIService {
}
// Server args
+ int maxMessageSize = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE);
TThreadPoolServer.Args sargs = new TThreadPoolServer.Args(serverSocket)
.processorFactory(processorFactory).transportFactory(transportFactory)
- .protocolFactory(new TBinaryProtocol.Factory()).executorService(executorService);
+ .protocolFactory(new TBinaryProtocol.Factory())
+ .inputProtocolFactory(new TBinaryProtocol.Factory(true, true, maxMessageSize))
+ .executorService(executorService);
// TCP Server
server = new TThreadPoolServer(sargs);