You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/11/22 02:46:34 UTC
[25/35] hive git commit: HIVE-14902: HiveServer2: Cleanup code which
checks for ThriftJDBCSerde usage (Ziyang Zhao reviewed by Vaibhav Gumashta)
HIVE-14902: HiveServer2: Cleanup code which checks for ThriftJDBCSerde usage (Ziyang Zhao reviewed by Vaibhav Gumashta)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bbcff5ea
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bbcff5ea
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bbcff5ea
Branch: refs/heads/hive-14535
Commit: bbcff5ea1c719438a4fb74cbe16c80a9787e278a
Parents: 3baca6c
Author: Vaibhav Gumashta <vg...@hortonworks.com>
Authored: Fri Nov 18 14:54:27 2016 -0800
Committer: Vaibhav Gumashta <vg...@hortonworks.com>
Committed: Fri Nov 18 14:54:27 2016 -0800
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 5 +----
.../apache/hadoop/hive/ql/exec/FileSinkOperator.java | 5 +----
.../apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 12 +++++++++++-
.../org/apache/hadoop/hive/ql/parse/TaskCompiler.java | 11 ++++++++++-
.../org/apache/hadoop/hive/ql/plan/FetchWork.java | 14 ++++++++++++++
.../org/apache/hadoop/hive/ql/plan/FileSinkDesc.java | 14 ++++++++++++++
.../hive/service/cli/operation/SQLOperation.java | 5 +----
7 files changed, 52 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bbcff5ea/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index b77948b..757c60c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -2184,10 +2184,7 @@ public class Driver implements CommandProcessor {
* indeed written using ThriftJDBCBinarySerDe, read one row from the output sequence file,
* since it is a blob of row batches.
*/
- if (fetchTask.getWork().isHiveServerQuery() && HiveConf.getBoolVar(conf,
- HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)
- && (fetchTask.getTblDesc().getSerdeClassName()
- .equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()))) {
+ if (fetchTask.getWork().isUsingThriftJDBCBinarySerDe()) {
maxRows = 1;
}
fetchTask.setMaxRows(maxRows);
http://git-wip-us.apache.org/repos/asf/hive/blob/bbcff5ea/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index eeba6cd..54d619c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -72,7 +72,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.SubStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
-import org.apache.hadoop.hive.serde2.thrift.ThriftJDBCBinarySerDe;
import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyShim;
import org.apache.hadoop.hive.shims.HadoopShims.StoragePolicyValue;
import org.apache.hadoop.hive.shims.ShimLoader;
@@ -1023,9 +1022,7 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
// If serializer is ThriftJDBCBinarySerDe, then it buffers rows to a certain limit (hive.server2.thrift.resultset.max.fetch.size)
// and serializes the whole batch when the buffer is full. The serialize returns null if the buffer is not full
// (the size of buffer is kept track of in the ThriftJDBCBinarySerDe).
- if (conf.isHiveServerQuery() && HiveConf.getBoolVar(hconf,
- HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS) &&
- serializer.getClass().getName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName())) {
+ if (conf.isUsingThriftJDBCBinarySerDe()) {
try {
recordValue = serializer.serialize(null, inputObjInspectors[0]);
if ( null != fpaths ) {
http://git-wip-us.apache.org/repos/asf/hive/blob/bbcff5ea/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 47feeaf..7d8b2bd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6990,7 +6990,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
dpCtx,
dest_path);
- fileSinkDesc.setHiveServerQuery(SessionState.get().isHiveServerQuery());
+ boolean isHiveServerQuery = SessionState.get().isHiveServerQuery();
+ fileSinkDesc.setHiveServerQuery(isHiveServerQuery);
// If this is an insert, update, or delete on an ACID table then mark that so the
// FileSinkOperator knows how to properly write to it.
if (destTableIsAcid) {
@@ -7033,6 +7034,15 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
fileSinkDesc.setStaticSpec(dpCtx.getSPPath());
}
+ if (isHiveServerQuery &&
+ null != table_desc &&
+ table_desc.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) &&
+ HiveConf.getBoolVar(conf,HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) {
+ fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(true);
+ } else {
+ fileSinkDesc.setIsUsingThriftJDBCBinarySerDe(false);
+ }
+
Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
fileSinkDesc, fsRS, input), inputRR);
http://git-wip-us.apache.org/repos/asf/hive/blob/bbcff5ea/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index 97cf585..8ce8ea3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -181,9 +181,18 @@ public abstract class TaskCompiler {
}
FetchWork fetch = new FetchWork(loadFileDesc.getSourcePath(), resultTab, outerQueryLimit);
- fetch.setHiveServerQuery(SessionState.get().isHiveServerQuery());
+ boolean isHiveServerQuery = SessionState.get().isHiveServerQuery();
+ fetch.setHiveServerQuery(isHiveServerQuery);
fetch.setSource(pCtx.getFetchSource());
fetch.setSink(pCtx.getFetchSink());
+ if (isHiveServerQuery &&
+ null != resultTab &&
+ resultTab.getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class.getName()) &&
+ HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)) {
+ fetch.setIsUsingThriftJDBCBinarySerDe(true);
+ } else {
+ fetch.setIsUsingThriftJDBCBinarySerDe(false);
+ }
pCtx.setFetchTask((FetchTask) TaskFactory.get(fetch, conf));
http://git-wip-us.apache.org/repos/asf/hive/blob/bbcff5ea/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java
index 8ea6440..2ecb6f8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java
@@ -65,6 +65,12 @@ public class FetchWork implements Serializable {
private boolean isHiveServerQuery;
+ /**
+ * Whether is a HiveServer query, and the destination table is
+ * indeed written using ThriftJDBCBinarySerDe
+ */
+ private boolean isUsingThriftJDBCBinarySerDe = false;
+
public boolean isHiveServerQuery() {
return isHiveServerQuery;
}
@@ -73,6 +79,14 @@ public class FetchWork implements Serializable {
this.isHiveServerQuery = isHiveServerQuery;
}
+ public boolean isUsingThriftJDBCBinarySerDe() {
+ return isUsingThriftJDBCBinarySerDe;
+ }
+
+ public void setIsUsingThriftJDBCBinarySerDe(boolean isUsingThriftJDBCBinarySerDe) {
+ this.isUsingThriftJDBCBinarySerDe = isUsingThriftJDBCBinarySerDe;
+ }
+
public FetchWork() {
}
http://git-wip-us.apache.org/repos/asf/hive/blob/bbcff5ea/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
index 07ed4fd..4d9139b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FileSinkDesc.java
@@ -97,6 +97,12 @@ public class FileSinkDesc extends AbstractOperatorDesc {
private Path destPath;
private boolean isHiveServerQuery;
+ /**
+ * Whether is a HiveServer query, and the destination table is
+ * indeed written using ThriftJDBCBinarySerDe
+ */
+ private boolean isUsingThriftJDBCBinarySerDe = false;
+
public FileSinkDesc() {
}
@@ -166,6 +172,14 @@ public class FileSinkDesc extends AbstractOperatorDesc {
this.isHiveServerQuery = isHiveServerQuery;
}
+ public boolean isUsingThriftJDBCBinarySerDe() {
+ return this.isUsingThriftJDBCBinarySerDe;
+ }
+
+ public void setIsUsingThriftJDBCBinarySerDe(boolean isUsingThriftJDBCBinarySerDe) {
+ this.isUsingThriftJDBCBinarySerDe = isUsingThriftJDBCBinarySerDe;
+ }
+
@Explain(displayName = "directory", explainLevels = { Level.EXTENDED })
public Path getDirName() {
return dirName;
http://git-wip-us.apache.org/repos/asf/hive/blob/bbcff5ea/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
index ba02c9c..80a615d 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
@@ -466,10 +466,7 @@ public class SQLOperation extends ExecuteStatementOperation {
FetchTask fetchTask = driver.getFetchTask();
boolean isBlobBased = false;
- if (fetchTask != null && fetchTask.getWork().isHiveServerQuery() && HiveConf.getBoolVar(queryState.getConf(),
- HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_SERIALIZE_IN_TASKS)
- && (fetchTask.getTblDesc().getSerdeClassName().equalsIgnoreCase(ThriftJDBCBinarySerDe.class
- .getName()))) {
+ if (fetchTask != null && fetchTask.getWork().isUsingThriftJDBCBinarySerDe()) {
// Just fetch one blob if we've serialized thrift objects in final tasks
maxRows = 1;
isBlobBased = true;