You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/30 21:17:21 UTC
[12/12] hive git commit: HIVE-14671 : Merge branch 'master' into
hive-14535 (Wei Zheng)
HIVE-14671 : Merge branch 'master' into hive-14535 (Wei Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/144efb0f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/144efb0f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/144efb0f
Branch: refs/heads/hive-14535
Commit: 144efb0fcc8b74484c860e3559666a313641c93d
Parents: 52e0f8f 4cd4251
Author: Wei Zheng <we...@apache.org>
Authored: Tue May 30 14:16:03 2017 -0700
Committer: Wei Zheng <we...@apache.org>
Committed: Tue May 30 14:16:03 2017 -0700
----------------------------------------------------------------------
.../hive/common/jsonexplain/DagJsonParser.java | 6 +-
.../java/org/apache/hive/http/HttpServer.java | 1 +
.../hive/http/Log4j2ConfiguratorServlet.java | 275 +++++++++++++++++++
.../hive/druid/DruidStorageHandlerUtils.java | 34 ++-
.../hadoop/hive/druid/io/DruidOutputFormat.java | 7 +-
.../hadoop/hive/druid/io/DruidRecordWriter.java | 2 +-
.../serde/HiveDruidSerializationModule.java | 37 +++
.../serde/PeriodGranularitySerializer.java | 54 ++++
.../hive/druid/DerbyConnectorTestUtility.java | 4 +-
.../hadoop/hive/druid/TestDruidSerDe.java | 2 +-
.../hive/druid/TestDruidStorageHandler.java | 2 +-
.../TestHiveDruidQueryBasedInputFormat.java | 38 +--
.../hive/ql/io/TestDruidRecordWriter.java | 11 +-
.../listener/DbNotificationListener.java | 11 +-
.../listener/TestDbNotificationListener.java | 28 +-
.../hive/ql/parse/TestReplicationScenarios.java | 130 +++++++++
.../apache/hadoop/hive/llap/LlapDaemonInfo.java | 14 +-
llap-server/bin/runLlapDaemon.sh | 2 +-
.../hive/llap/daemon/impl/LlapDaemon.java | 8 +-
.../hadoop/hive/llap/daemon/impl/QueryInfo.java | 2 +
.../llap/daemon/impl/TaskExecutorService.java | 42 ++-
.../llap/daemon/impl/TaskRunnerCallable.java | 14 +-
.../llap/io/encoded/OrcEncodedDataReader.java | 4 +-
.../hive/llap/metrics/LlapDaemonJvmInfo.java | 2 +
.../hive/llap/metrics/LlapDaemonJvmMetrics.java | 35 ++-
.../hive/llap/daemon/MiniLlapCluster.java | 2 +-
.../llap/tezplugins/LlapTaskCommunicator.java | 5 +-
.../upgrade/mysql/041-HIVE-16556.mysql.sql | 2 +-
.../hive/metastore/cache/CachedStore.java | 25 +-
.../hive/metastore/events/InsertEvent.java | 44 ++-
.../hive/metastore/messaging/InsertMessage.java | 24 +-
.../metastore/messaging/MessageFactory.java | 10 +-
.../messaging/json/JSONInsertMessage.java | 51 +++-
.../messaging/json/JSONMessageFactory.java | 10 +-
pom.xml | 2 +-
.../hive/ql/exec/AppMasterEventOperator.java | 14 +-
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 2 +-
.../hadoop/hive/ql/exec/DemuxOperator.java | 26 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 20 +-
.../hadoop/hive/ql/exec/GroupByOperator.java | 17 +-
.../hive/ql/exec/HashTableSinkOperator.java | 4 +-
.../hadoop/hive/ql/exec/JoinOperator.java | 2 +-
.../hadoop/hive/ql/exec/MapJoinOperator.java | 8 +-
.../apache/hadoop/hive/ql/exec/MapOperator.java | 10 +-
.../apache/hadoop/hive/ql/exec/MuxOperator.java | 8 +-
.../apache/hadoop/hive/ql/exec/Operator.java | 49 ++--
.../hive/ql/exec/OrcFileMergeOperator.java | 6 +-
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 12 +-
.../hadoop/hive/ql/exec/SMBMapJoinOperator.java | 8 +-
.../hadoop/hive/ql/exec/ScriptOperator.java | 8 +-
.../hadoop/hive/ql/exec/SelectOperator.java | 2 +-
.../hadoop/hive/ql/exec/TableScanOperator.java | 8 +-
.../hadoop/hive/ql/exec/UnionOperator.java | 2 +-
.../hadoop/hive/ql/exec/mr/ExecReducer.java | 12 +-
.../hadoop/hive/ql/exec/mr/ObjectCache.java | 5 +-
.../ql/exec/spark/SparkMapRecordHandler.java | 7 +-
.../ql/exec/spark/SparkReduceRecordHandler.java | 9 +-
.../ql/exec/tez/ColumnarSplitSizeEstimator.java | 5 +-
.../tez/HostAffinitySplitLocationProvider.java | 2 +-
.../hive/ql/exec/tez/LlapObjectCache.java | 12 +-
.../hive/ql/exec/tez/RecordProcessor.java | 6 -
.../mapjoin/VectorMapJoinCommonOperator.java | 12 +-
.../VectorMapJoinGenerateResultOperator.java | 8 +-
.../VectorMapJoinInnerBigOnlyLongOperator.java | 8 +-
...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 8 +-
...VectorMapJoinInnerBigOnlyStringOperator.java | 8 +-
.../mapjoin/VectorMapJoinInnerLongOperator.java | 8 +-
.../VectorMapJoinInnerMultiKeyOperator.java | 8 +-
.../VectorMapJoinInnerStringOperator.java | 8 +-
.../VectorMapJoinLeftSemiLongOperator.java | 8 +-
.../VectorMapJoinLeftSemiMultiKeyOperator.java | 8 +-
.../VectorMapJoinLeftSemiStringOperator.java | 8 +-
...ectorMapJoinOuterGenerateResultOperator.java | 12 +-
.../mapjoin/VectorMapJoinOuterLongOperator.java | 10 +-
.../VectorMapJoinOuterMultiKeyOperator.java | 10 +-
.../VectorMapJoinOuterStringOperator.java | 10 +-
.../fast/VectorMapJoinFastBytesHashTable.java | 6 +-
.../fast/VectorMapJoinFastLongHashTable.java | 6 +-
.../VectorReduceSinkCommonOperator.java | 8 +-
.../hadoop/hive/ql/io/orc/ExternalCache.java | 3 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 11 +-
.../vector/VectorizedParquetRecordReader.java | 3 +
.../stats/annotation/StatsRulesProcFactory.java | 56 ++--
.../parse/repl/dump/events/InsertHandler.java | 28 +-
.../hadoop/hive/ql/session/SessionState.java | 5 +-
.../results/clientpositive/llap/cte_mat_3.q.out | 2 +-
.../results/clientpositive/llap/cte_mat_4.q.out | 4 +-
.../results/clientpositive/llap/cte_mat_5.q.out | 2 +-
.../spark/spark_explainuser_1.q.out | 24 +-
.../hadoop/hive/serde2/lazy/LazyBinary.java | 5 +-
90 files changed, 1022 insertions(+), 479 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/144efb0f/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/144efb0f/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 351059b,3e09432..004aaf7
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@@ -305,90 -265,8 +303,90 @@@ public class FileSinkOperator extends T
}
}
- public Stat getStat() {
- return stat;
+ public void configureDynPartPath(String dirName, String childSpecPathDynLinkedPartitions) {
+ dirName = (childSpecPathDynLinkedPartitions == null) ? dirName :
+ dirName + Path.SEPARATOR + childSpecPathDynLinkedPartitions;
+ tmpPath = new Path(tmpPath, dirName);
+ if (taskOutputTempPath != null) {
+ taskOutputTempPath = new Path(taskOutputTempPath, dirName);
+ }
+ }
+
+ public void initializeBucketPaths(int filesIdx, String taskId, boolean isNativeTable,
+ boolean isSkewedStoredAsSubDirectories) {
+ if (isNativeTable) {
+ String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat);
+ if (!isMmTable) {
+ if (!bDynParts && !isSkewedStoredAsSubDirectories) {
+ finalPaths[filesIdx] = getFinalPath(taskId, parent, extension);
+ } else {
+ finalPaths[filesIdx] = getFinalPath(taskId, tmpPath, extension);
+ }
+ outPaths[filesIdx] = getTaskOutPath(taskId);
+ } else {
+ String subdirPath = AcidUtils.deltaSubdir(txnId, txnId, stmtId);
+ if (unionPath != null) {
+ // Create the union directory inside the MM directory.
+ subdirPath += Path.SEPARATOR + unionPath;
+ }
+ subdirPath += Path.SEPARATOR + taskId;
+ if (conf.isMerge()) {
+ // Make sure we don't collide with the source files.
+ // MM tables don't support concat so we don't expect the merge of merged files.
+ subdirPath += ".merged";
+ }
+ Path finalPath = null;
+ if (!bDynParts && !isSkewedStoredAsSubDirectories) {
+ finalPath = getFinalPath(subdirPath, specPath, extension);
+ } else {
+ // Note: tmpPath here has the correct partition key
+ finalPath = getFinalPath(subdirPath, tmpPath, extension);
+ }
+ // In the cases that have multi-stage insert, e.g. a "hive.skewjoin.key"-based skew join,
+ // it can happen that we want multiple commits into the same directory from different
+ // tasks (not just task instances). In non-MM case, Utilities.renameOrMoveFiles ensures
+ // unique names. We could do the same here, but this will still cause the old file to be
+ // deleted because it has not been committed in /this/ FSOP. We are going to fail to be
+ // safe. Potentially, we could implement some partial commit between stages, if this
+ // affects some less obscure scenario.
+ try {
+ FileSystem fpfs = finalPath.getFileSystem(hconf);
+ if (fpfs.exists(finalPath)) throw new RuntimeException(finalPath + " already exists");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ finalPaths[filesIdx] = finalPath;
+ outPaths[filesIdx] = finalPath;
+ }
- if (isInfoEnabled) {
++ if (LOG.isInfoEnabled()) {
+ LOG.info("Final Path: FS " + finalPaths[filesIdx]);
- if (isInfoEnabled && !isMmTable) {
++ if (LOG.isInfoEnabled() && !isMmTable) {
+ LOG.info("Writing to temp file: FS " + outPaths[filesIdx]);
+ }
+ }
+ } else {
+ finalPaths[filesIdx] = outPaths[filesIdx] = specPath;
+ }
+ }
+
+ public Path getTmpPath() {
+ return tmpPath;
+ }
+
+ public Path getTaskOutputTempPath() {
+ return taskOutputTempPath;
+ }
+
+ public void addToStat(String statType, long amount) {
+ if ("rowCount".equals(statType)) {
+ Utilities.LOG14535.info("Adding " + statType + " = " + amount + " to " + System.identityHashCode(this));
+ }
+ stat.addToStat(statType, amount);
+ }
+
+ public Collection<String> getStoredStats() {
+ Utilities.LOG14535.info("Getting stats from " + System.identityHashCode(this));
+ return stat.getStoredStats();
}
} // class FSPaths
@@@ -715,12 -578,30 +713,12 @@@
protected void createBucketForFileIdx(FSPaths fsp, int filesIdx)
throws HiveException {
try {
- if (isNativeTable) {
- fsp.finalPaths[filesIdx] = fsp.getFinalPath(taskId, fsp.tmpPath, null);
- if (LOG.isInfoEnabled()) {
- LOG.info("Final Path: FS " + fsp.finalPaths[filesIdx]);
- }
- fsp.outPaths[filesIdx] = fsp.getTaskOutPath(taskId);
- if (LOG.isInfoEnabled()) {
- LOG.info("Writing to temp file: FS " + fsp.outPaths[filesIdx]);
- }
- } else {
- fsp.finalPaths[filesIdx] = fsp.outPaths[filesIdx] = specPath;
- }
- // The reason to keep these instead of using
- // OutputFormat.getRecordWriter() is that
- // getRecordWriter does not give us enough control over the file name that
- // we create.
- String extension = Utilities.getFileExtension(jc, isCompressed, hiveOutputFormat);
- if (!bDynParts && !this.isSkewedStoredAsSubDirectories) {
- fsp.finalPaths[filesIdx] = fsp.getFinalPath(taskId, parent, extension);
- } else {
- fsp.finalPaths[filesIdx] = fsp.getFinalPath(taskId, fsp.tmpPath, extension);
- }
+ fsp.initializeBucketPaths(filesIdx, taskId, isNativeTable, isSkewedStoredAsSubDirectories);
+ Utilities.LOG14535.info("createBucketForFileIdx " + filesIdx + ": final path " + fsp.finalPaths[filesIdx]
+ + "; out path " + fsp.outPaths[filesIdx] +" (spec path " + specPath + ", tmp path "
+ + fsp.getTmpPath() + ", task " + taskId + ")"/*, new Exception()*/);
- if (isInfoEnabled) {
+ if (LOG.isInfoEnabled()) {
LOG.info("New Final Path: FS " + fsp.finalPaths[filesIdx]);
}
@@@ -860,12 -736,12 +858,12 @@@
if (conf.isGatherStats() && !isCollectRWStats) {
SerDeStats stats = serializer.getSerDeStats();
if (stats != null) {
- fpaths.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
+ fpaths.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
}
- fpaths.stat.addToStat(StatsSetupConst.ROW_COUNT, 1);
+ fpaths.addToStat(StatsSetupConst.ROW_COUNT, 1);
}
- if ((++numRows == cntr) && isLogInfoEnabled) {
+ if ((++numRows == cntr) && LOG.isInfoEnabled()) {
cntr = logEveryNRows == 0 ? cntr * 10 : numRows + logEveryNRows;
if (cntr < 0 || numRows < 0) {
cntr = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/144efb0f/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/144efb0f/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/144efb0f/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
index a845b50,d9547b9..44e50e2
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OrcFileMergeOperator.java
@@@ -130,9 -126,8 +130,9 @@@ public class OrcFileMergeOperator exten
options.bufferSize(compressBuffSize).enforceBufferSize();
}
+ Path outPath = getOutPath();
outWriter = OrcFile.createWriter(outPath, options);
- if (isLogDebugEnabled) {
+ if (LOG.isDebugEnabled()) {
LOG.info("ORC merge file output path: " + outPath);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/144efb0f/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/144efb0f/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------