You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/14 10:22:47 UTC
[24/24] hive git commit: HIVE-11811: LLAP: Merge master into branch
(Prasanth Jayachandran)
HIVE-11811: LLAP: Merge master into branch (Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b0154f4c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b0154f4c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b0154f4c
Branch: refs/heads/llap
Commit: b0154f4c40650e2d74d65e5fb41bcd66df3932da
Parents: cb9fab7 da0be3d
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Mon Sep 14 03:21:51 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Mon Sep 14 03:21:51 2015 -0500
----------------------------------------------------------------------
beeline/src/main/resources/beeline-log4j2.xml | 5 +-
.../hadoop/hive/common/jsonexplain/tez/Op.java | 8 +-
.../hive/common/jsonexplain/tez/Stage.java | 14 +-
.../common/jsonexplain/tez/TezJsonParser.java | 17 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 13 +
common/src/main/resources/hive-log4j2.xml | 5 +-
.../test/resources/hive-exec-log4j2-test.xml | 5 +-
common/src/test/resources/hive-log4j2-test.xml | 5 +-
data/conf/hive-log4j2.xml | 5 +-
data/files/dynpartdata1.txt | 5 +
data/files/dynpartdata2.txt | 6 +
.../deployers/config/hive/hive-log4j2.xml | 5 +-
.../svr/src/main/config/webhcat-log4j2.xml | 5 +-
.../hadoop/hive/metastore/ObjectStore.java | 19 +-
.../hive/metastore/tools/HiveMetaTool.java | 5 +
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 24 +-
.../hadoop/hive/ql/exec/KeyWrapperFactory.java | 4 +
.../hadoop/hive/ql/exec/MapJoinOperator.java | 5 +
.../apache/hadoop/hive/ql/exec/MoveTask.java | 12 +-
.../apache/hadoop/hive/ql/exec/Operator.java | 7 +
.../apache/hadoop/hive/ql/exec/StatsTask.java | 13 +-
.../persistence/BytesBytesMultiHashMap.java | 11 +-
.../persistence/HybridHashTableContainer.java | 68 +-
.../hadoop/hive/ql/exec/tez/DagUtils.java | 3 +
.../hive/ql/exec/tez/HashTableLoader.java | 7 +-
.../hadoop/hive/ql/exec/tez/InPlaceUpdates.java | 65 +
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 70 +-
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 4 +-
.../apache/hadoop/hive/ql/hooks/ATSHook.java | 9 +-
.../serde/ParquetHiveArrayInspector.java | 12 +
.../ql/io/parquet/timestamp/NanoTimeUtils.java | 23 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 25 +-
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 18 +-
.../hive/ql/optimizer/GenMapRedUtils.java | 57 +-
.../ql/optimizer/ReduceSinkMapJoinProc.java | 19 +-
.../calcite/reloperators/HiveLimit.java | 57 -
.../calcite/reloperators/HiveSort.java | 110 -
.../calcite/reloperators/HiveSortLimit.java | 110 +
.../calcite/stats/HiveRelMdMemory.java | 9 +-
.../calcite/stats/HiveRelMdParallelism.java | 4 +-
.../calcite/translator/ASTConverter.java | 24 +-
.../calcite/translator/HiveOpConverter.java | 8 +-
.../translator/PlanModifierForASTConv.java | 10 +-
.../translator/PlanModifierForReturnPath.java | 4 -
.../calcite/translator/PlanModifierUtil.java | 4 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 17 +-
.../apache/hadoop/hive/ql/parse/HiveParser.g | 2 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
.../sqlstd/SQLStdHiveAccessController.java | 5 +
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 84 +-
ql/src/main/resources/hive-exec-log4j2.xml | 5 +-
ql/src/main/resources/tez-container-log4j2.xml | 5 +-
.../serde/TestParquetTimestampUtils.java | 38 +-
...nMapRedUtilsUsePartitionColumnsNegative.java | 73 +
...nMapRedUtilsUsePartitionColumnsPositive.java | 61 +
.../authorization_set_show_current_role.q | 3 +
.../clientpositive/bucket_map_join_tez1.q | 31 +
.../queries/clientpositive/cbo_rp_auto_join17.q | 14 +
.../cbo_rp_cross_product_check_2.q | 31 +
.../test/queries/clientpositive/dynpart_merge.q | 28 +
.../parquet_mixed_partition_formats.q | 42 +
.../clientpositive/parquet_ppd_boolean.q | 42 +-
.../queries/clientpositive/parquet_ppd_char.q | 46 +-
.../queries/clientpositive/parquet_ppd_date.q | 64 +-
.../clientpositive/parquet_ppd_decimal.q | 106 +-
.../clientpositive/parquet_ppd_timestamp.q | 62 +-
.../clientpositive/parquet_ppd_varchar.q | 46 +-
.../clientpositive/parquet_predicate_pushdown.q | 20 +-
.../authorization_explain.q.java1.7.out | 2 +-
.../authorization_explain.q.java1.8.out | 2 +-
.../authorization_set_show_current_role.q.out | 8 +
.../clientpositive/cbo_rp_auto_join17.q.out | 118 +
.../cbo_rp_cross_product_check_2.q.out | 699 ++++
.../results/clientpositive/dynpart_merge.q.out | 99 +
.../clientpositive/explain_dependency.q.out | 18 +-
.../clientpositive/explain_dependency2.q.out | 16 +-
.../results/clientpositive/input4.q.java1.7.out | 2 +-
.../results/clientpositive/input4.q.java1.8.out | 2 +-
.../results/clientpositive/join0.q.java1.7.out | 2 +-
.../results/clientpositive/join0.q.java1.8.out | 4 +-
.../list_bucket_dml_6.q.java1.7.out | 12 +-
.../list_bucket_dml_6.q.java1.8.out | 12 +-
.../clientpositive/list_bucket_dml_7.q.out | 12 +-
.../results/clientpositive/parallel_join0.q.out | 2 +-
.../parquet_mixed_partition_formats.q.out | 303 ++
.../clientpositive/parquet_ppd_boolean.q.out | 194 +-
.../clientpositive/parquet_ppd_char.q.out | 224 +-
.../clientpositive/parquet_ppd_date.q.out | 324 +-
.../clientpositive/parquet_ppd_decimal.q.out | 594 +++-
.../clientpositive/parquet_ppd_timestamp.q.out | 314 +-
.../clientpositive/parquet_ppd_varchar.q.out | 224 +-
.../parquet_predicate_pushdown.q.out | 118 +-
.../clientpositive/plan_json.q.java1.7.out | 2 +-
.../clientpositive/plan_json.q.java1.8.out | 2 +-
.../spark/bucket_map_join_tez1.q.out | 357 ++
.../tez/bucket_map_join_tez1.q.out | 333 ++
.../clientpositive/tez/constprog_dpp.q.out | 4 +-
.../clientpositive/tez/explainuser_1.q.out | 496 +--
.../clientpositive/tez/explainuser_2.q.out | 3280 +++++++++---------
.../clientpositive/tez/explainuser_3.q.out | 10 +-
.../apache/hadoop/hive/serde2/WriteBuffers.java | 10 +-
.../thrift/RetryingThriftCLIServiceClient.java | 331 ++
.../cli/TestRetryingThriftCLIServiceClient.java | 133 +
testutils/ptest2/src/main/resources/log4j2.xml | 5 +-
104 files changed, 6795 insertions(+), 3218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/common/src/main/resources/hive-log4j2.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
index 540085a,92e5446..12da0f8
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
@@@ -364,27 -360,21 +365,33 @@@ public abstract class Operator<T extend
// derived classes can set this to different object if needed
outputObjInspector = inputObjInspectors[0];
- Collection<Future<?>> asyncInitOperations = initializeOp(hconf);
+ boolean isInitOk = false;
+ try {
+ initializeOp(hconf);
+ // sanity checks
+ if (!rootInitializeCalled
+ || childOperatorsArray.length != childOperators.size()) {
+ throw new AssertionError("Internal error during operator initialization");
+ }
+ if (isLogInfoEnabled) {
+ LOG.info("Initialization Done " + id + " " + getName());
+ }
- // sanity checks
- if (!rootInitializeCalled
- || asyncInitOperations == null
- || childOperatorsArray.length != childOperators.size()) {
- throw new AssertionError("Internal error during operator initialization");
+ initializeChildren(hconf);
+ isInitOk = true;
+ } finally {
+ // TODO: ugly hack because Java doesn't have dtors and Tez input hangs on shutdown.
+ if (!isInitOk) {
+ cancelAsyncInitOps();
+ }
}
+ if (isLogInfoEnabled) {
+ LOG.info("Initialization Done " + id + " " + getName() + " done is reset.");
+ }
+
+ initializeChildren(hconf);
+
// let's wait on the async ops before continuing
completeInitialization(asyncInitOperations);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/InPlaceUpdates.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/tez/InPlaceUpdates.java
index 0000000,6ecfe71..1c06692
mode 000000,100644..100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/InPlaceUpdates.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/InPlaceUpdates.java
@@@ -1,0 -1,65 +1,65 @@@
+ package org.apache.hadoop.hive.ql.exec.tez;
+
+ import static org.fusesource.jansi.Ansi.ansi;
+ import static org.fusesource.jansi.internal.CLibrary.STDERR_FILENO;
+ import static org.fusesource.jansi.internal.CLibrary.STDOUT_FILENO;
+ import static org.fusesource.jansi.internal.CLibrary.isatty;
+
+ import java.io.PrintStream;
+
-import jline.TerminalFactory;
-
+ import org.apache.hadoop.hive.conf.HiveConf;
+ import org.apache.hadoop.hive.ql.session.SessionState;
+ import org.fusesource.jansi.Ansi;
+
++import jline.TerminalFactory;
++
+ public class InPlaceUpdates {
+
- private static final int MIN_TERMINAL_WIDTH = 80;
++ public static final int MIN_TERMINAL_WIDTH = 80;
+
+ static boolean isUnixTerminal() {
+
+ String os = System.getProperty("os.name");
+ if (os.startsWith("Windows")) {
+ // we do not support Windows, we will revisit this if we really need it for windows.
+ return false;
+ }
+
+ // We must be on some unix variant..
+ // check if standard out is a terminal
+ try {
+ // isatty system call will return 1 if the file descriptor is terminal else 0
+ if (isatty(STDOUT_FILENO) == 0) {
+ return false;
+ }
+ if (isatty(STDERR_FILENO) == 0) {
+ return false;
+ }
+ } catch (NoClassDefFoundError ignore) {
+ // These errors happen if the JNI lib is not available for your platform.
+ return false;
+ } catch (UnsatisfiedLinkError ignore) {
+ // These errors happen if the JNI lib is not available for your platform.
+ return false;
+ }
+ return true;
+ }
+
+ public static boolean inPlaceEligible(HiveConf conf) {
+ boolean inPlaceUpdates = HiveConf.getBoolVar(conf, HiveConf.ConfVars.TEZ_EXEC_INPLACE_PROGRESS);
+
+ // we need at least 80 chars wide terminal to display in-place updates properly
+ return inPlaceUpdates && !SessionState.getConsole().getIsSilent() && isUnixTerminal()
+ && TerminalFactory.get().getWidth() >= MIN_TERMINAL_WIDTH;
+ }
+
+ public static void reprintLine(PrintStream out, String line) {
+ out.print(ansi().eraseLine(Ansi.Erase.ALL).a(line).a('\n').toString());
+ out.flush();
+ }
+
+ public static void rePositionCursor(PrintStream ps) {
+ ps.print(ansi().cursorUp(0).toString());
+ ps.flush();
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
index 28c7d20,1e1603b..1dfa092
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java
@@@ -20,25 -20,7 +20,22 @@@ package org.apache.hadoop.hive.ql.exec.
import static org.apache.tez.dag.api.client.DAGStatus.State.RUNNING;
import static org.fusesource.jansi.Ansi.ansi;
- import static org.fusesource.jansi.internal.CLibrary.STDERR_FILENO;
- import static org.fusesource.jansi.internal.CLibrary.STDOUT_FILENO;
- import static org.fusesource.jansi.internal.CLibrary.isatty;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.text.DecimalFormat;
+import java.text.NumberFormat;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.Heartbeater;
@@@ -65,8 -46,23 +62,6 @@@ import org.fusesource.jansi.Ansi
import com.google.common.base.Preconditions;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.text.DecimalFormat;
-import java.text.NumberFormat;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
--import jline.TerminalFactory;
--
/**
* TezJobMonitor keeps track of a tez job while it's being executed. It will
* print status to the console and retrieve final status of the job after
@@@ -75,9 -71,9 +70,9 @@@
public class TezJobMonitor {
private static final String CLASS_NAME = TezJobMonitor.class.getName();
- private static final int MIN_TERMINAL_WIDTH = 94;
+
private static final int COLUMN_1_WIDTH = 16;
- private static final int SEPARATOR_WIDTH = MIN_TERMINAL_WIDTH;
- private static final int SEPARATOR_WIDTH = 80;
++ private static final int SEPARATOR_WIDTH = InPlaceUpdates.MIN_TERMINAL_WIDTH;
// keep this within 80 chars width. If more columns needs to be added then update min terminal
// width requirement and separator width accordingly
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/main/resources/hive-exec-log4j2.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/ql/src/test/results/clientpositive/list_bucket_dml_6.q.java1.8.out
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b0154f4c/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
----------------------------------------------------------------------