You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/05/06 17:25:11 UTC
[21/39] hive git commit: Merge branch 'master' into llap
Merge branch 'master' into llap
Conflicts:
llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/99cb7f96
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/99cb7f96
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/99cb7f96
Branch: refs/heads/master
Commit: 99cb7f96f1a516698b587320e841d26cd7bb034d
Parents: 0afaa8f 2d28291
Author: Jason Dere <jd...@hortonworks.com>
Authored: Tue Apr 19 15:24:19 2016 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Tue Apr 19 15:24:19 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hive/beeline/BeeLine.java | 22 +-
.../hive/beeline/BeeLineCommandCompleter.java | 26 +-
.../beeline/SeparatedValuesOutputFormat.java | 2 +-
bin/ext/cleardanglingscratchdir.cmd | 35 +
bin/ext/cleardanglingscratchdir.sh | 28 +
bin/ext/llapstatus.sh | 42 +
.../apache/hadoop/hive/common/FileUtils.java | 2 +-
.../hadoop/hive/common/type/HiveBaseChar.java | 6 +
.../hadoop/hive/common/type/HiveVarchar.java | 4 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 11 +-
.../org/apache/hadoop/hive/conf/Validator.java | 2 +-
.../apache/hadoop/hive/ql/log/PerfLogger.java | 51 +-
.../apache/hive/common/util/StreamPrinter.java | 18 +-
.../java/org/apache/hive/http/HttpServer.java | 29 +-
.../hive/common/type/TestHiveBaseChar.java | 2 +
.../hadoop/hive/hbase/HBaseSerDeParameters.java | 6 +-
.../hive/hbase/HiveHBaseInputFormatUtil.java | 6 +-
.../test/results/positive/hbase_pushdown.q.out | 2 +-
.../test/results/positive/ppd_key_ranges.q.out | 2 +-
.../mapreduce/FileOutputCommitterContainer.java | 4 +-
.../mapreduce/FosterStorageHandler.java | 6 +-
.../streaming/StreamingIntegrationTester.java | 12 +-
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 15 +-
.../hive/metastore/TestMetaStoreMetrics.java | 13 +-
.../TestHiveAuthorizerCheckInvocation.java | 2 +-
.../plugin/TestHiveAuthorizerShowFilters.java | 4 +-
.../ql/session/TestClearDanglingScratchDir.java | 131 +
.../org/apache/hive/jdbc/TestJdbcDriver2.java | 78 +
.../apache/hive/jdbc/TestJdbcWithMiniHA.java | 200 ++
.../jdbc/authorization/TestHS2AuthzContext.java | 14 +-
.../authorization/TestJdbcMetadataApiAuth.java | 4 +-
.../hive/jdbc/miniHS2/StartMiniHS2Cluster.java | 2 +-
.../hive/jdbc/miniHS2/TestHs2Metrics.java | 41 +-
.../service/cli/session/TestQueryDisplay.java | 2 +
.../cli/thrift/TestThriftBinaryCLIService.java | 92 -
.../cli/thrift/TestThriftHttpCLIService.java | 241 --
.../thrift/ThriftCliServiceMessageSizeTest.java | 140 +
itests/qtest/pom.xml | 2 +-
.../hive/ql/security/DummyAuthenticator.java | 5 +
.../security/InjectableDummyAuthenticator.java | 5 +
...SQLStdHiveAuthorizationValidatorForTest.java | 47 +-
.../org/apache/hive/jdbc/HiveBaseResultSet.java | 10 +-
.../org/apache/hive/jdbc/HiveConnection.java | 84 +-
.../org/apache/hive/jdbc/HiveStatement.java | 18 +
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 20 +-
.../hive/llap/registry/ServiceInstance.java | 6 +
.../hive/llap/registry/ServiceRegistry.java | 4 +-
.../registry/impl/LlapFixedRegistryImpl.java | 29 +-
.../llap/registry/impl/LlapRegistryService.java | 17 +-
.../impl/LlapZookeeperRegistryImpl.java | 29 +-
llap-server/pom.xml | 55 +
llap-server/sql/serviceCheckScript.sql | 12 +
.../hive/llap/cli/LlapOptionsProcessor.java | 23 +-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 8 +
.../llap/cli/LlapStatusOptionsProcessor.java | 139 +
.../hive/llap/cli/LlapStatusServiceDriver.java | 821 ++++++
.../hive/llap/daemon/impl/LlapDaemon.java | 17 +-
.../daemon/services/impl/LlapWebServices.java | 176 +-
.../llap/io/decode/OrcEncodedDataConsumer.java | 45 +-
.../llap/metrics/LlapDaemonCacheMetrics.java | 4 +-
.../llap/metrics/LlapDaemonExecutorMetrics.java | 4 +-
.../llap/metrics/LlapDaemonQueueMetrics.java | 4 +-
.../hadoop/hive/llap/metrics/MetricsUtils.java | 1 -
.../hive/llap/security/SecretManager.java | 18 +-
.../llap/shufflehandler/ShuffleHandler.java | 2 +-
.../hadoop-metrics2.properties.template | 50 +
.../main/resources/llap-cli-log4j2.properties | 9 +-
.../hive/llap/daemon/MiniLlapCluster.java | 5 +-
.../hive/metastore/MetaStoreSchemaInfo.java | 4 +-
.../hadoop/hive/metastore/MetaStoreUtils.java | 28 +-
.../hive/metastore/hbase/HBaseImport.java | 4 +-
.../hive/metastore/hbase/HBaseReadWrite.java | 10 +-
orc/src/java/org/apache/orc/OrcUtils.java | 75 +
orc/src/java/org/apache/orc/Reader.java | 6 +
orc/src/java/org/apache/orc/RecordReader.java | 8 +-
.../java/org/apache/orc/TypeDescription.java | 62 +-
.../org/apache/orc/impl/BitFieldReader.java | 5 +-
.../java/org/apache/orc/impl/IntegerReader.java | 26 +-
.../apache/orc/impl/RunLengthByteReader.java | 36 +-
.../apache/orc/impl/RunLengthIntegerReader.java | 31 +-
.../orc/impl/RunLengthIntegerReaderV2.java | 33 +-
.../java/org/apache/orc/impl/WriterImpl.java | 47 +-
packaging/src/main/assembly/bin.xml | 9 +
pom.xml | 1 +
.../java/org/apache/hadoop/hive/ql/Driver.java | 22 +-
.../hive/ql/exec/AbstractFileMergeOperator.java | 9 +
.../hive/ql/exec/AppMasterEventOperator.java | 4 +-
.../hadoop/hive/ql/exec/CollectOperator.java | 11 +-
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 8 +-
.../hadoop/hive/ql/exec/DemuxOperator.java | 8 +-
.../hadoop/hive/ql/exec/DummyStoreOperator.java | 11 +-
.../hadoop/hive/ql/exec/FileSinkOperator.java | 4 +-
.../hadoop/hive/ql/exec/FilterOperator.java | 4 +-
.../hadoop/hive/ql/exec/ForwardOperator.java | 4 +-
.../hadoop/hive/ql/exec/GroupByOperator.java | 4 +-
.../hive/ql/exec/HashTableDummyOperator.java | 4 +-
.../ql/exec/LateralViewForwardOperator.java | 5 +-
.../hive/ql/exec/LateralViewJoinOperator.java | 4 +-
.../hadoop/hive/ql/exec/LimitOperator.java | 4 +-
.../hadoop/hive/ql/exec/ListSinkOperator.java | 11 +-
.../apache/hadoop/hive/ql/exec/MapOperator.java | 9 +-
.../apache/hadoop/hive/ql/exec/MoveTask.java | 1 +
.../apache/hadoop/hive/ql/exec/MuxOperator.java | 8 +-
.../apache/hadoop/hive/ql/exec/Operator.java | 22 +-
.../apache/hadoop/hive/ql/exec/PTFOperator.java | 4 +-
.../hadoop/hive/ql/exec/ScriptOperator.java | 40 +-
.../hadoop/hive/ql/exec/SelectOperator.java | 4 +-
.../ql/exec/SparkHashTableSinkOperator.java | 10 +-
.../hadoop/hive/ql/exec/TableScanOperator.java | 2 +-
.../apache/hadoop/hive/ql/exec/TaskRunner.java | 3 +-
.../hadoop/hive/ql/exec/TerminalOperator.java | 10 +
.../hadoop/hive/ql/exec/UDTFOperator.java | 8 +-
.../hadoop/hive/ql/exec/UnionOperator.java | 2 +-
.../hadoop/hive/ql/exec/mr/MapRedTask.java | 5 +-
.../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 9 +-
.../vector/VectorAppMasterEventOperator.java | 11 +-
.../ql/exec/vector/VectorFileSinkOperator.java | 8 +-
.../ql/exec/vector/VectorFilterOperator.java | 8 +-
.../ql/exec/vector/VectorGroupByOperator.java | 25 +-
.../ql/exec/vector/VectorLimitOperator.java | 5 +-
.../ql/exec/vector/VectorMapJoinOperator.java | 11 +-
.../VectorMapJoinOuterFilteredOperator.java | 8 +-
.../hive/ql/exec/vector/VectorMapOperator.java | 5 +-
.../exec/vector/VectorReduceSinkOperator.java | 8 +-
.../exec/vector/VectorSMBMapJoinOperator.java | 13 +-
.../ql/exec/vector/VectorSelectOperator.java | 21 +-
.../VectorSparkHashTableSinkOperator.java | 6 +-
...VectorSparkPartitionPruningSinkOperator.java | 8 +-
.../ql/exec/vector/VectorizationContext.java | 11 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 13 +-
.../hadoop/hive/ql/hooks/HookContext.java | 11 +-
.../hive/ql/index/IndexPredicateAnalyzer.java | 35 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 43 +-
.../hive/ql/io/orc/OrcRawRecordMerger.java | 7 +-
.../hadoop/hive/ql/io/orc/ReaderImpl.java | 12 +-
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 50 +-
.../hadoop/hive/ql/io/orc/SchemaEvolution.java | 234 +-
.../hive/ql/io/orc/TreeReaderFactory.java | 858 +++---
.../ql/io/orc/VectorizedOrcInputFormat.java | 32 +-
.../hadoop/hive/ql/io/orc/WriterImpl.java | 2 -
.../ql/io/parquet/convert/ETypeConverter.java | 3 +-
.../io/parquet/convert/HiveStructConverter.java | 2 +-
.../write/ParquetRecordWriterWrapper.java | 4 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../optimizer/ConstantPropagateProcFactory.java | 22 +-
.../hadoop/hive/ql/optimizer/IndexUtils.java | 2 +
.../SizeBasedBigTableSelectorForAutoSMJ.java | 2 +-
.../hive/ql/optimizer/StatsOptimizer.java | 37 +-
.../calcite/cost/HiveAlgorithmsUtil.java | 12 +-
.../hive/ql/optimizer/physical/LlapDecider.java | 12 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 2 +
.../stats/annotation/StatsRulesProcFactory.java | 107 +-
.../apache/hadoop/hive/ql/parse/ASTNode.java | 2 +-
.../hive/ql/parse/BaseSemanticAnalyzer.java | 4 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 4 +-
.../hadoop/hive/ql/parse/MaskAndFilterInfo.java | 38 +
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 8 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 3 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 53 +-
.../apache/hadoop/hive/ql/parse/TableMask.java | 85 +-
.../hadoop/hive/ql/parse/TableSample.java | 4 +-
.../hive/ql/parse/TypeCheckProcFactory.java | 84 +-
.../hive/ql/parse/spark/GenSparkUtils.java | 2 +-
.../SparkPartitionPruningSinkOperator.java | 16 +-
.../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 14 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 2 +-
.../hadoop/hive/ql/processors/CommandUtil.java | 6 +-
.../ql/security/HadoopDefaultAuthenticator.java | 5 +
.../ql/security/HiveAuthenticationProvider.java | 2 +
.../SessionStateConfigUserAuthenticator.java | 5 +
.../security/SessionStateUserAuthenticator.java | 5 +
.../AuthorizationMetaStoreFilterHook.java | 4 +-
.../plugin/HiveAuthorizationValidator.java | 12 +-
.../authorization/plugin/HiveAuthorizer.java | 68 +-
.../plugin/HiveAuthorizerImpl.java | 20 +-
.../authorization/plugin/HiveAuthzContext.java | 83 -
.../plugin/HivePrivilegeObject.java | 31 +
.../authorization/plugin/HiveV1Authorizer.java | 20 +-
.../authorization/plugin/QueryContext.java | 78 +
.../sqlstd/DummyHiveAuthorizationValidator.java | 21 +-
.../SQLStdHiveAuthorizationValidator.java | 22 +-
.../ql/session/ClearDanglingScratchDir.java | 176 ++
.../hadoop/hive/ql/session/OperationLog.java | 10 +-
.../hadoop/hive/ql/session/SessionState.java | 43 +-
.../hive/ql/stats/fs/FSStatsAggregator.java | 2 +-
.../hive/ql/stats/fs/FSStatsPublisher.java | 3 +-
.../hive/ql/txn/compactor/CompactorMR.java | 12 +-
.../org/apache/hadoop/hive/ql/udf/UDFSign.java | 15 +
.../apache/hadoop/hive/ql/udf/UDFToByte.java | 7 +-
.../apache/hadoop/hive/ql/udf/UDFToDouble.java | 6 +-
.../apache/hadoop/hive/ql/udf/UDFToFloat.java | 6 +-
.../apache/hadoop/hive/ql/udf/UDFToInteger.java | 4 +
.../apache/hadoop/hive/ql/udf/UDFToLong.java | 4 +
.../apache/hadoop/hive/ql/udf/UDFToShort.java | 4 +
.../ql/udf/generic/GenericUDAFComputeStats.java | 6 +
.../hive/ql/udf/generic/GenericUDFBetween.java | 2 +-
.../hive/ql/exec/TestFunctionRegistry.java | 26 +-
.../hadoop/hive/ql/exec/TestOperatorNames.java | 98 +
.../vector/util/FakeCaptureOutputOperator.java | 10 +-
.../util/FakeVectorDataSourceOperator.java | 11 +-
.../hive/ql/io/orc/TestTypeDescription.java | 4 +-
.../hive/ql/io/orc/TestVectorOrcFile.java | 1634 ++++++------
.../hive/ql/io/orc/TestVectorizedORCReader.java | 7 +-
.../hive/ql/metadata/TestTableIterable.java | 67 +
.../hive/ql/parse/TestSemanticAnalyzer.java | 37 +
.../ql/udf/generic/TestGenericUDFOPMinus.java | 4 +-
.../udf/generic/TestGenericUDFOPMultiply.java | 4 +-
.../ql/udf/generic/TestGenericUDFOPPlus.java | 4 +-
.../clientpositive/alter_partition_change_col.q | 1 +
.../clientpositive/alter_table_cascade.q | 1 +
.../clientpositive/cbo_rp_udf_udaf_stats_opt.q | 22 +
.../queries/clientpositive/float_equality.q | 3 +
ql/src/test/queries/clientpositive/foldts.q | 20 +
.../test/queries/clientpositive/type_widening.q | 6 +
.../queries/clientpositive/vector_between_in.q | 30 +
.../vector_orc_string_reader_empty_dict.q | 20 +
.../queries/clientpositive/windowing_gby2.q | 41 +
.../results/clientnegative/dyn_part_max.q.out | 2 +-
.../clientpositive/annotate_stats_filter.q.out | 2 +-
.../results/clientpositive/auto_join16.q.out | 4 +-
.../results/clientpositive/auto_join4.q.out | 2 +-
.../results/clientpositive/auto_join5.q.out | 2 +-
.../results/clientpositive/auto_join8.q.out | 2 +-
.../auto_join_reordering_values.q.out | 2 +-
.../clientpositive/auto_sortmerge_join_8.q.out | 2 +
.../test/results/clientpositive/cbo_const.q.out | 2 +-
.../results/clientpositive/cbo_rp_join1.q.out | 4 +-
.../cbo_rp_udf_udaf_stats_opt.q.out | 126 +
.../clientpositive/constprog_semijoin.q.out | 24 +-
.../clientpositive/correlationoptimizer13.q.out | 2 +-
.../clientpositive/correlationoptimizer9.q.out | 4 +-
.../clientpositive/dynamic_rdd_cache.q.out | 8 +-
.../dynpart_sort_optimization.q.out | 2 +-
.../clientpositive/explain_logical.q.out | 6 +-
.../clientpositive/filter_cond_pushdown.q.out | 8 +-
.../results/clientpositive/float_equality.q.out | 17 +
ql/src/test/results/clientpositive/foldts.q.out | 154 ++
.../groupby_multi_single_reducer3.q.out | 8 +-
.../clientpositive/groupby_position.q.out | 4 +-
.../identity_project_remove_skip.q.out | 2 +-
.../clientpositive/index_auto_mult_tables.q.out | 20 +-
.../index_auto_mult_tables_compact.q.out | 20 +-
.../clientpositive/index_auto_self_join.q.out | 20 +-
.../results/clientpositive/index_bitmap3.q.out | 4 +-
.../clientpositive/index_bitmap_auto.q.out | 4 +-
.../index_bitmap_compression.q.out | 4 +-
.../clientpositive/infer_const_type.q.out | 10 +-
.../clientpositive/input_testxpath4.q.out | 2 +-
ql/src/test/results/clientpositive/join16.q.out | 4 +-
ql/src/test/results/clientpositive/join19.q.out | 8 +-
ql/src/test/results/clientpositive/join4.q.out | 2 +-
ql/src/test/results/clientpositive/join42.q.out | 2 +-
ql/src/test/results/clientpositive/join5.q.out | 2 +-
ql/src/test/results/clientpositive/join8.q.out | 2 +-
.../clientpositive/join_grp_diff_keys.q.out | 2 +-
.../results/clientpositive/join_reorder2.q.out | 2 +-
.../results/clientpositive/join_reorder3.q.out | 2 +-
.../llap/dynamic_partition_pruning.q.out | 20 +-
.../llap/dynamic_partition_pruning_2.q.out | 30 +-
.../llap/hybridgrace_hashjoin_2.q.out | 12 +-
.../clientpositive/llap/tez_join_hash.q.out | 4 +
.../llap/tez_union_group_by.q.out | 2 +-
.../vectorized_dynamic_partition_pruning.q.out | 20 +-
.../clientpositive/louter_join_ppr.q.out | 4 +-
.../test/results/clientpositive/masking_1.q.out | 12 +-
.../test/results/clientpositive/masking_2.q.out | 10 +-
.../test/results/clientpositive/masking_3.q.out | 2 +-
.../test/results/clientpositive/masking_4.q.out | 4 +-
.../test/results/clientpositive/masking_5.q.out | 2 +-
.../clientpositive/masking_disablecbo_1.q.out | 14 +-
.../clientpositive/masking_disablecbo_2.q.out | 10 +-
.../clientpositive/masking_disablecbo_3.q.out | 2 +-
.../clientpositive/masking_disablecbo_4.q.out | 4 +-
.../results/clientpositive/multiMapJoin1.q.out | 16 +-
.../clientpositive/orc_predicate_pushdown.q.out | 24 +-
.../parquet_predicate_pushdown.q.out | 28 +-
ql/src/test/results/clientpositive/pcs.q.out | 6 +-
.../results/clientpositive/perf/query13.q.out | 6 +-
.../results/clientpositive/perf/query15.q.out | 2 +-
.../results/clientpositive/perf/query17.q.out | 14 +-
.../results/clientpositive/perf/query18.q.out | 6 +-
.../results/clientpositive/perf/query19.q.out | 4 +-
.../results/clientpositive/perf/query21.q.out | 2 +-
.../results/clientpositive/perf/query22.q.out | 2 +-
.../results/clientpositive/perf/query25.q.out | 12 +-
.../results/clientpositive/perf/query26.q.out | 4 +-
.../results/clientpositive/perf/query27.q.out | 4 +-
.../results/clientpositive/perf/query29.q.out | 18 +-
.../results/clientpositive/perf/query31.q.out | 12 +-
.../results/clientpositive/perf/query32.q.out | 2 +-
.../results/clientpositive/perf/query34.q.out | 6 +-
.../results/clientpositive/perf/query39.q.out | 8 +-
.../results/clientpositive/perf/query40.q.out | 2 +-
.../results/clientpositive/perf/query42.q.out | 2 +-
.../results/clientpositive/perf/query45.q.out | 4 +-
.../results/clientpositive/perf/query46.q.out | 14 +-
.../results/clientpositive/perf/query48.q.out | 6 +-
.../results/clientpositive/perf/query50.q.out | 6 +-
.../results/clientpositive/perf/query52.q.out | 2 +-
.../results/clientpositive/perf/query54.q.out | 10 +-
.../results/clientpositive/perf/query55.q.out | 2 +-
.../results/clientpositive/perf/query64.q.out | 16 +-
.../results/clientpositive/perf/query65.q.out | 4 +-
.../results/clientpositive/perf/query66.q.out | 4 +-
.../results/clientpositive/perf/query67.q.out | 2 +-
.../results/clientpositive/perf/query68.q.out | 4 +-
.../results/clientpositive/perf/query7.q.out | 4 +-
.../results/clientpositive/perf/query71.q.out | 12 +-
.../results/clientpositive/perf/query72.q.out | 6 +-
.../results/clientpositive/perf/query73.q.out | 6 +-
.../results/clientpositive/perf/query75.q.out | 14 +-
.../results/clientpositive/perf/query76.q.out | 6 +-
.../results/clientpositive/perf/query79.q.out | 4 +-
.../results/clientpositive/perf/query80.q.out | 6 +-
.../results/clientpositive/perf/query82.q.out | 4 +-
.../results/clientpositive/perf/query84.q.out | 4 +-
.../results/clientpositive/perf/query85.q.out | 10 +-
.../results/clientpositive/perf/query88.q.out | 32 +-
.../results/clientpositive/perf/query89.q.out | 16 +-
.../results/clientpositive/perf/query90.q.out | 4 +-
.../results/clientpositive/perf/query91.q.out | 8 +-
.../results/clientpositive/perf/query92.q.out | 4 +-
.../results/clientpositive/perf/query93.q.out | 2 +-
.../results/clientpositive/perf/query94.q.out | 2 +-
.../results/clientpositive/perf/query95.q.out | 2 +-
.../results/clientpositive/perf/query96.q.out | 4 +-
.../results/clientpositive/pointlookup.q.out | 12 +-
.../results/clientpositive/pointlookup2.q.out | 16 +-
.../results/clientpositive/pointlookup3.q.out | 8 +-
.../results/clientpositive/ppd_gby_join.q.out | 8 +-
.../test/results/clientpositive/ppd_join.q.out | 8 +-
.../test/results/clientpositive/ppd_join2.q.out | 12 +-
.../test/results/clientpositive/ppd_join3.q.out | 12 +-
.../test/results/clientpositive/ppd_join4.q.out | 2 +-
.../clientpositive/ppd_outer_join2.q.out | 8 +-
.../clientpositive/ppd_outer_join3.q.out | 8 +-
.../clientpositive/ppd_outer_join4.q.out | 12 +-
.../results/clientpositive/ppd_udf_case.q.out | 4 +-
.../test/results/clientpositive/ppd_union.q.out | 8 +-
ql/src/test/results/clientpositive/ppd_vc.q.out | 2 +-
.../clientpositive/rcfile_null_value.q.out | 2 +-
.../clientpositive/router_join_ppr.q.out | 4 +-
.../test/results/clientpositive/sample8.q.out | 4 +-
.../test/results/clientpositive/semijoin.q.out | 2 +-
.../test/results/clientpositive/semijoin2.q.out | 4 +-
.../test/results/clientpositive/semijoin4.q.out | 4 +-
.../clientpositive/skewjoin_mapjoin9.q.out | 4 +-
.../results/clientpositive/skewjoinopt12.q.out | 8 +-
.../results/clientpositive/skewjoinopt14.q.out | 4 +-
.../results/clientpositive/skewjoinopt16.q.out | 8 +-
.../results/clientpositive/skewjoinopt17.q.out | 8 +-
.../results/clientpositive/skewjoinopt2.q.out | 16 +-
.../results/clientpositive/smb_mapjoin_10.q.out | 2 +-
.../results/clientpositive/smb_mapjoin_14.q.out | 2 +-
.../clientpositive/sort_merge_join_desc_2.q.out | 2 +-
.../clientpositive/sort_merge_join_desc_3.q.out | 2 +-
.../clientpositive/sort_merge_join_desc_4.q.out | 4 +-
.../clientpositive/sort_merge_join_desc_8.q.out | 4 +-
.../clientpositive/spark/auto_join16.q.out | 4 +-
.../clientpositive/spark/auto_join4.q.out | 2 +-
.../clientpositive/spark/auto_join5.q.out | 2 +-
.../clientpositive/spark/auto_join8.q.out | 2 +-
.../spark/auto_join_reordering_values.q.out | 2 +-
.../spark/constprog_semijoin.q.out | 24 +-
.../spark/dynamic_rdd_cache.q.out | 8 +-
.../spark/groupby_multi_single_reducer3.q.out | 8 +-
.../clientpositive/spark/groupby_position.q.out | 4 +-
.../spark/identity_project_remove_skip.q.out | 2 +-
.../spark/index_auto_self_join.q.out | 12 +-
.../clientpositive/spark/index_bitmap3.q.out | 4 +-
.../spark/index_bitmap_auto.q.out | 4 +-
.../results/clientpositive/spark/join16.q.out | 4 +-
.../results/clientpositive/spark/join19.q.out | 8 +-
.../results/clientpositive/spark/join4.q.out | 2 +-
.../results/clientpositive/spark/join5.q.out | 2 +-
.../results/clientpositive/spark/join8.q.out | 2 +-
.../clientpositive/spark/join_reorder2.q.out | 2 +-
.../clientpositive/spark/join_reorder3.q.out | 2 +-
.../clientpositive/spark/louter_join_ppr.q.out | 4 +-
.../clientpositive/spark/ppd_gby_join.q.out | 8 +-
.../results/clientpositive/spark/ppd_join.q.out | 8 +-
.../clientpositive/spark/ppd_join2.q.out | 12 +-
.../clientpositive/spark/ppd_join3.q.out | 12 +-
.../clientpositive/spark/ppd_outer_join2.q.out | 8 +-
.../clientpositive/spark/ppd_outer_join3.q.out | 8 +-
.../clientpositive/spark/ppd_outer_join4.q.out | 12 +-
.../clientpositive/spark/router_join_ppr.q.out | 4 +-
.../results/clientpositive/spark/sample8.q.out | 4 +-
.../results/clientpositive/spark/semijoin.q.out | 2 +-
.../clientpositive/spark/skewjoinopt12.q.out | 8 +-
.../clientpositive/spark/skewjoinopt14.q.out | 4 +-
.../clientpositive/spark/skewjoinopt16.q.out | 8 +-
.../clientpositive/spark/skewjoinopt17.q.out | 8 +-
.../clientpositive/spark/skewjoinopt2.q.out | 16 +-
.../clientpositive/spark/smb_mapjoin_10.q.out | 2 +-
.../clientpositive/spark/smb_mapjoin_14.q.out | 2 +-
.../spark/sort_merge_join_desc_2.q.out | 22 +-
.../spark/sort_merge_join_desc_3.q.out | 22 +-
.../spark/sort_merge_join_desc_4.q.out | 26 +-
.../spark/sort_merge_join_desc_8.q.out | 42 +-
.../spark/spark_dynamic_partition_pruning.q.out | 2313 +++++++++-------
...k_vectorized_dynamic_partition_pruning.q.out | 2515 +++++++++++-------
.../clientpositive/spark/subquery_in.q.out | 2 +-
.../spark/vector_between_in.q.out | 332 +++
.../spark/vector_mapjoin_reduce.q.out | 6 +-
.../clientpositive/spark/vectorization_0.q.out | 2 +-
.../clientpositive/spark/vectorization_14.q.out | 2 +-
.../clientpositive/spark/vectorization_17.q.out | 2 +-
.../spark/vectorization_short_regress.q.out | 2 +-
.../spark/vectorized_string_funcs.q.out | 2 +-
.../results/clientpositive/subquery_in.q.out | 2 +-
.../clientpositive/tez/bucketpruning1.q.out | 32 +-
.../clientpositive/tez/constprog_semijoin.q.out | 24 +-
.../tez/dynamic_partition_pruning.q.out | 20 +-
.../tez/dynamic_partition_pruning_2.q.out | 30 +-
.../tez/dynpart_sort_optimization.q.out | 2 +-
.../clientpositive/tez/explainuser_1.q.out | 38 +-
.../clientpositive/tez/explainuser_2.q.out | 12 +-
.../clientpositive/tez/explainuser_3.q.out | 8 +-
.../tez/hybridgrace_hashjoin_2.q.out | 12 +-
.../clientpositive/tez/subquery_in.q.out | 2 +-
.../clientpositive/tez/tez_join_hash.q.out | 4 +
.../clientpositive/tez/tez_union_group_by.q.out | 2 +-
.../tez/vector_aggregate_without_gby.q.out | 6 +-
.../tez/vector_auto_smb_mapjoin_14.q.out | 38 +-
.../clientpositive/tez/vector_between_in.q.out | 336 +++
.../clientpositive/tez/vector_date_1.q.out | 4 +-
.../tez/vector_decimal_cast.q.out | 2 +-
.../tez/vector_decimal_expressions.q.out | 2 +-
.../tez/vector_groupby_mapjoin.q.out | 4 +-
.../clientpositive/tez/vector_interval_2.q.out | 12 +-
.../tez/vector_join_part_col_char.q.out | 10 +-
.../tez/vector_leftsemi_mapjoin.q.out | 12 +-
.../tez/vector_mapjoin_reduce.q.out | 6 +-
.../clientpositive/tez/vectorization_0.q.out | 2 +-
.../clientpositive/tez/vectorization_14.q.out | 2 +-
.../clientpositive/tez/vectorization_17.q.out | 2 +-
.../clientpositive/tez/vectorization_7.q.out | 4 +-
.../tez/vectorization_short_regress.q.out | 2 +-
.../vectorized_dynamic_partition_pruning.q.out | 20 +-
.../tez/vectorized_parquet_types.q.out | 2 +-
.../tez/vectorized_string_funcs.q.out | 2 +-
.../tez/vectorized_timestamp.q.out | 8 +-
.../results/clientpositive/type_widening.q.out | 112 +
.../results/clientpositive/udf_greatest.q.out | 4 +-
.../test/results/clientpositive/udf_least.q.out | 4 +-
.../clientpositive/udf_to_unix_timestamp.q.out | 2 +-
.../clientpositive/vector_between_in.q.out | 304 +++
.../results/clientpositive/vector_date_1.q.out | 4 +-
.../clientpositive/vector_decimal_cast.q.out | 2 +-
.../vector_decimal_expressions.q.out | 2 +-
.../clientpositive/vector_interval_2.q.out | 12 +-
.../vector_leftsemi_mapjoin.q.out | 12 +-
.../clientpositive/vector_mapjoin_reduce.q.out | 6 +-
.../vector_orc_string_reader_empty_dict.q.out | 62 +
.../clientpositive/vectorization_0.q.out | 2 +-
.../clientpositive/vectorization_14.q.out | 2 +-
.../clientpositive/vectorization_17.q.out | 2 +-
.../clientpositive/vectorization_7.q.out | 4 +-
.../vectorization_short_regress.q.out | 2 +-
.../vectorized_string_funcs.q.out | 2 +-
.../results/clientpositive/windowing_gby2.q.out | 652 +++++
.../serde2/MetadataTypedColumnsetSerDe.java | 2 +-
.../serde2/dynamic_type/thrift_grammar.java | 2 +-
.../hive/serde2/lazy/LazySerDeParameters.java | 3 +-
.../hadoop/hive/serde2/lazy/LazyUtils.java | 2 +-
.../primitive/JavaHiveCharObjectInspector.java | 15 +-
.../JavaHiveVarcharObjectInspector.java | 15 +-
.../PrimitiveObjectInspectorConverter.java | 8 +-
.../serde2/thrift/TCTLSeparatedProtocol.java | 25 +-
.../hive/serde2/typeinfo/TypeInfoUtils.java | 17 +-
.../TestObjectInspectorConverters.java | 95 +
.../TestStandardObjectInspectors.java | 14 +-
.../org/apache/hive/tmpl/QueryProfileTmpl.jamon | 16 +-
.../hive/service/cli/HiveSQLException.java | 6 +-
.../cli/operation/MetadataOperation.java | 6 +-
.../hive/service/cli/operation/Operation.java | 2 +-
.../service/cli/operation/SQLOperation.java | 5 +
.../cli/operation/SQLOperationDisplay.java | 9 +
.../service/cli/session/HiveSessionImpl.java | 2 +
.../service/cli/session/SessionManager.java | 21 +-
.../cli/thrift/ThriftBinaryCLIService.java | 64 +-
.../service/cli/thrift/ThriftCLIService.java | 61 +-
.../service/cli/thrift/ThriftHttpServlet.java | 17 +
.../apache/hive/service/server/HiveServer2.java | 4 +-
.../hive-webapps/hiveserver2/hiveserver2.jsp | 22 +-
.../cli/TestRetryingThriftCLIServiceClient.java | 130 +-
.../cli/thrift/ThriftCLIServiceTest.java | 4 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 34 +-
.../apache/hadoop/hive/shims/HadoopShims.java | 6 +
.../hive/ql/exec/vector/BytesColumnVector.java | 11 +
.../ql/exec/vector/TimestampColumnVector.java | 2 +-
.../hive/ql/exec/vector/UnionColumnVector.java | 2 -
.../junit/runners/ConcurrentTestRunner.java | 4 +-
494 files changed, 11925 insertions(+), 5644 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstance.java
----------------------------------------------------------------------
diff --cc llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstance.java
index a504146,7e37e96..9004d3c
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstance.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/ServiceInstance.java
@@@ -53,14 -53,13 +53,20 @@@ public interface ServiceInstance
*/
public int getShufflePort();
+
+ /**
+ * Address for services hosted on http
+ * @return
+ */
+ public String getServicesAddress();
/**
+ * OutputFormat endpoint for service instance
+ *
+ * @return
+ */
+ public int getOutputFormatPort();
+
+ /**
* Return the last known state (without refreshing)
*
* @return
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
----------------------------------------------------------------------
diff --cc llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
index 33ab591,bd814b9..4536a6e
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapFixedRegistryImpl.java
@@@ -52,7 -54,8 +54,9 @@@ public class LlapFixedRegistryImpl impl
private final int port;
private final int shuffle;
private final int mngPort;
+ private final int webPort;
+ private final int outputFormatPort;
+ private final String webScheme;
private final String[] hosts;
private final int memory;
private final int vcores;
@@@ -66,8 -69,12 +70,13 @@@
this.shuffle = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT);
this.resolveHosts = conf.getBoolean(FIXED_REGISTRY_RESOLVE_HOST_NAMES, true);
this.mngPort = HiveConf.getIntVar(conf, ConfVars.LLAP_MANAGEMENT_RPC_PORT);
+ this.outputFormatPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT);
+
+ this.webPort = HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_WEB_PORT);
+ boolean isSsl = HiveConf.getBoolVar(conf, ConfVars.LLAP_DAEMON_WEB_SSL);
+ this.webScheme = isSsl ? "https" : "http";
+
for (Map.Entry<String, String> kv : conf) {
if (kv.getKey().startsWith(HiveConf.PREFIX_LLAP)
|| kv.getKey().startsWith(HiveConf.PREFIX_HIVE_LLAP)) {
@@@ -153,11 -171,11 +173,16 @@@
}
@Override
+ public int getOutputFormatPort() {
+ return LlapFixedRegistryImpl.this.outputFormatPort;
+ }
+
+ @Override
+ public String getServicesAddress() {
+ return serviceAddress;
+ }
+
+ @Override
public boolean isAlive() {
return true;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
----------------------------------------------------------------------
diff --cc llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
index e49c047,6af30d4..9de4d17
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
@@@ -246,13 -238,8 +246,13 @@@ public class LlapZookeeperRegistryImpl
HiveConf.getIntVar(conf, ConfVars.LLAP_MANAGEMENT_RPC_PORT)));
}
+ public Endpoint getOutputFormatEndpoint() {
+ return RegistryTypeUtils.ipcEndpoint(IPC_OUTPUTFORMAT, new InetSocketAddress(hostname,
+ HiveConf.getIntVar(conf, ConfVars.LLAP_DAEMON_OUTPUT_SERVICE_PORT)));
+ }
+
@Override
- public void register() throws IOException {
+ public String register() throws IOException {
ServiceRecord srv = new ServiceRecord();
Endpoint rpcEndpoint = getRpcEndpoint();
srv.addInternalEndpoint(rpcEndpoint);
@@@ -323,7 -310,7 +324,8 @@@
private final int rpcPort;
private final int mngPort;
private final int shufflePort;
+ private final int outputFormatPort;
+ private final String serviceAddress;
public DynamicServiceInstance(ServiceRecord srv) throws IOException {
this.srv = srv;
@@@ -331,23 -322,22 +337,26 @@@
final Endpoint shuffle = srv.getInternalEndpoint(IPC_SHUFFLE);
final Endpoint rpc = srv.getInternalEndpoint(IPC_LLAP);
final Endpoint mng = srv.getInternalEndpoint(IPC_MNG);
+ final Endpoint outputFormat = srv.getInternalEndpoint(IPC_OUTPUTFORMAT);
+ final Endpoint services = srv.getExternalEndpoint(IPC_SERVICES);
this.host =
RegistryTypeUtils.getAddressField(rpc.addresses.get(0),
AddressTypes.ADDRESS_HOSTNAME_FIELD);
this.rpcPort =
- Integer.valueOf(RegistryTypeUtils.getAddressField(rpc.addresses.get(0),
+ Integer.parseInt(RegistryTypeUtils.getAddressField(rpc.addresses.get(0),
AddressTypes.ADDRESS_PORT_FIELD));
this.mngPort =
- Integer.valueOf(RegistryTypeUtils.getAddressField(mng.addresses.get(0),
+ Integer.parseInt(RegistryTypeUtils.getAddressField(mng.addresses.get(0),
AddressTypes.ADDRESS_PORT_FIELD));
this.shufflePort =
- Integer.valueOf(RegistryTypeUtils.getAddressField(shuffle.addresses.get(0),
+ Integer.parseInt(RegistryTypeUtils.getAddressField(shuffle.addresses.get(0),
AddressTypes.ADDRESS_PORT_FIELD));
+ this.outputFormatPort =
+ Integer.valueOf(RegistryTypeUtils.getAddressField(outputFormat.addresses.get(0),
+ AddressTypes.ADDRESS_PORT_FIELD));
+ this.serviceAddress =
+ RegistryTypeUtils.getAddressField(services.addresses.get(0), AddressTypes.ADDRESS_URI);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/99cb7f96/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
----------------------------------------------------------------------