You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2022/04/26 12:15:18 UTC
[hive] branch master updated: HIVE-26134: Remove Hive on Spark from the main branch (Peter Vary reviewed by Zoltan Haindrich) (#3201)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4c7dbd560f9 HIVE-26134: Remove Hive on Spark from the main branch (Peter Vary reviewed by Zoltan Haindrich) (#3201)
4c7dbd560f9 is described below
commit 4c7dbd560f9034fbce77d17a82bd0a4abc3091fd
Author: pvary <pv...@cloudera.com>
AuthorDate: Tue Apr 26 14:15:07 2022 +0200
HIVE-26134: Remove Hive on Spark from the main branch (Peter Vary reviewed by Zoltan Haindrich) (#3201)
---
README.md | 14 +-
bin/hive | 15 -
.../hive/common/jsonexplain/JsonParserFactory.java | 4 -
.../common/jsonexplain/spark/SparkJsonParser.java | 35 -
.../hadoop/hive/common/log/InPlaceUpdate.java | 4 -
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 220 +-
.../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 35 -
.../org/apache/hadoop/hive/ql/log/PerfLogger.java | 19 -
.../org/apache/hadoop/hive/conf/TestHiveConf.java | 13 -
data/conf/perf/tpcds30tb/spark/hive-site.xml | 214 -
data/conf/spark/local/hive-site.xml | 291 -
data/conf/spark/log4j2.properties | 69 -
data/conf/spark/standalone/hive-site.xml | 300 -
data/conf/spark/yarn-cluster/hive-site.xml | 336 -
dev-support/jenkins-common.sh | 222 -
dev-support/jenkins-submit-build.sh | 74 -
.../TestHiveIcebergStorageHandlerLocalScan.java | 3 +-
.../hive/TestHiveIcebergStorageHandlerNoScan.java | 3 +-
itests/bin/download-spark.sh | 39 -
.../hive-jmh/src/main/resources/log4j2.properties | 8 +-
itests/hive-unit/pom.xml | 52 -
.../ql/exec/spark/TestSmallTableCacheEviction.java | 146 -
.../ql/exec/spark/TestSparkSessionTimeout.java | 136 -
.../hive/ql/exec/spark/TestSparkStatistics.java | 101 -
.../hive/jdbc/TestJdbcWithLocalClusterSpark.java | 321 -
.../jdbc/TestJdbcWithMiniHS2ErasureCoding.java | 21 +-
.../TestMultiSessionsHS2WithLocalClusterSpark.java | 259 -
.../org/apache/hive/jdbc/TestRestrictedList.java | 11 -
itests/pom.xml | 6 -
itests/qtest-spark/pom.xml | 456 -
.../hadoop/hive/cli/TestLocalSparkCliDriver.java | 62 -
.../hive/cli/TestMiniSparkOnYarnCliDriver.java | 66 -
.../apache/hadoop/hive/cli/TestSparkCliDriver.java | 66 -
.../hive/cli/TestSparkNegativeCliDriver.java | 63 -
.../hive/cli/TestSparkTPCDS30TBPerfCliDriver.java | 78 -
.../test/resources/testconfiguration.properties | 856 -
.../apache/hadoop/hive/cli/control/CliConfigs.java | 121 -
.../org/apache/hadoop/hive/ql/QOutProcessor.java | 2 -
.../apache/hadoop/hive/ql/QTestMiniClusters.java | 39 +-
kryo-registrator/pom.xml | 79 -
.../org/apache/hive/spark/HiveKryoRegistrator.java | 72 -
.../hive/spark/NoHashCodeKryoSerializer.java | 65 -
packaging/pom.xml | 5 -
packaging/src/main/assembly/src.xml | 2 -
pom.xml | 53 -
ql/if/queryplan.thrift | 1 -
ql/pom.xml | 65 -
ql/src/gen/thrift/gen-cpp/queryplan_types.cpp | 4 +-
ql/src/gen/thrift/gen-cpp/queryplan_types.h | 3 +-
.../hadoop/hive/ql/plan/api/OperatorType.java | 5 +-
ql/src/gen/thrift/gen-php/OperatorType.php | 7 +-
ql/src/gen/thrift/gen-py/queryplan/ttypes.py | 9 +-
ql/src/gen/thrift/gen-rb/queryplan_types.rb | 7 +-
ql/src/java/org/apache/hadoop/hive/ql/Context.java | 7 -
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 5 -
.../java/org/apache/hadoop/hive/ql/Executor.java | 15 +-
.../hadoop/hive/ql/HashTableLoaderFactory.java | 2 -
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 31 -
.../hadoop/hive/ql/exec/FileSinkOperator.java | 8 -
.../hadoop/hive/ql/exec/GlobalWorkMapFactory.java | 3 +-
.../hadoop/hive/ql/exec/MapJoinOperator.java | 21 +-
.../ql/exec/MemoryExhaustionCheckerFactory.java | 7 +-
.../hadoop/hive/ql/exec/ObjectCacheFactory.java | 2 +-
.../hadoop/hive/ql/exec/OperatorFactory.java | 10 -
.../apache/hadoop/hive/ql/exec/OperatorUtils.java | 49 -
.../hadoop/hive/ql/exec/ParsedOutputFileName.java | 46 +-
.../apache/hadoop/hive/ql/exec/ScriptOperator.java | 22 -
.../hive/ql/exec/SparkHashTableSinkOperator.java | 206 -
.../hive/ql/exec/SparkMemoryExhaustionChecker.java | 91 -
.../apache/hadoop/hive/ql/exec/TaskFactory.java | 3 -
.../org/apache/hadoop/hive/ql/exec/Utilities.java | 21 +-
.../persistence/MapJoinTableContainerSerDe.java | 148 -
.../hadoop/hive/ql/exec/spark/CacheTran.java | 70 -
.../hadoop/hive/ql/exec/spark/GroupByShuffler.java | 41 -
.../hadoop/hive/ql/exec/spark/HashTableLoader.java | 209 -
.../ql/exec/spark/HiveBaseFunctionResultList.java | 123 -
.../hive/ql/exec/spark/HiveKVResultCache.java | 265 -
.../hadoop/hive/ql/exec/spark/HiveMapFunction.java | 65 -
.../ql/exec/spark/HiveMapFunctionResultList.java | 60 -
.../ql/exec/spark/HivePairFlatMapFunction.java | 87 -
.../hive/ql/exec/spark/HiveReduceFunction.java | 55 -
.../exec/spark/HiveReduceFunctionResultList.java | 69 -
.../hadoop/hive/ql/exec/spark/HiveSparkClient.java | 55 -
.../hive/ql/exec/spark/HiveSparkClientFactory.java | 272 -
.../hive/ql/exec/spark/HiveVoidFunction.java | 46 -
.../hadoop/hive/ql/exec/spark/KryoSerializer.java | 97 -
.../hive/ql/exec/spark/LocalHiveSparkClient.java | 250 -
.../apache/hadoop/hive/ql/exec/spark/MapInput.java | 110 -
.../apache/hadoop/hive/ql/exec/spark/MapTran.java | 46 -
.../hadoop/hive/ql/exec/spark/ReduceTran.java | 46 -
.../hive/ql/exec/spark/RemoteHiveSparkClient.java | 384 -
.../hive/ql/exec/spark/ShuffleKryoSerializer.java | 62 -
.../hadoop/hive/ql/exec/spark/ShuffleTran.java | 85 -
.../hadoop/hive/ql/exec/spark/SmallTableCache.java | 148 -
.../hadoop/hive/ql/exec/spark/SortByShuffler.java | 77 -
.../ql/exec/spark/SparkDynamicPartitionPruner.java | 285 -
.../hive/ql/exec/spark/SparkMapRecordHandler.java | 210 -
.../ql/exec/spark/SparkMergeFileRecordHandler.java | 125 -
.../hive/ql/exec/spark/SparkMetricUtils.java | 63 -
.../hadoop/hive/ql/exec/spark/SparkPlan.java | 242 -
.../hive/ql/exec/spark/SparkPlanGenerator.java | 408 -
.../hive/ql/exec/spark/SparkRecordHandler.java | 164 -
.../ql/exec/spark/SparkReduceRecordHandler.java | 660 -
.../hadoop/hive/ql/exec/spark/SparkReporter.java | 83 -
.../hadoop/hive/ql/exec/spark/SparkShuffler.java | 32 -
.../hadoop/hive/ql/exec/spark/SparkTask.java | 631 -
.../hadoop/hive/ql/exec/spark/SparkTran.java | 35 -
.../hadoop/hive/ql/exec/spark/SparkUtilities.java | 347 -
.../ql/exec/spark/Statistic/SparkStatistic.java | 36 -
.../exec/spark/Statistic/SparkStatisticGroup.java | 54 -
.../ql/exec/spark/Statistic/SparkStatistics.java | 42 -
.../spark/Statistic/SparkStatisticsBuilder.java | 73 -
.../exec/spark/Statistic/SparkStatisticsNames.java | 61 -
.../hive/ql/exec/spark/session/SparkSession.java | 106 -
.../ql/exec/spark/session/SparkSessionImpl.java | 386 -
.../ql/exec/spark/session/SparkSessionManager.java | 65 -
.../spark/session/SparkSessionManagerImpl.java | 191 -
.../ql/exec/spark/status/LocalSparkJobMonitor.java | 142 -
.../exec/spark/status/RemoteSparkJobMonitor.java | 205 -
.../hive/ql/exec/spark/status/RenderStrategy.java | 246 -
.../hive/ql/exec/spark/status/SparkJobMonitor.java | 87 -
.../hive/ql/exec/spark/status/SparkJobRef.java | 29 -
.../hive/ql/exec/spark/status/SparkJobState.java | 29 -
.../hive/ql/exec/spark/status/SparkJobStatus.java | 55 -
.../ql/exec/spark/status/SparkProgressMonitor.java | 155 -
.../hive/ql/exec/spark/status/SparkStage.java | 72 -
.../ql/exec/spark/status/SparkStageProgress.java | 80 -
.../exec/spark/status/impl/JobMetricsListener.java | 90 -
.../exec/spark/status/impl/LocalSparkJobRef.java | 67 -
.../spark/status/impl/LocalSparkJobStatus.java | 213 -
.../exec/spark/status/impl/RemoteSparkJobRef.java | 62 -
.../spark/status/impl/RemoteSparkJobStatus.java | 378 -
.../exec/spark/status/impl/SparkMetricsUtils.java | 81 -
.../hive/ql/exec/vector/VectorGroupByOperator.java | 2 +-
.../vector/VectorSparkHashTableSinkOperator.java | 123 -
.../VectorSparkPartitionPruningSinkOperator.java | 115 -
.../mapjoin/VectorMapJoinCommonOperator.java | 4 +-
.../apache/hadoop/hive/ql/history/HiveHistory.java | 5 +-
.../hadoop/hive/ql/hooks/HiveProtoLoggingHook.java | 5 +-
.../hadoop/hive/ql/io/CombineHiveInputFormat.java | 27 +-
.../apache/hadoop/hive/ql/io/HiveInputFormat.java | 26 +-
.../org/apache/hadoop/hive/ql/io/IOContextMap.java | 20 +-
.../hadoop/hive/ql/log/LogDivertAppender.java | 3 +-
.../DynamicPartitionPruningOptimization.java | 42 +-
.../hadoop/hive/ql/optimizer/GenMapRedUtils.java | 44 +-
.../apache/hadoop/hive/ql/optimizer/Optimizer.java | 12 +-
.../hive/ql/optimizer/QueryPlanPostProcessor.java | 8 -
.../hive/ql/optimizer/SimpleFetchOptimizer.java | 2 +-
.../hive/ql/optimizer/SparkMapJoinProcessor.java | 87 -
.../ql/optimizer/SparkRemoveDynamicPruning.java | 80 -
.../calcite/cost/HiveDefaultCostModel.java | 2 +-
.../physical/AnnotateRunTimeStatsOptimizer.java | 8 -
.../physical/GenSparkSkewJoinProcessor.java | 423 -
.../optimizer/physical/SparkCrossProductCheck.java | 137 -
.../SparkDynamicPartitionPruningResolver.java | 139 -
.../optimizer/physical/SparkMapJoinResolver.java | 394 -
.../hive/ql/optimizer/physical/Vectorizer.java | 119 +-
.../spark/CombineEquivalentWorkResolver.java | 457 -
.../spark/SetSparkReducerParallelism.java | 281 -
.../ql/optimizer/spark/SparkJoinHintOptimizer.java | 72 -
.../ql/optimizer/spark/SparkJoinOptimizer.java | 63 -
.../ql/optimizer/spark/SparkMapJoinOptimizer.java | 556 -
.../spark/SparkPartitionPruningSinkDesc.java | 159 -
.../spark/SparkReduceSinkMapJoinProc.java | 294 -
.../optimizer/spark/SparkSMBJoinHintOptimizer.java | 102 -
.../optimizer/spark/SparkSkewJoinProcFactory.java | 237 -
.../ql/optimizer/spark/SparkSkewJoinResolver.java | 126 -
.../optimizer/spark/SparkSortMergeJoinFactory.java | 117 -
.../spark/SparkSortMergeJoinOptimizer.java | 112 -
.../ql/optimizer/spark/SplitSparkWorkResolver.java | 193 -
.../stats/annotation/StatsRulesProcFactory.java | 4 +-
.../ql/optimizer/unionproc/UnionProcessor.java | 3 +-
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 15 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 +-
.../hadoop/hive/ql/parse/TaskCompilerFactory.java | 3 -
.../hive/ql/parse/spark/GenSparkProcContext.java | 190 -
.../hadoop/hive/ql/parse/spark/GenSparkUtils.java | 675 -
.../hadoop/hive/ql/parse/spark/GenSparkWork.java | 287 -
.../hive/ql/parse/spark/GenSparkWorkWalker.java | 114 -
.../ql/parse/spark/OptimizeSparkProcContext.java | 65 -
.../hadoop/hive/ql/parse/spark/SparkCompiler.java | 627 -
.../ql/parse/spark/SparkFileSinkProcessor.java | 57 -
.../spark/SparkPartitionPruningSinkOperator.java | 299 -
.../ql/parse/spark/SparkProcessAnalyzeTable.java | 156 -
.../hive/ql/parse/spark/SparkSMBMapJoinInfo.java | 33 -
.../hive/ql/parse/spark/SplitOpTreeForDPP.java | 169 -
.../ql/plan/ConditionalResolverMergeFiles.java | 2 -
.../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 2 +-
.../hadoop/hive/ql/plan/MapredLocalWork.java | 10 -
.../apache/hadoop/hive/ql/plan/ReduceSinkDesc.java | 9 +-
.../org/apache/hadoop/hive/ql/plan/ReduceWork.java | 2 +-
.../hive/ql/plan/SparkBucketMapJoinContext.java | 39 -
.../hadoop/hive/ql/plan/SparkEdgeProperty.java | 114 -
.../hive/ql/plan/SparkHashTableSinkDesc.java | 73 -
.../org/apache/hadoop/hive/ql/plan/SparkWork.java | 436 -
.../hive/ql/plan/VectorSparkHashTableSinkDesc.java | 35 -
.../plan/VectorSparkPartitionPruningSinkDesc.java | 35 -
.../hadoop/hive/ql/ppd/SyntheticJoinPredicate.java | 3 -
.../hadoop/hive/ql/processors/SetProcessor.java | 3 -
.../authorization/command/CommandAuthorizerV2.java | 10 -
.../hadoop/hive/ql/session/SessionState.java | 39 -
.../hive/ql/exec/ParsedOutputFileNameTest.java | 37 -
.../hadoop/hive/ql/exec/TestExplainTask.java | 32 +-
.../hadoop/hive/ql/exec/TestOperatorNames.java | 13 -
.../apache/hadoop/hive/ql/exec/TestUtilities.java | 5 -
.../hadoop/hive/ql/exec/mr/TestMapRedTask.java | 1 -
.../hive/ql/exec/mr/TestMapredLocalTask.java | 1 -
.../hive/ql/exec/spark/TestHiveKVResultCache.java | 350 -
.../hive/ql/exec/spark/TestHiveSparkClient.java | 117 -
.../ql/exec/spark/TestLocalHiveSparkClient.java | 149 -
.../hive/ql/exec/spark/TestSmallTableCache.java | 202 -
.../ql/exec/spark/TestSparkInvalidFileFormat.java | 84 -
.../hadoop/hive/ql/exec/spark/TestSparkPlan.java | 147 -
.../hadoop/hive/ql/exec/spark/TestSparkTask.java | 241 -
.../hive/ql/exec/spark/TestSparkUtilities.java | 180 -
.../spark/session/TestSparkSessionManagerImpl.java | 336 -
.../ql/exec/spark/status/TestSparkJobMonitor.java | 105 -
.../hadoop/hive/ql/exec/tez/TestTezTask.java | 1 -
.../vector/expressions/TestVectorArithmetic.java | 5 -
.../TestVectorArithmeticExpressions.java | 6 -
.../apache/hadoop/hive/ql/io/TestIOContextMap.java | 34 -
.../plan/TestExecutionEngineWorkConcurrency.java | 10 +-
.../hive/ql/processors/TestResetProcessor.java | 58 -
.../queries/clientnegative/spark_job_max_tasks.q | 11 -
.../queries/clientnegative/spark_stage_max_tasks.q | 11 -
.../spark_submit_negative_executor_cores.q | 5 -
.../spark_submit_negative_executor_memory.q | 5 -
.../queries/clientnegative/spark_task_failure.q | 10 -
.../queries/clientpositive/autoColumnStats_6.q | 1 -
.../clientpositive/disable_merge_for_bucketing.q | 1 -
.../queries/clientpositive/dynamic_rdd_cache.q | 113 -
.../clientpositive/exception_in_vertex_udf.q | 2 +-
ql/src/test/queries/clientpositive/merge1.q | 1 -
ql/src/test/queries/clientpositive/merge2.q | 1 -
ql/src/test/queries/clientpositive/merge_empty.q | 1 -
ql/src/test/queries/clientpositive/mm_all.q | 3 -
ql/src/test/queries/clientpositive/orc_merge1.q | 2 -
ql/src/test/queries/clientpositive/orc_merge2.q | 1 -
ql/src/test/queries/clientpositive/orc_merge3.q | 1 -
ql/src/test/queries/clientpositive/orc_merge4.q | 2 -
ql/src/test/queries/clientpositive/orc_merge5.q | 3 -
ql/src/test/queries/clientpositive/orc_merge6.q | 3 -
ql/src/test/queries/clientpositive/orc_merge7.q | 3 -
ql/src/test/queries/clientpositive/orc_merge8.q | 2 -
ql/src/test/queries/clientpositive/orc_merge9.q | 1 -
.../queries/clientpositive/orc_merge_diff_fs.q | 2 -
.../queries/clientpositive/orc_merge_incompat1.q | 1 -
.../queries/clientpositive/orc_merge_incompat2.q | 1 -
.../runtime_skewjoin_mapjoin_spark.q | 2 +-
.../clientpositive/skewjoin_union_remove_1.q | 1 -
.../clientpositive/skewjoin_union_remove_2.q | 1 -
.../clientpositive/spark_combine_equivalent_work.q | 33 -
.../spark_combine_equivalent_work_2.q | 42 -
.../queries/clientpositive/spark_constprog_dpp.q | 17 -
.../spark_dynamic_partition_pruning.q | 209 -
.../spark_dynamic_partition_pruning_2.q | 136 -
.../spark_dynamic_partition_pruning_3.q | 230 -
.../spark_dynamic_partition_pruning_4.q | 178 -
.../spark_dynamic_partition_pruning_5.q | 26 -
.../spark_dynamic_partition_pruning_6.q | 59 -
.../spark_dynamic_partition_pruning_7.q | 22 -
.../spark_dynamic_partition_pruning_mapjoin_only.q | 52 -
...k_dynamic_partition_pruning_recursive_mapjoin.q | 148 -
.../clientpositive/spark_explain_groupbyshuffle.q | 9 -
.../queries/clientpositive/spark_explainuser_1.q | 681 -
.../clientpositive/spark_in_process_launcher.q | 6 -
.../queries/clientpositive/spark_local_queries.q | 19 -
.../spark_multi_insert_parallel_orderby.q | 68 -
.../clientpositive/spark_opt_shuffle_serde.q | 8 -
.../queries/clientpositive/spark_union_merge.q | 42 -
.../queries/clientpositive/spark_use_op_stats.q | 43 -
.../spark_use_ts_stats_for_mapjoin.q | 78 -
.../spark_vectorized_dynamic_partition_pruning.q | 197 -
.../test/queries/clientpositive/union_remove_1.q | 1 -
.../test/queries/clientpositive/union_remove_10.q | 1 -
.../test/queries/clientpositive/union_remove_11.q | 1 -
.../test/queries/clientpositive/union_remove_12.q | 1 -
.../test/queries/clientpositive/union_remove_13.q | 1 -
.../test/queries/clientpositive/union_remove_14.q | 1 -
.../test/queries/clientpositive/union_remove_15.q | 1 -
.../test/queries/clientpositive/union_remove_16.q | 1 -
.../test/queries/clientpositive/union_remove_17.q | 1 -
.../test/queries/clientpositive/union_remove_18.q | 1 -
.../test/queries/clientpositive/union_remove_19.q | 1 -
.../test/queries/clientpositive/union_remove_2.q | 1 -
.../test/queries/clientpositive/union_remove_20.q | 1 -
.../test/queries/clientpositive/union_remove_21.q | 1 -
.../test/queries/clientpositive/union_remove_22.q | 1 -
.../test/queries/clientpositive/union_remove_23.q | 1 -
.../test/queries/clientpositive/union_remove_24.q | 1 -
.../test/queries/clientpositive/union_remove_25.q | 1 -
.../test/queries/clientpositive/union_remove_3.q | 1 -
.../test/queries/clientpositive/union_remove_4.q | 1 -
.../test/queries/clientpositive/union_remove_5.q | 1 -
.../test/queries/clientpositive/union_remove_6.q | 1 -
.../test/queries/clientpositive/union_remove_7.q | 1 -
.../test/queries/clientpositive/union_remove_8.q | 1 -
.../test/queries/clientpositive/union_remove_9.q | 1 -
.../spark/groupby2_map_skew_multi_distinct.q.out | 9 -
.../spark/groupby2_multi_distinct.q.out | 9 -
.../spark/groupby3_map_skew_multi_distinct.q.out | 9 -
.../spark/groupby3_multi_distinct.q.out | 9 -
.../spark/groupby_grouping_sets7.q.out | 9 -
.../clientnegative/spark/spark_job_max_tasks.q.out | 93 -
.../spark/spark_stage_max_tasks.q.out | 93 -
.../spark_submit_negative_executor_cores.q.out | 5 -
.../spark_submit_negative_executor_memory.q.out | 5 -
.../clientnegative/spark/spark_task_failure.q.out | 57 -
.../clientpositive/llap/acid_no_buckets.q.out | 60 +-
.../llap/constraints_explain_ddl.q.out | 58 +-
.../llap/convert_decimal64_to_decimal.q.out | 16 +-
.../llap/correlationoptimizer4.q.out | 48 +-
.../llap/expanding_multivaluecolvector.q.out | 4 +-
.../llap/fetch_deleted_rows_vector.q.out | 4 +-
.../results/clientpositive/llap/keep_uniform.q.out | 24 +-
.../results/clientpositive/llap/llap_acid.q.out | 12 +-
.../clientpositive/llap/llap_acid_fast.q.out | 12 +-
.../llap/llap_decimal64_reader.q.out | 4 +-
.../clientpositive/llap/llap_partitioned.q.out | 8 +-
.../llap/llap_vector_nohybridgrace.q.out | 16 +-
.../llap/mapjoin_decimal_vectorized.q.out | 32 +-
.../results/clientpositive/llap/mergejoin.q.out | 88 +-
.../llap/orc_struct_type_vectorization.q.out | 4 +-
.../llap/parquet_complex_types_vectorization.q.out | 24 +-
.../llap/parquet_map_type_vectorization.q.out | 8 +-
.../clientpositive/llap/parquet_no_row_serde.q.out | 6 +-
.../llap/parquet_struct_type_vectorization.q.out | 4 +-
.../llap/parquet_vectorization_0.q.out | 22 +-
.../llap/parquet_vectorization_1.q.out | 4 +-
.../llap/parquet_vectorization_12.q.out | 8 +-
.../llap/parquet_vectorization_13.q.out | 16 +-
.../llap/parquet_vectorization_14.q.out | 8 +-
.../llap/parquet_vectorization_15.q.out | 6 +-
.../llap/parquet_vectorization_16.q.out | 4 +-
.../llap/parquet_vectorization_17.q.out | 4 +-
.../llap/parquet_vectorization_2.q.out | 4 +-
.../llap/parquet_vectorization_3.q.out | 4 +-
.../llap/parquet_vectorization_4.q.out | 4 +-
.../llap/parquet_vectorization_5.q.out | 4 +-
.../llap/parquet_vectorization_7.q.out | 8 +-
.../llap/parquet_vectorization_8.q.out | 8 +-
.../llap/parquet_vectorization_9.q.out | 4 +-
.../llap/parquet_vectorization_div0.q.out | 8 +-
.../llap/parquet_vectorization_limit.q.out | 26 +-
.../llap/parquet_vectorization_offset_limit.q.out | 4 +-
.../llap/parquet_vectorization_part_project.q.out | 2 +-
.../llap/parquet_vectorization_pushdown.q.out | 2 +-
.../llap/probedecode_mapjoin_keyexpr.q.out | 4 +-
.../llap/probedecode_mapjoin_simple.q.out | 4 +-
.../llap/probedecode_mapjoin_stats.q.out | 8 +-
.../llap/probedecode_static_keytype.q.out | 8 +-
.../test/results/clientpositive/llap/sysdb.q.out | 4 +-
.../llap/temp_table_llap_partitioned.q.out | 8 +-
.../clientpositive/llap/udf_ltrim_vector.q.out | 8 +-
.../clientpositive/llap/udf_rtrim_vector.q.out | 8 +-
.../clientpositive/llap/udf_trim_vector.q.out | 8 +-
.../llap/vector_adaptor_usage_mode.q.out | 12 +-
.../clientpositive/llap/vector_aggregate_9.q.out | 12 +-
.../llap/vector_aggregate_without_gby.q.out | 4 +-
.../llap/vector_annotate_stats_select.q.out | 16 +-
.../llap/vector_auto_smb_mapjoin_14.q.out | 40 +-
.../llap/vector_between_columns.q.out | 12 +-
.../clientpositive/llap/vector_between_in.q.out | 64 +-
.../llap/vector_binary_join_groupby.q.out | 20 +-
.../clientpositive/llap/vector_bucket.q.out | 2 +-
.../clientpositive/llap/vector_case_when_2.q.out | 28 +-
.../llap/vector_case_when_conversion.q.out | 16 +-
.../clientpositive/llap/vector_cast_constant.q.out | 8 +-
.../clientpositive/llap/vector_cast_filter.q.out | 24 +-
.../clientpositive/llap/vector_char_2.q.out | 16 +-
.../clientpositive/llap/vector_char_mapjoin1.q.out | 24 +-
.../clientpositive/llap/vector_char_simple.q.out | 8 +-
.../llap/vector_char_varchar_1.q.out | 8 +-
.../clientpositive/llap/vector_coalesce.q.out | 20 +-
.../clientpositive/llap/vector_coalesce_2.q.out | 8 +-
.../clientpositive/llap/vector_coalesce_3.q.out | 4 +-
.../clientpositive/llap/vector_coalesce_4.q.out | 4 +-
.../clientpositive/llap/vector_complex_all.q.out | 32 +-
.../clientpositive/llap/vector_complex_join.q.out | 12 +-
.../results/clientpositive/llap/vector_count.q.out | 14 +-
.../llap/vector_count_distinct.q.out | 8 +-
.../clientpositive/llap/vector_data_types.q.out | 8 +-
.../clientpositive/llap/vector_date_1.q.out | 20 +-
...r_decimal64_add_decimal64column_diffscale.q.out | 4 +-
.../llap/vector_decimal64_case_when_nvl.q.out | 12 +-
.../llap/vector_decimal64_case_when_nvl_cbo.q.out | 12 +-
.../vector_decimal64_div_decimal64column.q.out | 4 +-
.../vector_decimal64_div_decimal64scalar.q.out | 8 +-
.../vector_decimal64_mul_decimal64column.q.out | 4 +-
.../vector_decimal64_mul_decimal64scalar.q.out | 4 +-
.../llap/vector_decimal64_mul_intcolumn.q.out | 8 +-
.../llap/vector_decimal64_multi_vertex.q.out | 8 +-
.../clientpositive/llap/vector_decimal_1.q.out | 36 +-
.../clientpositive/llap/vector_decimal_10_0.q.out | 8 +-
.../clientpositive/llap/vector_decimal_2.q.out | 64 +-
.../clientpositive/llap/vector_decimal_6.q.out | 18 +-
.../llap/vector_decimal_aggregate.q.out | 16 +-
.../llap/vector_decimal_col_scalar_division.q.out | 4 +-
.../llap/vector_decimal_expressions.q.out | 8 +-
.../clientpositive/llap/vector_decimal_join.q.out | 8 +-
.../llap/vector_decimal_mapjoin.q.out | 24 +-
.../llap/vector_decimal_partition.q.out | 4 +-
.../llap/vector_decimal_precision.q.out | 8 +-
.../clientpositive/llap/vector_decimal_round.q.out | 24 +-
.../llap/vector_decimal_round_2.q.out | 16 +-
.../llap/vector_decimal_trailing.q.out | 4 +-
.../clientpositive/llap/vector_decimal_udf.q.out | 60 +-
.../llap/vector_delete_orig_table.q.out | 4 +-
.../clientpositive/llap/vector_distinct_2.q.out | 4 +-
.../clientpositive/llap/vector_empty_where.q.out | 32 +-
.../llap/vector_full_outer_join.q.out | 24 +-
.../llap/vector_fullouter_mapjoin_1_fast.q.out | 90 +-
.../vector_fullouter_mapjoin_1_optimized.q.out | 90 +-
...or_fullouter_mapjoin_1_optimized_passthru.q.out | 90 +-
.../clientpositive/llap/vector_groupby4.q.out | 16 +-
.../clientpositive/llap/vector_groupby6.q.out | 16 +-
.../clientpositive/llap/vector_groupby_3.q.out | 4 +-
.../clientpositive/llap/vector_groupby_cube1.q.out | 50 +-
.../llap/vector_groupby_grouping_id1.q.out | 24 +-
.../llap/vector_groupby_grouping_id2.q.out | 104 +-
.../llap/vector_groupby_grouping_id3.q.out | 8 +-
.../llap/vector_groupby_grouping_sets1.q.out | 28 +-
.../llap/vector_groupby_grouping_sets2.q.out | 30 +-
.../llap/vector_groupby_grouping_sets3.q.out | 10 +-
.../llap/vector_groupby_grouping_sets3_dec.q.out | 16 +-
.../llap/vector_groupby_grouping_sets4.q.out | 40 +-
.../llap/vector_groupby_grouping_sets5.q.out | 28 +-
.../llap/vector_groupby_grouping_sets6.q.out | 8 +-
.../vector_groupby_grouping_sets_grouping.q.out | 68 +-
.../llap/vector_groupby_grouping_sets_limit.q.out | 48 +-
.../llap/vector_groupby_grouping_window.q.out | 8 +-
.../llap/vector_groupby_mapjoin.q.out | 20 +-
.../llap/vector_groupby_reduce.q.out | 32 +-
.../llap/vector_groupby_rollup1.q.out | 34 +-
.../llap/vector_groupby_sort_11.q.out | 34 +-
.../llap/vector_groupby_sort_8.q.out | 4 +-
.../clientpositive/llap/vector_grouping_sets.q.out | 8 +-
.../llap/vector_identity_reuse.q.out | 12 +-
.../clientpositive/llap/vector_if_expr.q.out | 4 +-
.../clientpositive/llap/vector_if_expr_2.q.out | 4 +-
.../llap/vector_include_no_sel.q.out | 6 +-
.../clientpositive/llap/vector_inner_join.q.out | 36 +-
.../clientpositive/llap/vector_interval_1.q.out | 32 +-
.../clientpositive/llap/vector_interval_2.q.out | 40 +-
.../llap/vector_interval_arithmetic.q.out | 28 +-
.../llap/vector_interval_mapjoin.q.out | 4 +-
.../clientpositive/llap/vector_join30.q.out | 164 +-
.../clientpositive/llap/vector_join_filters.q.out | 40 +-
.../clientpositive/llap/vector_join_nulls.q.out | 40 +-
.../llap/vector_left_outer_join.q.out | 4 +-
.../llap/vector_left_outer_join2.q.out | 16 +-
.../llap/vector_leftsemi_mapjoin.q.out | 784 +-
.../clientpositive/llap/vector_like_2.q.out | 4 +-
.../llap/vector_llap_io_data_conversion.q.out | 4 +-
.../clientpositive/llap/vector_llap_text_1.q.out | 8 +-
.../llap/vector_mapjoin_complex_values.q.out | 12 +-
.../llap/vector_mapjoin_reduce.q.out | 24 +-
.../llap/vector_mr_diff_schema_alias.q.out | 4 +-
.../llap/vector_null_projection.q.out | 6 +-
.../clientpositive/llap/vector_nullsafe_join.q.out | 112 +-
.../llap/vector_number_compare_projection.q.out | 8 +-
.../llap/vector_orc_merge_incompat_schema.q.out | 2 +-
.../llap/vector_orc_nested_column_pruning.q.out | 46 +-
.../clientpositive/llap/vector_order_null.q.out | 44 +-
.../clientpositive/llap/vector_orderby_5.q.out | 8 +-
.../clientpositive/llap/vector_outer_join0.q.out | 8 +-
.../clientpositive/llap/vector_outer_join1.q.out | 24 +-
.../clientpositive/llap/vector_outer_join2.q.out | 16 +-
.../llap/vector_outer_join_constants.q.out | 30 +-
.../llap/vector_outer_join_no_keys.q.out | 16 +-
.../llap/vector_outer_reference_windowed.q.out | 84 +-
.../llap/vector_partition_diff_num_cols.q.out | 20 +-
.../llap/vector_partitioned_date_time.q.out | 72 +-
.../results/clientpositive/llap/vector_ptf_1.q.out | 4 +-
.../llap/vector_ptf_bounded_start.q.out | 106 +-
.../llap/vector_ptf_classcast_exception.q.out | 12 +-
.../llap/vector_ptf_count_distinct.q.out | 222 +-
.../llap/vector_ptf_last_value_streaming.q.out | 4 +-
.../clientpositive/llap/vector_ptf_lead_lag.q.out | 12 +-
.../llap/vector_ptf_part_simple.q.out | 124 +-
.../clientpositive/llap/vector_reduce1.q.out | 4 +-
.../clientpositive/llap/vector_reduce2.q.out | 4 +-
.../clientpositive/llap/vector_reduce3.q.out | 4 +-
.../llap/vector_reduce_groupby_decimal.q.out | 8 +-
.../vector_reduce_groupby_duplicate_cols.q.out | 6 +-
.../clientpositive/llap/vector_retry_failure.q.out | 4 +-
.../llap/vector_reuse_scratchcols.q.out | 8 +-
.../clientpositive/llap/vector_string_concat.q.out | 8 +-
.../llap/vector_tablesample_rows.q.out | 2 +-
.../clientpositive/llap/vector_topnkey.q.out | 8 +-
.../results/clientpositive/llap/vector_udf1.q.out | 10 +-
.../clientpositive/llap/vector_union_null.q.out | 4 +-
.../llap/vector_varchar_mapjoin1.q.out | 6 +-
.../llap/vector_varchar_simple.q.out | 8 +-
.../llap/vector_when_case_null.q.out | 4 +-
.../clientpositive/llap/vector_windowing.q.out | 260 +-
.../llap/vector_windowing_expressions.q.out | 40 +-
.../clientpositive/llap/vector_windowing_gby.q.out | 10 +-
.../llap/vector_windowing_gby2.q.out | 42 +-
.../llap/vector_windowing_multipartitioning.q.out | 44 +-
.../llap/vector_windowing_navfn.q.out | 44 +-
.../llap/vector_windowing_order_null.q.out | 32 +-
.../llap/vector_windowing_range_multiorder.q.out | 44 +-
.../llap/vector_windowing_rank.q.out | 34 +-
.../llap/vector_windowing_row_number.q.out | 12 +-
.../llap/vector_windowing_streaming.q.out | 20 +-
.../llap/vector_windowing_windowspec.q.out | 44 +-
.../llap/vector_windowing_windowspec4.q.out | 4 +-
.../clientpositive/llap/vectorization_0.q.out | 22 +-
.../clientpositive/llap/vectorization_1.q.out | 4 +-
.../clientpositive/llap/vectorization_12.q.out | 8 +-
.../clientpositive/llap/vectorization_13.q.out | 16 +-
.../clientpositive/llap/vectorization_14.q.out | 8 +-
.../clientpositive/llap/vectorization_15.q.out | 6 +-
.../clientpositive/llap/vectorization_16.q.out | 4 +-
.../clientpositive/llap/vectorization_17.q.out | 4 +-
.../clientpositive/llap/vectorization_2.q.out | 4 +-
.../clientpositive/llap/vectorization_3.q.out | 4 +-
.../clientpositive/llap/vectorization_4.q.out | 4 +-
.../clientpositive/llap/vectorization_5.q.out | 4 +-
.../clientpositive/llap/vectorization_7.q.out | 8 +-
.../clientpositive/llap/vectorization_8.q.out | 8 +-
.../clientpositive/llap/vectorization_9.q.out | 4 +-
.../clientpositive/llap/vectorization_div0.q.out | 16 +-
.../llap/vectorization_input_format_excludes.q.out | 8 +-
.../clientpositive/llap/vectorization_limit.q.out | 34 +-
.../llap/vectorization_nested_udf.q.out | 4 +-
.../llap/vectorization_numeric_overflows.q.out | 32 +-
.../llap/vectorization_offset_limit.q.out | 4 +-
.../llap/vectorization_parquet_projection.q.out | 10 +-
.../llap/vectorization_part_project.q.out | 2 +-
.../llap/vectorization_pushdown.q.out | 2 +-
.../llap/vectorization_short_regress.q.out | 96 +-
.../clientpositive/llap/vectorized_case.q.out | 8 +-
.../llap/vectorized_decimal64_boundary.q.out | 4 +-
.../llap/vectorized_distinct_gby.q.out | 16 +-
.../vectorized_dynamic_partition_pruning.q.out | 100 +-
.../vectorized_dynamic_semijoin_reduction.q.out | 110 +-
.../vectorized_dynamic_semijoin_reduction2.q.out | 12 +-
.../vectorized_insert_into_bucketed_table.q.out | 2 +-
.../clientpositive/llap/vectorized_join46.q.out | 52 +-
.../clientpositive/llap/vectorized_join46_mr.q.out | 52 +-
.../clientpositive/llap/vectorized_mapjoin.q.out | 8 +-
.../clientpositive/llap/vectorized_mapjoin2.q.out | 8 +-
.../clientpositive/llap/vectorized_mapjoin3.q.out | 30 +-
.../llap/vectorized_nested_mapjoin.q.out | 4 +-
.../clientpositive/llap/vectorized_parquet.q.out | 2 +-
.../llap/vectorized_parquet_types.q.out | 10 +-
.../clientpositive/llap/vectorized_ptf.q.out | 166 +-
.../llap/vectorized_shufflejoin.q.out | 6 +-
.../clientpositive/llap/vectorized_stats.q.out | 24 +-
.../clientpositive/llap/vectorized_timestamp.q.out | 12 +-
.../llap/vectorized_timestamp_funcs.q.out | 28 +-
.../clientpositive/llap/windowing_distinct.q.out | 6 +-
.../clientpositive/spark/add_part_multiple.q.out | 119 -
.../clientpositive/spark/alter_merge_orc.q.out | 263 -
.../spark/alter_merge_stats_orc.q.out | 376 -
.../clientpositive/spark/annotate_stats_join.q.out | 1138 -
.../results/clientpositive/spark/auto_join0.q.out | 180 -
.../results/clientpositive/spark/auto_join1.q.out | 130 -
.../results/clientpositive/spark/auto_join10.q.out | 139 -
.../results/clientpositive/spark/auto_join11.q.out | 139 -
.../results/clientpositive/spark/auto_join12.q.out | 176 -
.../results/clientpositive/spark/auto_join13.q.out | 181 -
.../results/clientpositive/spark/auto_join14.q.out | 142 -
.../results/clientpositive/spark/auto_join15.q.out | 139 -
.../results/clientpositive/spark/auto_join16.q.out | 139 -
.../results/clientpositive/spark/auto_join17.q.out | 132 -
.../results/clientpositive/spark/auto_join18.q.out | 207 -
.../spark/auto_join18_multi_distinct.q.out | 215 -
.../results/clientpositive/spark/auto_join19.q.out | 154 -
.../results/clientpositive/spark/auto_join2.q.out | 160 -
.../results/clientpositive/spark/auto_join20.q.out | 344 -
.../results/clientpositive/spark/auto_join21.q.out | 638 -
.../results/clientpositive/spark/auto_join22.q.out | 149 -
.../results/clientpositive/spark/auto_join23.q.out | 209 -
.../results/clientpositive/spark/auto_join24.q.out | 135 -
.../results/clientpositive/spark/auto_join26.q.out | 173 -
.../results/clientpositive/spark/auto_join27.q.out | 190 -
.../results/clientpositive/spark/auto_join28.q.out | 541 -
.../results/clientpositive/spark/auto_join29.q.out | 3806 ---
.../results/clientpositive/spark/auto_join3.q.out | 155 -
.../results/clientpositive/spark/auto_join30.q.out | 1277 -
.../results/clientpositive/spark/auto_join31.q.out | 176 -
.../results/clientpositive/spark/auto_join32.q.out | 617 -
.../results/clientpositive/spark/auto_join4.q.out | 176 -
.../results/clientpositive/spark/auto_join5.q.out | 176 -
.../results/clientpositive/spark/auto_join6.q.out | 178 -
.../results/clientpositive/spark/auto_join7.q.out | 223 -
.../results/clientpositive/spark/auto_join8.q.out | 179 -
.../results/clientpositive/spark/auto_join9.q.out | 138 -
.../clientpositive/spark/auto_join_filters.q.out | 941 -
.../clientpositive/spark/auto_join_nulls.q.out | 354 -
.../spark/auto_join_reordering_values.q.out | 656 -
.../clientpositive/spark/auto_join_stats.q.out | 361 -
.../clientpositive/spark/auto_join_stats2.q.out | 338 -
.../spark/auto_join_without_localtask.q.out | 578 -
.../clientpositive/spark/auto_smb_mapjoin_14.q.out | 1840 --
.../spark/auto_sortmerge_join_1.q.out | 813 -
.../spark/auto_sortmerge_join_12.q.out | 675 -
.../spark/auto_sortmerge_join_13.q.out | 671 -
.../spark/auto_sortmerge_join_14.q.out | 212 -
.../spark/auto_sortmerge_join_15.q.out | 180 -
.../spark/auto_sortmerge_join_16.q.out | 705 -
.../spark/auto_sortmerge_join_16.q.out_spark | 705 -
.../spark/auto_sortmerge_join_2.q.out | 559 -
.../spark/auto_sortmerge_join_3.q.out | 642 -
.../spark/auto_sortmerge_join_4.q.out | 658 -
.../spark/auto_sortmerge_join_5.q.out | 587 -
.../spark/auto_sortmerge_join_6.q.out | 1262 -
.../spark/auto_sortmerge_join_7.q.out | 840 -
.../spark/auto_sortmerge_join_8.q.out | 840 -
.../spark/auto_sortmerge_join_9.q.out | 3079 --
.../spark/avro_compression_enabled_native.q.out | 35 -
.../clientpositive/spark/avro_decimal_native.q.out | 187 -
.../results/clientpositive/spark/avro_joins.q.out | 191 -
.../clientpositive/spark/avro_joins_native.q.out | 86 -
.../results/clientpositive/spark/bucket2.q.out | 478 -
.../results/clientpositive/spark/bucket3.q.out | 492 -
.../results/clientpositive/spark/bucket4.q.out | 484 -
.../clientpositive/spark/bucket4.q.out_spark | 479 -
.../results/clientpositive/spark/bucket5.q.out | 470 -
.../results/clientpositive/spark/bucket6.q.out | 211 -
.../results/clientpositive/spark/bucket7.q.out | 56 -
.../clientpositive/spark/bucket_map_join_1.q.out | 292 -
.../clientpositive/spark/bucket_map_join_2.q.out | 292 -
.../spark/bucket_map_join_spark1.q.out | 759 -
.../spark/bucket_map_join_spark2.q.out | 754 -
.../spark/bucket_map_join_spark3.q.out | 743 -
.../spark/bucket_map_join_spark4.q.out | 864 -
.../spark/bucket_map_join_tez1.q.out | 7504 -----
.../spark/bucket_map_join_tez2.q.out | 2567 --
.../clientpositive/spark/bucketmapjoin1.q.out | 1092 -
.../clientpositive/spark/bucketmapjoin10.q.out | 495 -
.../clientpositive/spark/bucketmapjoin11.q.out | 892 -
.../clientpositive/spark/bucketmapjoin12.q.out | 624 -
.../clientpositive/spark/bucketmapjoin13.q.out | 1212 -
.../clientpositive/spark/bucketmapjoin2.q.out | 1333 -
.../clientpositive/spark/bucketmapjoin3.q.out | 894 -
.../clientpositive/spark/bucketmapjoin4.q.out | 854 -
.../clientpositive/spark/bucketmapjoin5.q.out | 1016 -
.../clientpositive/spark/bucketmapjoin6.q.out | 142 -
.../clientpositive/spark/bucketmapjoin7.q.out | 339 -
.../spark/bucketmapjoin7.q.out_spark | 331 -
.../clientpositive/spark/bucketmapjoin8.q.out | 610 -
.../clientpositive/spark/bucketmapjoin9.q.out | 629 -
.../spark/bucketmapjoin_negative.q.out | 325 -
.../spark/bucketmapjoin_negative2.q.out | 397 -
.../spark/bucketmapjoin_negative3.q.out | 2086 --
.../spark/bucketsortoptimize_insert_2.q.out | 1132 -
.../spark/bucketsortoptimize_insert_4.q.out | 457 -
.../spark/bucketsortoptimize_insert_6.q.out | 1286 -
.../spark/bucketsortoptimize_insert_7.q.out | 557 -
.../spark/bucketsortoptimize_insert_8.q.out | 436 -
.../results/clientpositive/spark/cbo_gby.q.out | 118 -
.../clientpositive/spark/cbo_gby_empty.q.out | 75 -
.../results/clientpositive/spark/cbo_limit.q.out | 239 -
.../clientpositive/spark/cbo_semijoin.q.out | 438 -
.../clientpositive/spark/cbo_simple_select.q.out | 991 -
.../results/clientpositive/spark/cbo_stats.q.out | 12 -
.../results/clientpositive/spark/cbo_subq_in.q.out | 123 -
.../clientpositive/spark/cbo_subq_not_in.q.out | 307 -
.../clientpositive/spark/cbo_udf_udaf.q.out | 119 -
.../results/clientpositive/spark/cbo_union.q.out | 914 -
.../clientpositive/spark/column_access_stats.q.out | 921 -
.../spark/constprog_partitioner.q.out | 189 -
.../clientpositive/spark/constprog_semijoin.q.out | 839 -
.../test/results/clientpositive/spark/count.q.out | 953 -
.../spark/create_merge_compressed.q.out | 138 -
.../results/clientpositive/spark/cross_join.q.out | 456 -
.../spark/cross_product_check_1.q.out | 658 -
.../spark/cross_product_check_2.q.out | 675 -
.../test/results/clientpositive/spark/ctas.q.out | 909 -
.../spark/custom_input_output_format.q.out | 98 -
.../results/clientpositive/spark/date_join1.q.out | 76 -
.../results/clientpositive/spark/decimal_1_1.q.out | 104 -
.../clientpositive/spark/decimal_join.q.out | 76 -
.../spark/disable_merge_for_bucketing.q.out | 516 -
.../spark/disable_merge_for_bucketing.q.out_spark | 511 -
.../clientpositive/spark/dynamic_rdd_cache.q.out | 1258 -
.../spark/dynpart_sort_optimization.q.out | 3927 ---
.../clientpositive/spark/empty_dir_in_table.q.out | 46 -
.../clientpositive/spark/enforce_order.q.out | 80 -
.../clientpositive/spark/escape_clusterby1.q.out | 116 -
.../spark/escape_distributeby1.q.out | 116 -
.../clientpositive/spark/escape_orderby1.q.out | 114 -
.../clientpositive/spark/escape_sortby1.q.out | 114 -
.../spark/explaindenpendencydiffengs.q.out | 62 -
...xternal_table_with_space_in_location_path.q.out | 88 -
.../spark/file_with_header_footer.q.out | 184 -
.../spark/filter_join_breaktask.q.out | 390 -
.../spark/filter_join_breaktask2.q.out | 268 -
.../spark/gen_udf_example_add10.q.out | 109 -
.../results/clientpositive/spark/groupby1.q.out | 426 -
.../results/clientpositive/spark/groupby10.q.out | 671 -
.../results/clientpositive/spark/groupby11.q.out | 850 -
.../clientpositive/spark/groupby1_map.q.out | 416 -
.../clientpositive/spark/groupby1_map_nomap.q.out | 416 -
.../clientpositive/spark/groupby1_map_skew.q.out | 433 -
.../clientpositive/spark/groupby1_noskew.q.out | 409 -
.../results/clientpositive/spark/groupby2.q.out | 130 -
.../clientpositive/spark/groupby2_map.q.out | 121 -
.../spark/groupby2_map_multi_distinct.q.out | 238 -
.../clientpositive/spark/groupby2_map_skew.q.out | 137 -
.../clientpositive/spark/groupby2_noskew.q.out | 114 -
.../spark/groupby2_noskew_multi_distinct.q.out | 117 -
.../results/clientpositive/spark/groupby3.q.out | 159 -
.../clientpositive/spark/groupby3_map.q.out | 172 -
.../spark/groupby3_map_multi_distinct.q.out | 162 -
.../clientpositive/spark/groupby3_map_skew.q.out | 168 -
.../clientpositive/spark/groupby3_noskew.q.out | 146 -
.../spark/groupby3_noskew_multi_distinct.q.out | 156 -
.../results/clientpositive/spark/groupby4.q.out | 123 -
.../clientpositive/spark/groupby4_map.q.out | 102 -
.../clientpositive/spark/groupby4_map_skew.q.out | 102 -
.../clientpositive/spark/groupby4_noskew.q.out | 108 -
.../results/clientpositive/spark/groupby5.q.out | 438 -
.../clientpositive/spark/groupby5_map.q.out | 103 -
.../clientpositive/spark/groupby5_map_skew.q.out | 103 -
.../clientpositive/spark/groupby5_noskew.q.out | 421 -
.../results/clientpositive/spark/groupby6.q.out | 123 -
.../clientpositive/spark/groupby6_map.q.out | 114 -
.../clientpositive/spark/groupby6_map_skew.q.out | 129 -
.../clientpositive/spark/groupby6_noskew.q.out | 108 -
.../results/clientpositive/spark/groupby7.q.out | 668 -
.../clientpositive/spark/groupby7_map.q.out | 793 -
.../spark/groupby7_map_multi_single_reducer.q.out | 784 -
.../clientpositive/spark/groupby7_map_skew.q.out | 810 -
.../clientpositive/spark/groupby7_noskew.q.out | 785 -
.../groupby7_noskew_multi_single_reducer.q.out | 209 -
.../results/clientpositive/spark/groupby8.q.out | 1584 -
.../clientpositive/spark/groupby8_map.q.out | 783 -
.../clientpositive/spark/groupby8_map_skew.q.out | 807 -
.../clientpositive/spark/groupby8_noskew.q.out | 783 -
.../results/clientpositive/spark/groupby9.q.out | 4015 ---
.../clientpositive/spark/groupby_bigdata.q.out | 11 -
.../spark/groupby_complex_types.q.out | 1197 -
...roupby_complex_types_multi_single_reducer.q.out | 246 -
.../clientpositive/spark/groupby_cube1.q.out | 793 -
.../spark/groupby_grouping_id2.q.out | 346 -
.../clientpositive/spark/groupby_map_ppr.q.out | 302 -
.../spark/groupby_map_ppr_multi_distinct.q.out | 304 -
.../groupby_multi_insert_common_distinct.q.out | 208 -
.../spark/groupby_multi_single_reducer.q.out | 753 -
.../spark/groupby_multi_single_reducer2.q.out | 199 -
.../spark/groupby_multi_single_reducer3.q.out | 744 -
.../clientpositive/spark/groupby_position.q.out | 1707 -
.../results/clientpositive/spark/groupby_ppr.q.out | 295 -
.../spark/groupby_ppr_multi_distinct.q.out | 582 -
.../clientpositive/spark/groupby_resolution.q.out | 870 -
.../clientpositive/spark/groupby_rollup1.q.out | 614 -
.../clientpositive/spark/groupby_sort_1_23.q.out | 4497 ---
.../spark/groupby_sort_skew_1_23.q.out | 4685 ---
.../test/results/clientpositive/spark/having.q.out | 1326 -
.../spark/identity_project_remove_skip.q.out | 119 -
.../spark/import_exported_table.q.out | 28 -
.../spark/infer_bucket_sort_bucketed_table.q.out | 106 -
.../spark/infer_bucket_sort_map_operators.q.out | 614 -
.../spark/infer_bucket_sort_merge.q.out | 106 -
.../spark/infer_bucket_sort_num_buckets.q.out | 374 -
.../infer_bucket_sort_reducers_power_two.q.out | 302 -
.../results/clientpositive/spark/innerjoin.q.out | 1273 -
.../results/clientpositive/spark/input12.q.out | 704 -
.../results/clientpositive/spark/input13.q.out | 733 -
.../results/clientpositive/spark/input14.q.out | 212 -
.../results/clientpositive/spark/input16_cc.q.out | 528 -
.../results/clientpositive/spark/input17.q.out | 136 -
.../results/clientpositive/spark/input18.q.out | 212 -
.../clientpositive/spark/input1_limit.q.out | 213 -
.../results/clientpositive/spark/input_part2.q.out | 522 -
.../results/clientpositive/spark/insert1.q.out | 449 -
.../clientpositive/spark/insert_into1.q.out | 666 -
.../clientpositive/spark/insert_into2.q.out | 551 -
.../clientpositive/spark/insert_into3.q.out | 375 -
.../spark/insert_overwrite_directory2.q.out | 54 -
.../test/results/clientpositive/spark/join0.q.out | 244 -
.../test/results/clientpositive/spark/join1.q.out | 1158 -
.../test/results/clientpositive/spark/join10.q.out | 1144 -
.../test/results/clientpositive/spark/join11.q.out | 266 -
.../test/results/clientpositive/spark/join12.q.out | 384 -
.../test/results/clientpositive/spark/join13.q.out | 332 -
.../test/results/clientpositive/spark/join14.q.out | 1894 --
.../test/results/clientpositive/spark/join15.q.out | 1136 -
.../test/results/clientpositive/spark/join16.q.out | 87 -
.../test/results/clientpositive/spark/join17.q.out | 1333 -
.../test/results/clientpositive/spark/join18.q.out | 492 -
.../spark/join18_multi_distinct.q.out | 500 -
.../test/results/clientpositive/spark/join19.q.out | 359 -
.../test/results/clientpositive/spark/join2.q.out | 642 -
.../test/results/clientpositive/spark/join20.q.out | 1366 -
.../test/results/clientpositive/spark/join21.q.out | 629 -
.../test/results/clientpositive/spark/join22.q.out | 128 -
.../test/results/clientpositive/spark/join23.q.out | 206 -
.../test/results/clientpositive/spark/join24.q.out | 29 -
.../test/results/clientpositive/spark/join25.q.out | 175 -
.../test/results/clientpositive/spark/join26.q.out | 507 -
.../test/results/clientpositive/spark/join27.q.out | 179 -
.../test/results/clientpositive/spark/join28.q.out | 289 -
.../test/results/clientpositive/spark/join29.q.out | 199 -
.../test/results/clientpositive/spark/join3.q.out | 2806 --
.../test/results/clientpositive/spark/join30.q.out | 173 -
.../test/results/clientpositive/spark/join31.q.out | 208 -
.../test/results/clientpositive/spark/join32.q.out | 491 -
.../clientpositive/spark/join32_lessSize.q.out | 2644 --
.../test/results/clientpositive/spark/join33.q.out | 491 -
.../test/results/clientpositive/spark/join34.q.out | 450 -
.../test/results/clientpositive/spark/join35.q.out | 481 -
.../test/results/clientpositive/spark/join36.q.out | 487 -
.../test/results/clientpositive/spark/join37.q.out | 175 -
.../test/results/clientpositive/spark/join38.q.out | 173 -
.../test/results/clientpositive/spark/join39.q.out | 693 -
.../test/results/clientpositive/spark/join4.q.out | 188 -
.../test/results/clientpositive/spark/join40.q.out | 3926 ---
.../test/results/clientpositive/spark/join41.q.out | 200 -
.../test/results/clientpositive/spark/join5.q.out | 186 -
.../test/results/clientpositive/spark/join6.q.out | 191 -
.../test/results/clientpositive/spark/join7.q.out | 236 -
.../test/results/clientpositive/spark/join8.q.out | 185 -
.../test/results/clientpositive/spark/join9.q.out | 1339 -
.../results/clientpositive/spark/join_1to1.q.out | 1352 -
.../clientpositive/spark/join_alt_syntax.q.out | 761 -
.../results/clientpositive/spark/join_array.q.out | 63 -
.../clientpositive/spark/join_casesensitive.q.out | 127 -
.../spark/join_cond_pushdown_1.q.out | 469 -
.../spark/join_cond_pushdown_2.q.out | 320 -
.../spark/join_cond_pushdown_3.q.out | 477 -
.../spark/join_cond_pushdown_4.q.out | 324 -
.../spark/join_cond_pushdown_unqual1.q.out | 549 -
.../spark/join_cond_pushdown_unqual2.q.out | 372 -
.../spark/join_cond_pushdown_unqual3.q.out | 557 -
.../spark/join_cond_pushdown_unqual4.q.out | 376 -
.../results/clientpositive/spark/join_empty.q.out | 56 -
.../spark/join_filters_overlap.q.out | 1865 --
.../clientpositive/spark/join_hive_626.q.out | 209 -
.../clientpositive/spark/join_literals.q.out | 37 -
.../clientpositive/spark/join_map_ppr.q.out | 1006 -
.../spark/join_merge_multi_expressions.q.out | 151 -
.../clientpositive/spark/join_merging.q.out | 269 -
.../clientpositive/spark/join_nullsafe.q.out | 1693 -
.../results/clientpositive/spark/join_rc.q.out | 1170 -
.../clientpositive/spark/join_reorder.q.out | 776 -
.../clientpositive/spark/join_reorder2.q.out | 404 -
.../clientpositive/spark/join_reorder3.q.out | 404 -
.../clientpositive/spark/join_reorder4.q.out | 399 -
.../results/clientpositive/spark/join_star.q.out | 1009 -
.../results/clientpositive/spark/join_thrift.q.out | 129 -
.../results/clientpositive/spark/join_vc.q.out | 415 -
.../results/clientpositive/spark/join_view.q.out | 157 -
.../spark/lateral_view_explode2.q.out | 115 -
.../spark/lateral_view_multi_lateralviews.q.out | 62 -
.../clientpositive/spark/leftsemijoin.q.out | 329 -
.../clientpositive/spark/leftsemijoin_mr.q.out | 94 -
.../spark/leftsemijoin_mr.q.out_spark | 94 -
.../clientpositive/spark/limit_pushdown.q.out | 1552 -
.../clientpositive/spark/limit_pushdown2.q.out | 1263 -
.../clientpositive/spark/list_bucket_dml_10.q.out | 237 -
.../clientpositive/spark/list_bucket_dml_2.q.out | 577 -
.../clientpositive/spark/load_dyn_part1.q.out | 2257 --
.../clientpositive/spark/load_dyn_part10.q.out | 1152 -
.../clientpositive/spark/load_dyn_part11.q.out | 2076 --
.../clientpositive/spark/load_dyn_part12.q.out | 2076 --
.../clientpositive/spark/load_dyn_part13.q.out | 215 -
.../clientpositive/spark/load_dyn_part14.q.out | 199 -
.../clientpositive/spark/load_dyn_part15.q.out | 40 -
.../clientpositive/spark/load_dyn_part2.q.out | 2158 --
.../clientpositive/spark/load_dyn_part3.q.out | 2158 --
.../clientpositive/spark/load_dyn_part4.q.out | 4195 ---
.../clientpositive/spark/load_dyn_part5.q.out | 1056 -
.../clientpositive/spark/load_dyn_part6.q.out | 2076 --
.../clientpositive/spark/load_dyn_part7.q.out | 564 -
.../clientpositive/spark/load_dyn_part8.q.out | 2738 --
.../clientpositive/spark/load_dyn_part9.q.out | 1152 -
.../results/clientpositive/spark/load_fs2.q.out | 118 -
.../load_hdfs_file_with_space_in_the_name.q.out | 25 -
.../clientpositive/spark/louter_join_ppr.q.out | 1348 -
.../results/clientpositive/spark/mapjoin1.q.out | 655 -
.../clientpositive/spark/mapjoin_addjar.q.out | 1060 -
.../clientpositive/spark/mapjoin_decimal.q.out | 405 -
.../clientpositive/spark/mapjoin_distinct.q.out | 598 -
.../spark/mapjoin_filter_on_outerjoin.q.out | 395 -
.../clientpositive/spark/mapjoin_mapjoin.q.out | 873 -
.../clientpositive/spark/mapjoin_memcheck.q.out | 145 -
.../clientpositive/spark/mapjoin_subquery.q.out | 514 -
.../clientpositive/spark/mapjoin_subquery2.q.out | 238 -
.../clientpositive/spark/mapjoin_test_outer.q.out | 1408 -
.../results/clientpositive/spark/mapreduce1.q.out | 623 -
.../results/clientpositive/spark/mapreduce2.q.out | 618 -
.../test/results/clientpositive/spark/merge1.q.out | 755 -
.../test/results/clientpositive/spark/merge2.q.out | 755 -
.../results/clientpositive/spark/mergejoins.q.out | 334 -
.../clientpositive/spark/mergejoins_mixed.q.out | 1800 --
.../spark/metadata_only_queries.q.out | 826 -
.../spark/metadata_only_queries_with_filters.q.out | 240 -
.../clientpositive/spark/multi_insert.q.out | 2409 --
.../clientpositive/spark/multi_insert_gby.q.out | 718 -
.../clientpositive/spark/multi_insert_gby2.q.out | 166 -
.../clientpositive/spark/multi_insert_gby3.q.out | 2022 --
.../spark/multi_insert_lateral_view.q.out | 1564 -
.../clientpositive/spark/multi_insert_mixed.q.out | 905 -
...ulti_insert_move_tasks_share_dependencies.q.out | 3881 ---
.../spark/multi_insert_with_join.q.out | 128 -
.../clientpositive/spark/multi_join_union.q.out | 204 -
.../spark/multigroupby_singlemr.q.out | 822 -
.../results/clientpositive/spark/nullgroup.q.out | 292 -
.../results/clientpositive/spark/nullgroup2.q.out | 322 -
.../results/clientpositive/spark/nullgroup4.q.out | 346 -
.../spark/nullgroup4_multi_distinct.q.out | 148 -
.../clientpositive/spark/optimize_nullscan.q.out | 1992 --
.../results/clientpositive/spark/orc_merge1.q.out | 486 -
.../results/clientpositive/spark/orc_merge2.q.out | 287 -
.../results/clientpositive/spark/orc_merge3.q.out | 216 -
.../results/clientpositive/spark/orc_merge4.q.out | 240 -
.../results/clientpositive/spark/orc_merge5.q.out | 337 -
.../results/clientpositive/spark/orc_merge6.q.out | 512 -
.../results/clientpositive/spark/orc_merge7.q.out | 648 -
.../results/clientpositive/spark/orc_merge8.q.out | 133 -
.../results/clientpositive/spark/orc_merge9.q.out | 188 -
.../clientpositive/spark/orc_merge_diff_fs.q.out | 530 -
.../clientpositive/spark/orc_merge_incompat1.q.out | 244 -
.../clientpositive/spark/orc_merge_incompat2.q.out | 400 -
.../test/results/clientpositive/spark/order.q.out | 160 -
.../test/results/clientpositive/spark/order2.q.out | 15 -
.../clientpositive/spark/outer_join_ppr.q.out | 674 -
.../results/clientpositive/spark/parallel.q.out | 1446 -
.../clientpositive/spark/parallel_join0.q.out | 244 -
.../clientpositive/spark/parallel_join1.q.out | 1158 -
.../clientpositive/spark/parallel_orderby.q.out | 312 -
.../clientpositive/spark/parquet_join.q.out | 369 -
.../spark/parquet_vectorization_0.q.out | 31106 ------------------
.../spark/parquet_vectorization_1.q.out | 209 -
.../spark/parquet_vectorization_10.q.out | 410 -
.../spark/parquet_vectorization_11.q.out | 174 -
.../spark/parquet_vectorization_12.q.out | 819 -
.../spark/parquet_vectorization_13.q.out | 672 -
.../spark/parquet_vectorization_14.q.out | 912 -
.../spark/parquet_vectorization_15.q.out | 299 -
.../spark/parquet_vectorization_16.q.out | 732 -
.../spark/parquet_vectorization_17.q.out | 550 -
.../spark/parquet_vectorization_2.q.out | 217 -
.../spark/parquet_vectorization_3.q.out | 227 -
.../spark/parquet_vectorization_4.q.out | 217 -
.../spark/parquet_vectorization_5.q.out | 204 -
.../spark/parquet_vectorization_6.q.out | 1730 -
.../spark/parquet_vectorization_7.q.out | 476 -
.../spark/parquet_vectorization_8.q.out | 450 -
.../spark/parquet_vectorization_9.q.out | 732 -
.../spark/parquet_vectorization_decimal_date.q.out | 109 -
.../spark/parquet_vectorization_div0.q.out | 624 -
.../spark/parquet_vectorization_limit.q.out | 891 -
.../spark/parquet_vectorization_nested_udf.q.out | 9 -
.../spark/parquet_vectorization_not.q.out | 58 -
.../spark/parquet_vectorization_offset_limit.q.out | 195 -
.../spark/parquet_vectorization_part.q.out | 72 -
.../spark/parquet_vectorization_part_project.q.out | 152 -
.../spark/parquet_vectorization_part_varchar.q.out | 72 -
.../spark/parquet_vectorization_pushdown.q.out | 100 -
ql/src/test/results/clientpositive/spark/pcr.q.out | 5231 ----
.../clientpositive/spark/ppd_gby_join.q.out | 548 -
.../results/clientpositive/spark/ppd_join.q.out | 1080 -
.../results/clientpositive/spark/ppd_join2.q.out | 3388 --
.../results/clientpositive/spark/ppd_join3.q.out | 3468 ---
.../results/clientpositive/spark/ppd_join4.q.out | 138 -
.../results/clientpositive/spark/ppd_join5.q.out | 325 -
.../clientpositive/spark/ppd_join_filter.q.out | 1286 -
.../clientpositive/spark/ppd_multi_insert.q.out | 2616 --
.../clientpositive/spark/ppd_outer_join1.q.out | 250 -
.../clientpositive/spark/ppd_outer_join2.q.out | 490 -
.../clientpositive/spark/ppd_outer_join3.q.out | 490 -
.../clientpositive/spark/ppd_outer_join4.q.out | 774 -
.../clientpositive/spark/ppd_outer_join5.q.out | 445 -
.../clientpositive/spark/ppd_transform.q.out | 453 -
ql/src/test/results/clientpositive/spark/ptf.q.out | 4987 ---
.../results/clientpositive/spark/ptf_decimal.q.out | 168 -
.../clientpositive/spark/ptf_general_queries.q.out | 93 -
.../clientpositive/spark/ptf_matchpath.q.out | 403 -
.../results/clientpositive/spark/ptf_rcfile.q.out | 86 -
.../clientpositive/spark/ptf_register_tblfn.q.out | 88 -
.../results/clientpositive/spark/ptf_seqfile.q.out | 86 -
.../clientpositive/spark/ptf_streaming.q.out | 2652 --
.../clientpositive/spark/quotedid_smb.q.out | 139 -
.../clientpositive/spark/rcfile_bigdata.q.out | 47 -
.../clientpositive/spark/reduce_deduplicate.q.out | 413 -
.../spark/reduce_deduplicate_exclude_join.q.out | 98 -
.../clientpositive/spark/remote_script.q.out | 49 -
.../spark/root_dir_external_table.q.out | 28 -
.../clientpositive/spark/router_join_ppr.q.out | 1348 -
.../spark/runtime_skewjoin_mapjoin_spark.q.out | 347 -
.../results/clientpositive/spark/sample1.q.out | 708 -
.../results/clientpositive/spark/sample10.q.out | 463 -
.../results/clientpositive/spark/sample2.q.out | 688 -
.../results/clientpositive/spark/sample3.q.out | 257 -
.../results/clientpositive/spark/sample4.q.out | 688 -
.../results/clientpositive/spark/sample5.q.out | 411 -
.../results/clientpositive/spark/sample6.q.out | 3349 --
.../results/clientpositive/spark/sample7.q.out | 420 -
.../results/clientpositive/spark/sample8.q.out | 945 -
.../results/clientpositive/spark/sample9.q.out | 540 -
.../clientpositive/spark/schemeAuthority.q.out | 84 -
.../clientpositive/spark/schemeAuthority2.q.out | 53 -
.../clientpositive/spark/script_env_var1.q.out | 14 -
.../clientpositive/spark/script_env_var2.q.out | 14 -
.../results/clientpositive/spark/script_pipe.q.out | 132 -
.../results/clientpositive/spark/scriptfile1.q.out | 47 -
.../clientpositive/spark/scriptfile1.q.out_spark | 47 -
.../results/clientpositive/spark/semijoin.q.out | 3289 --
.../results/clientpositive/spark/skewjoin.q.out | 1540 -
.../clientpositive/spark/skewjoin_noskew.q.out | 232 -
.../spark/skewjoin_union_remove_1.q.out | 726 -
.../spark/skewjoin_union_remove_2.q.out | 253 -
.../clientpositive/spark/skewjoinopt1.q.out | 694 -
.../clientpositive/spark/skewjoinopt10.q.out | 241 -
.../clientpositive/spark/skewjoinopt11.q.out | 265 -
.../clientpositive/spark/skewjoinopt12.q.out | 188 -
.../clientpositive/spark/skewjoinopt13.q.out | 207 -
.../clientpositive/spark/skewjoinopt14.q.out | 261 -
.../clientpositive/spark/skewjoinopt15.q.out | 726 -
.../clientpositive/spark/skewjoinopt16.q.out | 188 -
.../clientpositive/spark/skewjoinopt17.q.out | 399 -
.../clientpositive/spark/skewjoinopt18.q.out | 163 -
.../clientpositive/spark/skewjoinopt19.q.out | 195 -
.../clientpositive/spark/skewjoinopt2.q.out | 705 -
.../clientpositive/spark/skewjoinopt20.q.out | 195 -
.../clientpositive/spark/skewjoinopt3.q.out | 356 -
.../clientpositive/spark/skewjoinopt4.q.out | 352 -
.../clientpositive/spark/skewjoinopt5.q.out | 195 -
.../clientpositive/spark/skewjoinopt6.q.out | 195 -
.../clientpositive/spark/skewjoinopt7.q.out | 253 -
.../clientpositive/spark/skewjoinopt8.q.out | 251 -
.../clientpositive/spark/skewjoinopt9.q.out | 312 -
.../clientpositive/spark/smb_mapjoin_1.q.out | 632 -
.../clientpositive/spark/smb_mapjoin_10.q.out | 122 -
.../clientpositive/spark/smb_mapjoin_11.q.out | 2092 --
.../clientpositive/spark/smb_mapjoin_12.q.out | 520 -
.../clientpositive/spark/smb_mapjoin_13.q.out | 496 -
.../clientpositive/spark/smb_mapjoin_14.q.out | 1448 -
.../clientpositive/spark/smb_mapjoin_15.q.out | 952 -
.../clientpositive/spark/smb_mapjoin_16.q.out | 117 -
.../clientpositive/spark/smb_mapjoin_17.q.out | 966 -
.../clientpositive/spark/smb_mapjoin_18.q.out | 477 -
.../clientpositive/spark/smb_mapjoin_19.q.out | 256 -
.../clientpositive/spark/smb_mapjoin_2.q.out | 596 -
.../clientpositive/spark/smb_mapjoin_20.q.out | 1419 -
.../clientpositive/spark/smb_mapjoin_21.q.out | 561 -
.../clientpositive/spark/smb_mapjoin_22.q.out | 282 -
.../clientpositive/spark/smb_mapjoin_25.q.out | 336 -
.../clientpositive/spark/smb_mapjoin_3.q.out | 592 -
.../clientpositive/spark/smb_mapjoin_4.q.out | 1216 -
.../clientpositive/spark/smb_mapjoin_5.q.out | 1216 -
.../clientpositive/spark/smb_mapjoin_6.q.out | 2612 --
.../clientpositive/spark/smb_mapjoin_7.q.out | 1237 -
.../clientpositive/spark/smb_mapjoin_8.q.out | 464 -
.../test/results/clientpositive/spark/sort.q.out | 1132 -
.../spark/spark_combine_equivalent_work.q.out | 115 -
.../spark/spark_combine_equivalent_work_2.q.out | 250 -
.../clientpositive/spark/spark_constprog_dpp.q.out | 155 -
.../spark/spark_dynamic_partition_pruning.q.out | 7395 -----
.../spark/spark_dynamic_partition_pruning_2.q.out | 1235 -
.../spark/spark_dynamic_partition_pruning_3.q.out | 1977 --
.../spark/spark_dynamic_partition_pruning_4.q.out | 2477 --
.../spark/spark_dynamic_partition_pruning_5.q.out | 370 -
.../spark/spark_dynamic_partition_pruning_6.q.out | 650 -
.../spark/spark_dynamic_partition_pruning_7.q.out | 387 -
...rk_dynamic_partition_pruning_mapjoin_only.q.out | 531 -
...namic_partition_pruning_recursive_mapjoin.q.out | 1245 -
.../spark/spark_explain_groupbyshuffle.q.out | 132 -
.../clientpositive/spark/spark_explainuser_1.q.out | 6533 ----
.../spark/spark_in_process_launcher.q.out | 103 -
.../clientpositive/spark/spark_local_queries.q.out | 138 -
.../spark_multi_insert_parallel_orderby.q.out | 2051 --
.../spark/spark_opt_shuffle_serde.q.out | 216 -
.../clientpositive/spark/spark_union_merge.q.out | 618 -
.../clientpositive/spark/spark_use_op_stats.q.out | 349 -
.../spark/spark_use_ts_stats_for_mapjoin.q.out | 634 -
...park_vectorized_dynamic_partition_pruning.q.out | 13612 --------
.../test/results/clientpositive/spark/stats0.q.out | 2654 --
.../test/results/clientpositive/spark/stats1.q.out | 243 -
.../results/clientpositive/spark/stats10.q.out | 534 -
.../results/clientpositive/spark/stats12.q.out | 374 -
.../results/clientpositive/spark/stats13.q.out | 365 -
.../results/clientpositive/spark/stats14.q.out | 317 -
.../results/clientpositive/spark/stats15.q.out | 317 -
.../results/clientpositive/spark/stats16.q.out | 103 -
.../results/clientpositive/spark/stats18.q.out | 113 -
.../test/results/clientpositive/spark/stats2.q.out | 257 -
.../test/results/clientpositive/spark/stats3.q.out | 261 -
.../test/results/clientpositive/spark/stats5.q.out | 81 -
.../test/results/clientpositive/spark/stats6.q.out | 245 -
.../test/results/clientpositive/spark/stats7.q.out | 201 -
.../test/results/clientpositive/spark/stats8.q.out | 724 -
.../test/results/clientpositive/spark/stats9.q.out | 87 -
.../clientpositive/spark/stats_noscan_1.q.out | 498 -
.../clientpositive/spark/stats_noscan_2.q.out | 380 -
.../clientpositive/spark/stats_only_null.q.out | 467 -
.../results/clientpositive/spark/statsfs.q.out | 481 -
.../clientpositive/spark/subquery_exists.q.out | 1420 -
.../results/clientpositive/spark/subquery_in.q.out | 6157 ----
.../clientpositive/spark/subquery_multi.q.out | 4797 ---
.../spark/subquery_multiinsert.q.out | 937 -
.../spark/subquery_nested_subquery.q.out | 38 -
.../clientpositive/spark/subquery_notin.q.out | 8336 -----
.../clientpositive/spark/subquery_null_agg.q.out | 186 -
.../clientpositive/spark/subquery_scalar.q.out | 7399 -----
.../clientpositive/spark/subquery_select.q.out | 5936 ----
.../spark/subquery_shared_alias.q.out | 23 -
.../clientpositive/spark/subquery_views.q.out | 612 -
.../spark/table_access_keys_stats.q.out | 581 -
.../results/clientpositive/spark/temp_table.q.out | 597 -
.../clientpositive/spark/temp_table_external.q.out | 34 -
.../clientpositive/spark/temp_table_gb1.q.out | 65 -
.../clientpositive/spark/temp_table_join1.q.out | 378 -
.../clientpositive/spark/tez_join_tests.q.out | 2227 --
.../clientpositive/spark/tez_joins_explain.q.out | 717 -
.../results/clientpositive/spark/timestamp_1.q.out | 526 -
.../results/clientpositive/spark/timestamp_2.q.out | 526 -
.../results/clientpositive/spark/timestamp_3.q.out | 120 -
.../spark/timestamp_comparison.q.out | 99 -
.../clientpositive/spark/timestamp_lazy.q.out | 49 -
.../clientpositive/spark/timestamp_null.q.out | 38 -
.../clientpositive/spark/timestamp_udf.q.out | 216 -
.../results/clientpositive/spark/transform1.q.out | 150 -
.../results/clientpositive/spark/transform2.q.out | 9 -
.../clientpositive/spark/transform_ppr1.q.out | 523 -
.../clientpositive/spark/transform_ppr2.q.out | 420 -
.../spark/truncate_column_buckets.q.out | 58 -
.../results/clientpositive/spark/uber_reduce.q.out | 25 -
.../clientpositive/spark/udaf_collect_set.q.out | 216 -
.../clientpositive/spark/udf_example_add.q.out | 76 -
.../results/clientpositive/spark/udf_in_file.q.out | 97 -
.../results/clientpositive/spark/udf_max.q.out | 64 -
.../results/clientpositive/spark/udf_min.q.out | 64 -
.../clientpositive/spark/udf_percentile.q.out | 442 -
.../test/results/clientpositive/spark/union.q.out | 594 -
.../results/clientpositive/spark/union10.q.out | 179 -
.../results/clientpositive/spark/union11.q.out | 173 -
.../results/clientpositive/spark/union12.q.out | 225 -
.../results/clientpositive/spark/union13.q.out | 1072 -
.../results/clientpositive/spark/union14.q.out | 157 -
.../results/clientpositive/spark/union15.q.out | 161 -
.../results/clientpositive/spark/union16.q.out | 189 -
.../results/clientpositive/spark/union17.q.out | 896 -
.../results/clientpositive/spark/union18.q.out | 1201 -
.../results/clientpositive/spark/union19.q.out | 1044 -
.../test/results/clientpositive/spark/union2.q.out | 77 -
.../results/clientpositive/spark/union20.q.out | 209 -
.../results/clientpositive/spark/union21.q.out | 760 -
.../results/clientpositive/spark/union22.q.out | 1512 -
.../results/clientpositive/spark/union23.q.out | 1110 -
.../results/clientpositive/spark/union24.q.out | 1564 -
.../results/clientpositive/spark/union25.q.out | 221 -
.../results/clientpositive/spark/union26.q.out | 1283 -
.../results/clientpositive/spark/union27.q.out | 169 -
.../results/clientpositive/spark/union28.q.out | 208 -
.../results/clientpositive/spark/union29.q.out | 187 -
.../test/results/clientpositive/spark/union3.q.out | 213 -
.../results/clientpositive/spark/union30.q.out | 258 -
.../results/clientpositive/spark/union31.q.out | 912 -
.../results/clientpositive/spark/union32.q.out | 770 -
.../results/clientpositive/spark/union33.q.out | 340 -
.../results/clientpositive/spark/union34.q.out | 430 -
.../test/results/clientpositive/spark/union4.q.out | 145 -
.../test/results/clientpositive/spark/union5.q.out | 137 -
.../test/results/clientpositive/spark/union6.q.out | 162 -
.../test/results/clientpositive/spark/union7.q.out | 153 -
.../test/results/clientpositive/spark/union8.q.out | 1593 -
.../test/results/clientpositive/spark/union9.q.out | 81 -
.../results/clientpositive/spark/union_date.q.out | 138 -
.../clientpositive/spark/union_date_trim.q.out | 52 -
.../clientpositive/spark/union_lateralview.q.out | 291 -
.../results/clientpositive/spark/union_null.q.out | 86 -
.../results/clientpositive/spark/union_ppr.q.out | 702 -
.../clientpositive/spark/union_remove_1.q.out | 199 -
.../clientpositive/spark/union_remove_10.q.out | 282 -
.../clientpositive/spark/union_remove_11.q.out | 270 -
.../clientpositive/spark/union_remove_12.q.out | 270 -
.../clientpositive/spark/union_remove_13.q.out | 294 -
.../clientpositive/spark/union_remove_14.q.out | 270 -
.../clientpositive/spark/union_remove_15.q.out | 238 -
.../clientpositive/spark/union_remove_16.q.out | 280 -
.../clientpositive/spark/union_remove_17.q.out | 230 -
.../clientpositive/spark/union_remove_18.q.out | 280 -
.../clientpositive/spark/union_remove_19.q.out | 491 -
.../clientpositive/spark/union_remove_2.q.out | 232 -
.../clientpositive/spark/union_remove_20.q.out | 207 -
.../clientpositive/spark/union_remove_21.q.out | 193 -
.../clientpositive/spark/union_remove_22.q.out | 361 -
.../clientpositive/spark/union_remove_23.q.out | 268 -
.../clientpositive/spark/union_remove_24.q.out | 207 -
.../clientpositive/spark/union_remove_25.q.out | 579 -
.../clientpositive/spark/union_remove_3.q.out | 220 -
.../clientpositive/spark/union_remove_4.q.out | 249 -
.../clientpositive/spark/union_remove_5.q.out | 282 -
.../clientpositive/spark/union_remove_6.q.out | 226 -
.../clientpositive/spark/union_remove_6_subq.q.out | 1153 -
.../clientpositive/spark/union_remove_7.q.out | 199 -
.../clientpositive/spark/union_remove_8.q.out | 232 -
.../clientpositive/spark/union_remove_9.q.out | 290 -
.../clientpositive/spark/union_script.q.out | 1524 -
.../clientpositive/spark/union_top_level.q.out | 1133 -
.../results/clientpositive/spark/union_view.q.out | 1006 -
.../results/clientpositive/spark/uniquejoin.q.out | 174 -
.../clientpositive/spark/varchar_join1.q.out | 139 -
.../clientpositive/spark/vector_between_in.q.out | 1768 --
.../spark/vector_cast_constant.q.out | 294 -
.../clientpositive/spark/vector_char_4.q.out | 199 -
.../spark/vector_count_distinct.q.out | 1511 -
.../clientpositive/spark/vector_data_types.q.out | 464 -
.../spark/vector_decimal_aggregate.q.out | 842 -
.../spark/vector_decimal_mapjoin.q.out | 1877 --
.../clientpositive/spark/vector_distinct_2.q.out | 1922 --
.../results/clientpositive/spark/vector_elt.q.out | 225 -
.../clientpositive/spark/vector_groupby_3.q.out | 1927 --
.../clientpositive/spark/vector_inner_join.q.out | 1677 -
.../spark/vector_left_outer_join.q.out | 201 -
.../spark/vector_mapjoin_reduce.q.out | 545 -
.../clientpositive/spark/vector_orderby_5.q.out | 258 -
.../clientpositive/spark/vector_outer_join0.q.out | 400 -
.../clientpositive/spark/vector_outer_join1.q.out | 982 -
.../clientpositive/spark/vector_outer_join2.q.out | 530 -
.../clientpositive/spark/vector_outer_join3.q.out | 360 -
.../clientpositive/spark/vector_outer_join4.q.out | 818 -
.../clientpositive/spark/vector_outer_join5.q.out | 550 -
.../spark/vector_string_concat.q.out | 531 -
.../clientpositive/spark/vector_varchar_4.q.out | 199 -
.../clientpositive/spark/vectorization_0.q.out | 31166 -------------------
.../clientpositive/spark/vectorization_1.q.out | 224 -
.../clientpositive/spark/vectorization_10.q.out | 417 -
.../clientpositive/spark/vectorization_11.q.out | 181 -
.../clientpositive/spark/vectorization_12.q.out | 844 -
.../clientpositive/spark/vectorization_13.q.out | 696 -
.../clientpositive/spark/vectorization_14.q.out | 937 -
.../clientpositive/spark/vectorization_15.q.out | 308 -
.../clientpositive/spark/vectorization_16.q.out | 748 -
.../clientpositive/spark/vectorization_17.q.out | 566 -
.../clientpositive/spark/vectorization_2.q.out | 232 -
.../clientpositive/spark/vectorization_3.q.out | 242 -
.../clientpositive/spark/vectorization_4.q.out | 232 -
.../clientpositive/spark/vectorization_5.q.out | 219 -
.../clientpositive/spark/vectorization_6.q.out | 1737 --
.../clientpositive/spark/vectorization_9.q.out | 748 -
.../spark/vectorization_decimal_date.q.out | 109 -
.../clientpositive/spark/vectorization_div0.q.out | 885 -
.../vectorization_input_format_excludes.q.out | 1610 -
.../spark/vectorization_nested_udf.q.out | 135 -
.../clientpositive/spark/vectorization_not.q.out | 58 -
.../spark/vectorization_parquet_projection.q.out | 709 -
.../clientpositive/spark/vectorization_part.q.out | 72 -
.../spark/vectorization_part_project.q.out | 152 -
.../spark/vectorization_pushdown.q.out | 100 -
.../spark/vectorization_short_regress.q.out | 4482 ---
.../clientpositive/spark/vectorized_case.q.out | 1322 -
.../clientpositive/spark/vectorized_mapjoin.q.out | 216 -
.../spark/vectorized_math_funcs.q.out | 294 -
.../spark/vectorized_nested_mapjoin.q.out | 180 -
.../clientpositive/spark/vectorized_ptf.q.out | 6571 ----
.../spark/vectorized_rcfile_columnar.q.out | 56 -
.../spark/vectorized_shufflejoin.q.out | 204 -
.../spark/vectorized_string_funcs.q.out | 156 -
.../spark/vectorized_timestamp_funcs.q.out | 1404 -
.../results/clientpositive/spark/windowing.q.out | 2383 --
.../tez/vector_non_string_partition.q.out | 8 +-
.../results/clientpositive/vector_udf_trunc.q.out | 20 +-
service/pom.xml | 12 +
.../java/org/apache/hive/service/ServiceUtils.java | 3 +-
.../cli/SparkProgressMonitorStatusMapper.java | 52 -
.../hive/service/cli/thrift/ThriftCLIService.java | 4 -
.../apache/hive/service/server/HiveServer2.java | 12 -
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 63 -
.../hadoop/hive/shims/MiniSparkOnYARNCluster.java | 41 -
.../org/apache/hadoop/hive/shims/HadoopShims.java | 3 -
spark-client/pom.xml | 205 -
.../hive/spark/client/AbstractSparkClient.java | 604 -
.../org/apache/hive/spark/client/BaseProtocol.java | 225 -
.../java/org/apache/hive/spark/client/Job.java | 32 -
.../org/apache/hive/spark/client/JobContext.java | 67 -
.../apache/hive/spark/client/JobContextImpl.java | 84 -
.../org/apache/hive/spark/client/JobHandle.java | 104 -
.../apache/hive/spark/client/JobHandleImpl.java | 244 -
.../hive/spark/client/JobResultSerializer.java | 113 -
.../hive/spark/client/MetricsCollection.java | 320 -
.../apache/hive/spark/client/MonitorCallback.java | 30 -
.../org/apache/hive/spark/client/RemoteDriver.java | 572 -
.../org/apache/hive/spark/client/SparkClient.java | 120 -
.../hive/spark/client/SparkClientFactory.java | 103 -
.../hive/spark/client/SparkClientUtilities.java | 212 -
.../spark/client/SparkLauncherSparkClient.java | 220 -
.../hive/spark/client/SparkSubmitSparkClient.java | 263 -
.../hive/spark/client/metrics/DataReadMethod.java | 29 -
.../hive/spark/client/metrics/InputMetrics.java | 59 -
.../apache/hive/spark/client/metrics/Metrics.java | 158 -
.../hive/spark/client/metrics/OutputMetrics.java | 57 -
.../spark/client/metrics/ShuffleReadMetrics.java | 100 -
.../spark/client/metrics/ShuffleWriteMetrics.java | 66 -
.../hive/spark/client/rpc/KryoMessageCodec.java | 181 -
.../org/apache/hive/spark/client/rpc/README.md | 33 -
.../java/org/apache/hive/spark/client/rpc/Rpc.java | 513 -
.../hive/spark/client/rpc/RpcConfiguration.java | 219 -
.../hive/spark/client/rpc/RpcDispatcher.java | 198 -
.../apache/hive/spark/client/rpc/RpcException.java | 29 -
.../apache/hive/spark/client/rpc/RpcServer.java | 495 -
.../apache/hive/spark/client/rpc/SaslHandler.java | 114 -
.../apache/hive/spark/counter/SparkCounter.java | 109 -
.../hive/spark/counter/SparkCounterGroup.java | 85 -
.../apache/hive/spark/counter/SparkCounters.java | 159 -
.../apache/hive/spark/client/TestJobHandle.java | 104 -
.../hive/spark/client/TestJobResultSerializer.java | 157 -
.../hive/spark/client/TestMetricsCollection.java | 182 -
.../apache/hive/spark/client/TestSparkClient.java | 543 -
.../spark/client/TestSparkLauncherSparkClient.java | 94 -
.../spark/client/rpc/TestKryoMessageCodec.java | 233 -
.../org/apache/hive/spark/client/rpc/TestRpc.java | 490 -
spark-client/src/test/resources/log4j2.properties | 38 -
.../common/metrics/common/MetricsConstant.java | 2 -
.../hadoop/hive/metastore/conf/MetastoreConf.java | 2 +-
streaming/pom.xml | 6 +
1314 files changed, 3365 insertions(+), 600746 deletions(-)
diff --git a/README.md b/README.md
index 53a21151688..c31d6108292 100644
--- a/README.md
+++ b/README.md
@@ -16,8 +16,7 @@ using SQL. Built on top of Apache Hadoop (TM), it provides:
* Access to files stored either directly in Apache HDFS (TM) or in other
data storage systems such as Apache HBase (TM)
-* Query execution using Apache Hadoop MapReduce, Apache Tez
- or Apache Spark frameworks.
+* Query execution using Apache Hadoop MapReduce or Apache Tez frameworks.
Hive provides standard SQL functionality, including many of the later
2003 and 2011 features for analytics. These include OLAP functions,
@@ -26,18 +25,13 @@ extended with user code via user defined functions (UDFs), user defined
aggregates (UDAFs), and user defined table functions (UDTFs).
Hive users have a choice of 3 runtimes when executing SQL queries.
-Users can choose between Apache Hadoop MapReduce, Apache Tez or
-Apache Spark frameworks as their execution backend. MapReduce is a
+Users can choose between Apache Hadoop MapReduce or Apache Tez
+frameworks as their execution backend. MapReduce is a
mature framework that is proven at large scales. However, MapReduce
is a purely batch framework, and queries using it may experience
higher latencies (tens of seconds), even over small datasets. Apache
Tez is designed for interactive query, and has substantially reduced
-overheads versus MapReduce. Apache Spark is a cluster computing
-framework that's built outside of MapReduce, but on top of HDFS,
-with a notion of composable and transformable distributed collection
-of items called Resilient Distributed Dataset (RDD) which allows
-processing and analysis without traditional intermediate stages that
-MapReduce introduces.
+overheads versus MapReduce.
Users are free to switch back and forth between these frameworks
at any time. In each case, Hive is best suited for use cases
diff --git a/bin/hive b/bin/hive
index b31bc8562e7..cceb7bc4cdf 100755
--- a/bin/hive
+++ b/bin/hive
@@ -113,21 +113,6 @@ if [ -f "${HIVE_CONF_DIR}/hive-env.sh" ]; then
. "${HIVE_CONF_DIR}/hive-env.sh"
fi
-if [ "$SERVICE" != "beeline" ]; then
- if [[ -z "$SPARK_HOME" ]]
- then
- bin=`dirname "$0"`
- # many hadoop installs are in dir/{spark,hive,hadoop,..}
- if test -e $bin/../../spark; then
- sparkHome=$(readlink -f $bin/../../spark)
- if [[ -d $sparkHome ]]
- then
- export SPARK_HOME=$sparkHome
- fi
- fi
- fi
-fi
-
CLASSPATH="${TEZ_CONF_DIR:-/etc/tez/conf}:${HIVE_CONF_DIR}"
HIVE_LIB=${HIVE_HOME}/lib
diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java
index 081acb7ff21..6b9bedec70d 100644
--- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java
+++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/JsonParserFactory.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.common.jsonexplain;
-import org.apache.hadoop.hive.common.jsonexplain.spark.SparkJsonParser;
import org.apache.hadoop.hive.common.jsonexplain.tez.TezJsonParser;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -36,9 +35,6 @@ public class JsonParserFactory {
if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
return new TezJsonParser();
}
- if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
- return new SparkJsonParser();
- }
return null;
}
}
diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/spark/SparkJsonParser.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/spark/SparkJsonParser.java
deleted file mode 100644
index 9afa7582a21..00000000000
--- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/spark/SparkJsonParser.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common.jsonexplain.spark;
-
-import org.apache.hadoop.hive.common.jsonexplain.DagJsonParser;
-
-
-public class SparkJsonParser extends DagJsonParser {
-
- @Override
- public String mapEdgeType(String edgeName) {
- return edgeName;
- }
-
- @Override
- public String getFrameworkName() {
- return "Spark";
- }
-}
\ No newline at end of file
diff --git a/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java b/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java
index 5b99d1c3bdd..c9023fe7dd1 100644
--- a/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java
+++ b/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java
@@ -186,10 +186,6 @@ public class InPlaceUpdate {
inPlaceUpdates = HiveConf.getBoolVar(conf, HiveConf.ConfVars.TEZ_EXEC_INPLACE_PROGRESS);
}
- if (engine.equals("spark")) {
- inPlaceUpdates = HiveConf.getBoolVar(conf, HiveConf.ConfVars.SPARK_EXEC_INPLACE_PROGRESS);
- }
-
return inPlaceUpdates && isUnixTerminal();
}
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index c629e8f698c..99964fc7732 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -101,7 +101,6 @@ public class HiveConf extends Configuration {
private final List<String> rscList = new ArrayList<>();
private Pattern modWhiteListPattern = null;
- private volatile boolean isSparkConfigUpdated = false;
private static final int LOG_PREFIX_LENGTH = 64;
public enum ResultFileFormat {
@@ -157,14 +156,6 @@ public class HiveConf extends Configuration {
return ResultFileFormat.from(this.getVar(ConfVars.HIVEQUERYRESULTFILEFORMAT));
}
- public boolean getSparkConfigUpdated() {
- return isSparkConfigUpdated;
- }
-
- public void setSparkConfigUpdated(boolean isSparkConfigUpdated) {
- this.isSparkConfigUpdated = isSparkConfigUpdated;
- }
-
public interface EncoderDecoder<K, V> {
V encode(K key);
K decode(V value);
@@ -374,9 +365,6 @@ public class HiveConf extends Configuration {
public static final String HIVE_SERVER2_AUTHENTICATION_LDAP_USERMEMBERSHIPKEY_NAME =
"hive.server2.authentication.ldap.userMembershipKey";
- public static final String HIVE_SPARK_SUBMIT_CLIENT = "spark-submit";
- public static final String HIVE_SPARK_LAUNCHER_CLIENT = "spark-launcher";
-
/**
* dbVars are the parameters can be set per database. If these
* parameters are set as a database property, when switching to that
@@ -1765,8 +1753,6 @@ public class HiveConf extends Configuration {
"in another session. Currently there is no tag duplication check, user need to make sure his tag is unique. " +
"Also 'kill query' needs to be issued to all HiveServer2 instances to proper kill the queries"),
- HIVESPARKJOBNAMELENGTH("hive.spark.jobname.length", 100000, "max jobname length for Hive on " +
- "Spark queries"),
HIVEJOBNAMELENGTH("hive.jobname.length", 50, "max jobname length"),
// hive jar
@@ -1963,7 +1949,7 @@ public class HiveConf extends Configuration {
HIVEMAPJOINUSEOPTIMIZEDTABLE("hive.mapjoin.optimized.hashtable", true,
"Whether Hive should use memory-optimized hash table for MapJoin.\n" +
- "Only works on Tez and Spark, because memory-optimized hashtable cannot be serialized."),
+ "Only works on Tez because memory-optimized hashtable cannot be serialized."),
HIVEMAPJOINOPTIMIZEDTABLEPROBEPERCENT("hive.mapjoin.optimized.hashtable.probe.percent",
(float) 0.5, "Probing space percentage of the optimized hashtable"),
HIVEMAPJOINPARALELHASHTABLETHREADS("hive.mapjoin.hashtable.load.threads", 2,
@@ -2204,7 +2190,6 @@ public class HiveConf extends Configuration {
HIVEMERGEMAPREDFILES("hive.merge.mapredfiles", false,
"Merge small files at the end of a map-reduce job"),
HIVEMERGETEZFILES("hive.merge.tezfiles", false, "Merge small files at the end of a Tez DAG"),
- HIVEMERGESPARKFILES("hive.merge.sparkfiles", false, "Merge small files at the end of a Spark DAG Transformation"),
HIVEMERGEMAPFILESSIZE("hive.merge.size.per.task", (long) (256 * 1000 * 1000),
"Size of merged files at the end of the job"),
HIVEMERGEMAPFILESAVGSIZE("hive.merge.smallfiles.avgsize", (long) (16 * 1000 * 1000),
@@ -2431,20 +2416,13 @@ public class HiveConf extends Configuration {
"This number means how much memory the local task can take to hold the key/value into an in-memory hash table \n" +
"when this map join is followed by a group by. If the local task's memory usage is more than this number, \n" +
"the local task will abort by itself. It means the data of the small table is too large " +
- "to be held in memory. Does not apply to Hive-on-Spark (replaced by " +
- "hive.mapjoin.max.gc.time.percentage)"),
+ "to be held in memory."),
HIVEHASHTABLEMAXMEMORYUSAGE("hive.mapjoin.localtask.max.memory.usage", (float) 0.90,
"This number means how much memory the local task can take to hold the key/value into an in-memory hash table. \n" +
"If the local task's memory usage is more than this number, the local task will abort by itself. \n" +
- "It means the data of the small table is too large to be held in memory. Does not apply to " +
- "Hive-on-Spark (replaced by hive.mapjoin.max.gc.time.percentage)"),
+ "It means the data of the small table is too large to be held in memory."),
HIVEHASHTABLESCALE("hive.mapjoin.check.memory.rows", (long)100000,
"The number means after how many rows processed it needs to check the memory usage"),
- HIVEHASHTABLEMAXGCTIMEPERCENTAGE("hive.mapjoin.max.gc.time.percentage", (float) 0.60,
- new RangeValidator(0.0f, 1.0f), "This number means how much time (what percentage, " +
- "0..1, of wallclock time) the JVM is allowed to spend in garbage collection when running " +
- "the local task. If GC time percentage exceeds this number, the local task will abort by " +
- "itself. Applies to Hive-on-Spark only"),
HIVEINPUTFORMAT("hive.input.format", "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat",
"The default input format. Set this to HiveInputFormat if you encounter problems with CombineHiveInputFormat."),
@@ -2465,8 +2443,7 @@ public class HiveConf extends Configuration {
"Whether to grant access to the hs2/hive user for queries"),
HIVEQUERYNAME ("hive.query.name", null,
"This named is used by Tez to set the dag name. This name in turn will appear on \n" +
- "the Tez UI representing the work that was done. Used by Spark to set the query name, will show up in the\n" +
- "Spark UI."),
+ "the Tez UI representing the work that was done."),
HIVETEZJOBNAME("tez.job.name", "HIVE-%s",
"This named is used by Tez to set the job name. This name in turn will appear on \n" +
"the Yarn UI representing the Yarn Application Name. And The job name may be a \n" +
@@ -2700,9 +2677,6 @@ public class HiveConf extends Configuration {
"Analyzes and merges equiv downstream operators after a successful shared work optimization step."),
HIVE_SHARED_WORK_PARALLEL_EDGE_SUPPORT("hive.optimize.shared.work.parallel.edge.support", true,
"Lets the shared work optimizer to create parallel edges in case they are for semijoins or mapjoins."),
- HIVE_COMBINE_EQUIVALENT_WORK_OPTIMIZATION("hive.combine.equivalent.work.optimization", true, "Whether to " +
- "combine equivalent work objects during physical optimization.\n This optimization looks for equivalent " +
- "work objects and combines them if they meet certain preconditions. Spark only."),
HIVE_REMOVE_SQ_COUNT_CHECK("hive.optimize.remove.sq_count_check", true,
"Whether to remove an extra join with sq_count_check for scalar subqueries "
+ "with constant group by keys."),
@@ -3545,9 +3519,6 @@ public class HiveConf extends Configuration {
"hive.test.authz.sstd.hs2.mode", false, "test hs2 mode from .q tests", true),
HIVE_AUTHORIZATION_ENABLED("hive.security.authorization.enabled", false,
"enable or disable the Hive client authorization"),
- HIVE_AUTHORIZATION_ENABLED_ON_SPARK_VIEWS("hive.security.authorization.enabled.on.spark.views",true,
- "The configuration is gives the flexibility to admin to user whether to turn on/off the authorization model" +
- "introduced in HIVE-24026"),
HIVE_AUTHORIZATION_KERBEROS_USE_SHORTNAME("hive.security.authorization.kerberos.use.shortname", true,
"use short name in Kerberos cluster"),
HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager",
@@ -3691,13 +3662,6 @@ public class HiveConf extends Configuration {
HIVE_EXPLAIN_USER("hive.explain.user", true,
"Whether to show explain result at user level.\n" +
"When enabled, will log EXPLAIN output for the query at user level. Tez only."),
- HIVE_SPARK_EXPLAIN_USER("hive.spark.explain.user", false,
- "Whether to show explain result at user level.\n" +
- "When enabled, will log EXPLAIN output for the query at user level. Spark only."),
- HIVE_SPARK_LOG_EXPLAIN_WEBUI("hive.spark.log.explain.webui", true, "Whether to show the " +
- "explain plan in the Spark Web UI. Only shows the regular EXPLAIN plan, and ignores " +
- "any extra EXPLAIN configuration (e.g. hive.spark.explain.user, etc.). The explain " +
- "plan for each stage is truncated at 100,000 characters."),
// prefix used to auto generated column aliases (this should be started with '_')
HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c",
@@ -4403,7 +4367,7 @@ public class HiveConf extends Configuration {
HIVE_SERVER2_JOB_CREDENTIAL_PROVIDER_PATH("hive.server2.job.credential.provider.path", "",
"If set, this configuration property should provide a comma-separated list of URLs that indicates the type and " +
"location of providers to be used by hadoop credential provider API. It provides HiveServer2 the ability to provide job-specific " +
- "credential providers for jobs run using Tez, MR, Spark execution engines."),
+ "credential providers for jobs run using Tez, MR execution engines."),
HIVE_MOVE_FILES_THREAD_COUNT("hive.mv.files.thread", 15, new SizeValidator(0L, true, 1024L, true), "Number of threads"
+ " used to move files in move task. Set it to 0 to disable multi-threaded file moves. This parameter is also used by"
+ " MSCK to check tables."),
@@ -4467,8 +4431,8 @@ public class HiveConf extends Configuration {
HIVE_DECODE_PARTITION_NAME("hive.decode.partition.name", false,
"Whether to show the unquoted partition names in query results."),
- HIVE_EXECUTION_ENGINE("hive.execution.engine", "mr", new StringSet(true, "mr", "tez", "spark"),
- "Chooses execution engine. Options are: mr (Map reduce, default), tez, spark. While MR\n" +
+ HIVE_EXECUTION_ENGINE("hive.execution.engine", "mr", new StringSet(true, "mr", "tez"),
+ "Chooses execution engine. Options are: mr (Map reduce, default), tez. While MR\n" +
"remains the default engine for historical reasons, it is itself a historical engine\n" +
"and is deprecated in Hive 2 line. It may be removed without further warning."),
@@ -4639,11 +4603,8 @@ public class HiveConf extends Configuration {
"Whether to generate the splits locally or in the AM (tez only)"),
HIVE_TEZ_GENERATE_CONSISTENT_SPLITS("hive.tez.input.generate.consistent.splits", true,
"Whether to generate consistent split locations when generating splits in the AM"),
- HIVE_PREWARM_ENABLED("hive.prewarm.enabled", false, "Enables container prewarm for Tez/Spark (Hadoop 2 only)"),
- HIVE_PREWARM_NUM_CONTAINERS("hive.prewarm.numcontainers", 10, "Controls the number of containers to prewarm for Tez/Spark (Hadoop 2 only)"),
- HIVE_PREWARM_SPARK_TIMEOUT("hive.prewarm.spark.timeout", "5000ms",
- new TimeValidator(TimeUnit.MILLISECONDS),
- "Time to wait to finish prewarming spark executors"),
+ HIVE_PREWARM_ENABLED("hive.prewarm.enabled", false, "Enables container prewarm for Tez(Hadoop 2 only)"),
+ HIVE_PREWARM_NUM_CONTAINERS("hive.prewarm.numcontainers", 10, "Controls the number of containers to prewarm for Tez (Hadoop 2 only)"),
HIVESTAGEIDREARRANGE("hive.stageid.rearrange", "none", new StringSet("none", "idonly", "traverse", "execution"), ""),
HIVEEXPLAINDEPENDENCYAPPENDTASKTYPES("hive.explain.dependency.append.tasktype", false, ""),
HIVEUSEGOOGLEREGEXENGINE("hive.use.googleregex.engine",false,"whether to use google regex engine or not, default regex engine is java.util.regex"),
@@ -4798,11 +4759,9 @@ public class HiveConf extends Configuration {
"hive.server2.in.place.progress",
true,
"Allows hive server 2 to send progress bar update information. This is currently available"
- + " only if the execution engine is tez or Spark."),
+ + " only if the execution engine is tez."),
TEZ_DAG_STATUS_CHECK_INTERVAL("hive.tez.dag.status.check.interval", "500ms",
new TimeValidator(TimeUnit.MILLISECONDS), "Interval between subsequent DAG status invocation."),
- SPARK_EXEC_INPLACE_PROGRESS("hive.spark.exec.inplace.progress", true,
- "Updates spark job execution progress in-place in the terminal."),
TEZ_CONTAINER_MAX_JAVA_HEAP_FRACTION("hive.tez.container.max.java.heap.fraction", 0.8f,
"This is to override the tez setting with the same name"),
TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION_MIN("hive.tez.task.scale.memory.reserve-fraction.min",
@@ -5393,79 +5352,6 @@ public class HiveConf extends Configuration {
"validated for all SQL operations after every defined interval (default: 500ms) and corresponding action\n" +
"defined in the trigger will be taken"),
- SPARK_USE_OP_STATS("hive.spark.use.op.stats", true,
- "Whether to use operator stats to determine reducer parallelism for Hive on Spark.\n" +
- "If this is false, Hive will use source table stats to determine reducer\n" +
- "parallelism for all first level reduce tasks, and the maximum reducer parallelism\n" +
- "from all parents for all the rest (second level and onward) reducer tasks."),
- SPARK_USE_TS_STATS_FOR_MAPJOIN("hive.spark.use.ts.stats.for.mapjoin", false,
- "If this is set to true, mapjoin optimization in Hive/Spark will use statistics from\n" +
- "TableScan operators at the root of operator tree, instead of parent ReduceSink\n" +
- "operators of the Join operator."),
- SPARK_OPTIMIZE_SHUFFLE_SERDE("hive.spark.optimize.shuffle.serde", true,
- "If this is set to true, Hive on Spark will register custom serializers for data types\n" +
- "in shuffle. This should result in less shuffled data."),
- SPARK_CLIENT_FUTURE_TIMEOUT("hive.spark.client.future.timeout",
- "60s", new TimeValidator(TimeUnit.SECONDS),
- "Timeout for requests between Hive client and remote Spark driver."),
- SPARK_JOB_MONITOR_TIMEOUT("hive.spark.job.monitor.timeout",
- "60s", new TimeValidator(TimeUnit.SECONDS),
- "Timeout for job monitor to get Spark job state."),
- SPARK_RPC_CLIENT_CONNECT_TIMEOUT("hive.spark.client.connect.timeout",
- "1000ms", new TimeValidator(TimeUnit.MILLISECONDS),
- "Timeout for remote Spark driver in connecting back to Hive client."),
- SPARK_RPC_CLIENT_HANDSHAKE_TIMEOUT("hive.spark.client.server.connect.timeout",
- "90000ms", new TimeValidator(TimeUnit.MILLISECONDS),
- "Timeout for handshake between Hive client and remote Spark driver. Checked by both processes."),
- SPARK_RPC_SECRET_RANDOM_BITS("hive.spark.client.secret.bits", "256",
- "Number of bits of randomness in the generated secret for communication between Hive client and remote Spark driver. " +
- "Rounded down to the nearest multiple of 8."),
- SPARK_RPC_MAX_THREADS("hive.spark.client.rpc.threads", 8,
- "Maximum number of threads for remote Spark driver's RPC event loop."),
- SPARK_RPC_MAX_MESSAGE_SIZE("hive.spark.client.rpc.max.size", 50 * 1024 * 1024,
- "Maximum message size in bytes for communication between Hive client and remote Spark driver. Default is 50MB."),
- SPARK_RPC_CHANNEL_LOG_LEVEL("hive.spark.client.channel.log.level", null,
- "Channel logging level for remote Spark driver. One of {DEBUG, ERROR, INFO, TRACE, WARN}."),
- SPARK_RPC_SASL_MECHANISM("hive.spark.client.rpc.sasl.mechanisms", "DIGEST-MD5",
- "Name of the SASL mechanism to use for authentication."),
- SPARK_RPC_SERVER_ADDRESS("hive.spark.client.rpc.server.address", "",
- "The server address of HiverServer2 host to be used for communication between Hive client and remote Spark driver. " +
- "Default is empty, which means the address will be determined in the same way as for hive.server2.thrift.bind.host." +
- "This is only necessary if the host has multiple network addresses and if a different network address other than " +
- "hive.server2.thrift.bind.host is to be used."),
- SPARK_RPC_SERVER_PORT("hive.spark.client.rpc.server.port", "", "A list of port ranges which can be used by RPC server " +
- "with the format of 49152-49222,49228 and a random one is selected from the list. Default is empty, which randomly " +
- "selects one port from all available ones."),
- SPARK_DYNAMIC_PARTITION_PRUNING(
- "hive.spark.dynamic.partition.pruning", false,
- "When dynamic pruning is enabled, joins on partition keys will be processed by writing\n" +
- "to a temporary HDFS file, and read later for removing unnecessary partitions."),
- SPARK_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE(
- "hive.spark.dynamic.partition.pruning.max.data.size", 100*1024*1024L,
- "Maximum total data size in dynamic pruning."),
- SPARK_DYNAMIC_PARTITION_PRUNING_MAP_JOIN_ONLY(
- "hive.spark.dynamic.partition.pruning.map.join.only", false,
- "Turn on dynamic partition pruning only for map joins.\n" +
- "If hive.spark.dynamic.partition.pruning is set to true, this parameter value is ignored."),
- SPARK_USE_GROUPBY_SHUFFLE(
- "hive.spark.use.groupby.shuffle", true,
- "Spark groupByKey transformation has better performance but uses unbounded memory." +
- "Turn this off when there is a memory issue."),
- SPARK_JOB_MAX_TASKS("hive.spark.job.max.tasks", -1, "The maximum number of tasks a Spark job may have.\n" +
- "If a Spark job contains more tasks than the maximum, it will be cancelled. A value of -1 means no limit."),
- SPARK_STAGE_MAX_TASKS("hive.spark.stage.max.tasks", -1, "The maximum number of tasks a stage in a Spark job may have.\n" +
- "If a Spark job stage contains more tasks than the maximum, the job will be cancelled. A value of -1 means no limit."),
- SPARK_CLIENT_TYPE("hive.spark.client.type", HIVE_SPARK_SUBMIT_CLIENT,
- "Controls how the Spark application is launched. If " + HIVE_SPARK_SUBMIT_CLIENT + " is " +
- "specified (default) then the spark-submit shell script is used to launch the Spark " +
- "app. If " + HIVE_SPARK_LAUNCHER_CLIENT + " is specified then Spark's " +
- "InProcessLauncher is used to programmatically launch the app."),
- SPARK_SESSION_TIMEOUT("hive.spark.session.timeout", "30m", new TimeValidator(TimeUnit.MINUTES,
- 30L, true, null, true), "Amount of time the Spark Remote Driver should wait for " +
- " a Spark job to be submitted before shutting down. Minimum value is 30 minutes"),
- SPARK_SESSION_TIMEOUT_PERIOD("hive.spark.session.timeout.period", "60s",
- new TimeValidator(TimeUnit.SECONDS, 60L, true, null, true),
- "How frequently to check for idle Spark sessions. Minimum value is 60 seconds."),
NWAYJOINREORDER("hive.reorder.nway.joins", true,
"Runs reordering of tables within single n-way join (i.e.: picks streamtable)"),
HIVE_MERGE_NWAY_JOINS("hive.merge.nway.joins", false,
@@ -5534,22 +5420,12 @@ public class HiveConf extends Configuration {
"hive.server2.service.users," +
"hive.privilege.synchronizer," +
"hive.privilege.synchronizer.interval," +
- "hive.spark.client.connect.timeout," +
- "hive.spark.client.server.connect.timeout," +
- "hive.spark.client.channel.log.level," +
- "hive.spark.client.rpc.max.size," +
- "hive.spark.client.rpc.threads," +
- "hive.spark.client.secret.bits," +
"hive.query.max.length," +
- "hive.spark.client.rpc.server.address," +
- "hive.spark.client.rpc.server.port," +
- "hive.spark.client.rpc.sasl.mechanisms," +
"hive.druid.broker.address.default," +
"hive.druid.coordinator.address.default," +
"hikaricp.," +
"hadoop.bin.path," +
"yarn.bin.path," +
- "spark.home," +
"hive.driver.parallel.compilation.global.limit," +
"hive.zookeeper.ssl.keystore.location," +
"hive.zookeeper.ssl.keystore.password," +
@@ -5578,12 +5454,6 @@ public class HiveConf extends Configuration {
HIVE_CONF_INTERNAL_VARIABLE_LIST("hive.conf.internal.variable.list",
"hive.added.files.path,hive.added.jars.path,hive.added.archives.path",
"Comma separated list of variables which are used internally and should not be configurable."),
- HIVE_SPARK_RSC_CONF_LIST("hive.spark.rsc.conf.list",
- SPARK_OPTIMIZE_SHUFFLE_SERDE.varname + "," +
- SPARK_CLIENT_FUTURE_TIMEOUT.varname + "," +
- SPARK_CLIENT_TYPE.varname,
- "Comma separated list of variables which are related to remote spark context.\n" +
- "Changing these variables will result in re-creating the spark session."),
HIVE_QUERY_MAX_LENGTH("hive.query.max.length", "10Mb", new SizeValidator(), "The maximum" +
" size of a query string. Enforced after variable substitutions."),
HIVE_QUERY_TIMEOUT_SECONDS("hive.query.timeout.seconds", "0s",
@@ -5714,7 +5584,7 @@ public class HiveConf extends Configuration {
HIVE_ADDITIONAL_CONFIG_FILES("hive.additional.config.files", "",
"The names of additional config files, such as ldap-site.xml," +
- "spark-site.xml, etc in comma separated list.");
+ "tez-site.xml, etc in comma separated list.");
public final String varname;
public final String altName;
@@ -6000,9 +5870,6 @@ public class HiveConf extends Configuration {
// When either name or value is null, the set method below will fail,
// and throw IllegalArgumentException
set(name, value);
- if (isSparkRelatedConfig(name)) {
- isSparkConfigUpdated = true;
- }
}
}
@@ -6020,32 +5887,6 @@ public class HiveConf extends Configuration {
return false;
}
- /**
- * check whether spark related property is updated, which includes spark configurations,
- * RSC configurations and yarn configuration in Spark on YARN mode.
- * @param name
- * @return
- */
- private boolean isSparkRelatedConfig(String name) {
- boolean result = false;
- if (name.startsWith("spark")) { // Spark property.
- // for now we don't support changing spark app name on the fly
- result = !name.equals("spark.app.name");
- } else if (name.startsWith("yarn")) { // YARN property in Spark on YARN mode.
- String sparkMaster = get("spark.master");
- if (sparkMaster != null && sparkMaster.startsWith("yarn")) {
- result = true;
- }
- } else if (rscList.stream().anyMatch(rscVar -> rscVar.equals(name))) { // Remote Spark Context property.
- result = true;
- } else if (name.equals("mapreduce.job.queuename")) {
- // a special property starting with mapreduce that we would also like to effect if it changes
- result = true;
- }
-
- return result;
- }
-
public static int getIntVar(Configuration conf, ConfVars var) {
assert (var.valClass == Integer.class) : var.varname;
if (var.altName != null) {
@@ -6263,17 +6104,6 @@ public class HiveConf extends Configuration {
conf.setBoolean(var.varname, val);
}
- /* Dynamic partition pruning is enabled in some or all cases if either
- * hive.spark.dynamic.partition.pruning is true or
- * hive.spark.dynamic.partition.pruning.map.join.only is true
- */
- public static boolean isSparkDPPAny(Configuration conf) {
- return (conf.getBoolean(ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING.varname,
- ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING.defaultBoolVal) ||
- conf.getBoolean(ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING_MAP_JOIN_ONLY.varname,
- ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING_MAP_JOIN_ONLY.defaultBoolVal));
- }
-
public boolean getBoolVar(ConfVars var) {
return getBoolVar(this, var);
}
@@ -6443,7 +6273,6 @@ public class HiveConf extends Configuration {
super(other);
hiveJar = other.hiveJar;
auxJars = other.auxJars;
- isSparkConfigUpdated = other.isSparkConfigUpdated;
origProp = (Properties)other.origProp.clone();
restrictList.addAll(other.restrictList);
hiddenSet.addAll(other.hiddenSet);
@@ -6580,7 +6409,6 @@ public class HiveConf extends Configuration {
setupRestrictList();
hiddenSet.clear();
hiddenSet.addAll(HiveConfUtil.getHiddenSet(this));
- setupRSCList();
}
/**
@@ -6996,17 +6824,6 @@ public class HiveConf extends Configuration {
restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname);
restrictList.add(ConfVars.HIVE_CONF_HIDDEN_LIST.varname);
restrictList.add(ConfVars.HIVE_CONF_INTERNAL_VARIABLE_LIST.varname);
- restrictList.add(ConfVars.HIVE_SPARK_RSC_CONF_LIST.varname);
- }
-
- private void setupRSCList() {
- rscList.clear();
- String vars = this.getVar(ConfVars.HIVE_SPARK_RSC_CONF_LIST);
- if (vars != null) {
- for (String var : vars.split(",")) {
- rscList.add(var.trim());
- }
- }
}
/**
@@ -7030,21 +6847,6 @@ public class HiveConf extends Configuration {
return isWebUiEnabled() && this.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_MAX_HISTORIC_QUERIES) > 0;
}
- /* Dynamic partition pruning is enabled in some or all cases
- */
- public boolean isSparkDPPAny() {
- return isSparkDPPAny(this);
- }
-
- /* Dynamic partition pruning is enabled only for map join
- * hive.spark.dynamic.partition.pruning is false and
- * hive.spark.dynamic.partition.pruning.map.join.only is true
- */
- public boolean isSparkDPPOnlyMapjoin() {
- return (!this.getBoolVar(ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING) &&
- this.getBoolVar(ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING_MAP_JOIN_ONLY));
- }
-
public static boolean isLoadMetastoreConfig() {
return loadMetastoreConfig;
}
diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 7f13403f555..749b76b41fd 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -508,10 +508,6 @@ public enum ErrorMsg {
FILE_NOT_FOUND(20012, "File not found: {0}", "64000", true),
WRONG_FILE_FORMAT(20013, "Wrong file format. Please check the file's format.", "64000", true),
- SPARK_CREATE_CLIENT_INVALID_QUEUE(20014, "Spark app for session {0} was submitted to an invalid" +
- " queue: {1}. Please fix and try again.", true),
- SPARK_RUNTIME_OOM(20015, "Spark job failed because of out of memory."),
-
REPL_FILE_MISSING_FROM_SRC_AND_CM_PATH(20016, "File is missing from both source and cm path."),
REPL_EXTERNAL_SERVICE_CONNECTION_ERROR(20017, "Failed to connect to {0} service. Error code {1}.",true),
CLIENT_POLLING_OPSTATUS_INTERRUPTED(20018, "Interrupted while polling on the operation status", "70100"),
@@ -578,42 +574,11 @@ public enum ErrorMsg {
CONCATENATE_UNSUPPORTED_TABLE_TRANSACTIONAL(30035,
"Concatenate/Merge can not be performed on transactional tables"),
- SPARK_GET_JOB_INFO_TIMEOUT(30036,
- "Spark job timed out after {0} seconds while getting job info", true),
- SPARK_JOB_MONITOR_TIMEOUT(30037, "Job hasn''t been submitted after {0}s." +
- " Aborting it.\nPossible reasons include network issues, " +
- "errors in remote driver or the cluster has no available resources, etc.\n" +
- "Please check YARN or Spark driver''s logs for further information.\n" +
- "The timeout is controlled by " + HiveConf.ConfVars.SPARK_JOB_MONITOR_TIMEOUT + ".", true),
-
- // Various errors when creating Spark client
- SPARK_CREATE_CLIENT_TIMEOUT(30038,
- "Timed out while creating Spark client for session {0}.", true),
- SPARK_CREATE_CLIENT_QUEUE_FULL(30039,
- "Failed to create Spark client for session {0} because job queue is full: {1}.", true),
- SPARK_CREATE_CLIENT_INTERRUPTED(30040,
- "Interrupted while creating Spark client for session {0}", true),
- SPARK_CREATE_CLIENT_ERROR(30041,
- "Failed to create Spark client for Spark session {0}: {1}", true),
- SPARK_CREATE_CLIENT_INVALID_RESOURCE_REQUEST(30042,
- "Failed to create Spark client for session {0} due to invalid resource request: {1}", true),
- SPARK_CREATE_CLIENT_CLOSED_SESSION(30043,
- "Cannot create Spark client on a closed session {0}", true),
-
- SPARK_JOB_INTERRUPTED(30044, "Spark job was interrupted while executing"),
- SPARK_GET_JOB_INFO_INTERRUPTED(30045, "Spark job was interrupted while getting job info"),
- SPARK_GET_JOB_INFO_EXECUTIONERROR(30046, "Spark job failed in execution while getting job info due to exception {0}", true),
-
REPL_FILE_SYSTEM_OPERATION_RETRY(30047, "Replication file system operation retry expired. Error {0}",
true),
- SPARK_GET_STAGES_INFO_TIMEOUT(30048, "Spark job GetSparkStagesInfoJob timed out after {0} seconds.", true),
- SPARK_GET_STAGES_INFO_INTERRUPTED(30049, "Spark job GetSparkStagesInfoJob was interrupted."),
- SPARK_GET_STAGES_INFO_EXECUTIONERROR(30050, "Spark job GetSparkStagesInfoJob failed in execution while getting job info due to exception {0}", true),
//========================== 40000 range starts here ========================//
- SPARK_JOB_RUNTIME_ERROR(40001, "Spark job failed due to: {0}", true),
- SPARK_TASK_RUNTIME_ERROR(40002, "Spark job failed due to task failures: {0}", true),
REPL_DATABASE_IS_TARGET_OF_REPLICATION(40003, "Cannot dump database as it is a Target of replication."),
REPL_INVALID_DB_OR_TABLE_PATTERN(40005,
"Invalid pattern for the DB or table name in the replication policy. "
diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
index f39582351bb..6254e3932ab 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
@@ -73,22 +73,6 @@ public class PerfLogger {
public static final String TEZ_GET_SESSION = "TezGetSession";
public static final String SAVE_TO_RESULTS_CACHE = "saveToResultsCache";
- public static final String SPARK_SUBMIT_TO_RUNNING = "SparkSubmitToRunning";
- public static final String SPARK_BUILD_PLAN = "SparkBuildPlan";
- public static final String SPARK_BUILD_RDD_GRAPH = "SparkBuildRDDGraph";
- public static final String SPARK_CREATE_EXPLAIN_PLAN = "SparkCreateExplainPlan.";
- public static final String SPARK_SUBMIT_JOB = "SparkSubmitJob";
- public static final String SPARK_RUN_JOB = "SparkRunJob";
- public static final String SPARK_CREATE_TRAN = "SparkCreateTran.";
- public static final String SPARK_RUN_STAGE = "SparkRunStage.";
- public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators";
- public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree";
- public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree";
- public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree";
- public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable.";
- public static final String SPARK_DYNAMICALLY_PRUNE_PARTITIONS =
- "SparkDynamicallyPrunePartitions.";
-
public static final String FILE_MOVES = "FileMoves";
public static final String LOAD_TABLE = "LoadTable";
public static final String LOAD_PARTITION = "LoadPartition";
@@ -102,9 +86,6 @@ public class PerfLogger {
public static final String HIVE_GET_PARTITIONS_BY_EXPR = "getPartitionsByExpr";
public static final String HIVE_GET_TABLE_COLUMN_STATS = "getTableColumnStatistics";
public static final String HIVE_GET_AGGR_COL_STATS = "getAggrColStatsFor";
- public static final String HIVE_GET_PK = "getPrimaryKeys";
- public static final String HIVE_GET_FK = "getForeignKeys";
- public static final String HIVE_GET_UNIQ_CONSTRAINT = "getUniqueConstraints";
public static final String HIVE_GET_NOT_NULL_CONSTRAINT = "getNotNullConstraints";
public static final String HIVE_GET_TABLE_CONSTRAINTS = "getTableConstraints";
diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
index efc50ff3e8b..2d0b38e57da 100644
--- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
+++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
@@ -175,19 +175,6 @@ public class TestHiveConf {
}
}
- @Test
- public void testSparkConfigUpdate(){
- HiveConf conf = new HiveConf();
- Assert.assertFalse(conf.getSparkConfigUpdated());
-
- conf.verifyAndSet("spark.master", "yarn");
- Assert.assertTrue(conf.getSparkConfigUpdated());
- conf.verifyAndSet("hive.execution.engine", "spark");
- Assert.assertTrue("Expected spark config updated.", conf.getSparkConfigUpdated());
-
- conf.setSparkConfigUpdated(false);
- Assert.assertFalse(conf.getSparkConfigUpdated());
- }
@Test
public void testEncodingDecoding() throws UnsupportedEncodingException {
HiveConf conf = new HiveConf();
diff --git a/data/conf/perf/tpcds30tb/spark/hive-site.xml b/data/conf/perf/tpcds30tb/spark/hive-site.xml
deleted file mode 100644
index 6a341a8176c..00000000000
--- a/data/conf/perf/tpcds30tb/spark/hive-site.xml
+++ /dev/null
@@ -1,214 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
- <property>
- <name>hive.in.test</name>
- <value>true</value>
- <description>Internal marker for test. Used for masking env-dependent values</description>
- </property>
- <property>
- <name>hive.rpc.query.plan</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.llap.execution.mode</name>
- <value>only</value>
- </property>
- <property>
- <name>hive.exec.post.hooks</name>
- <value>
- org.apache.hadoop.hive.ql.hooks.HiveProtoLoggingHook
- </value>
- </property>
- <property>
- <name>hive.zookeeper.kerberos.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>hive.privilege.synchronizer</name>
- <value>false</value>
- </property>
- <property>
- <name>javax.jdo.option.ConnectionDriverName</name>
- <value>org.postgresql.Driver</value>
- </property>
- <property>
- <name>hive.exec.scratchdir</name>
- <value>${test.tmp.dir}/scratchdir</value>
- <description>Scratch space for Hive jobs</description>
- </property>
- <property>
- <name>hive.server2.tez.sessions.init.threads</name>
- <value>1</value>
- </property>
- <property>
- <name>hive.optimize.dynamic.partition.hashjoin</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.exec.orc.split.strategy</name>
- <value>BI</value>
- </property>
- <property>
- <name>hive.mapjoin.bucket.cache.size</name>
- <value>10000</value>
- </property>
- <property>
- <name>hive.current.database</name>
- <value>tpcds_bin_partitioned_orc_30000</value>
- </property>
- <property>
- <name>hive.optimize.metadataonly</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.vectorized.execution.mapjoin.native.fast.hashtable.enabled</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.execution.mode</name>
- <value>llap</value>
- </property>
- <property>
- <name>hive.prewarm.numcontainers</name>
- <value>3</value>
- </property>
- <property>
- <name>hive.tez.dynamic.partition.pruning</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.support.concurrency</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.txn.manager</name>
- <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
- </property>
- <property>
- <name>hive.llap.kerberos.enabled</name>
- <value>false</value>
- </property>
- <property>
- <name>hive.load.data.owner</name>
- <value>hive</value>
- </property>
- <property>
- <name>hive.enforce.sortmergebucketmapjoin</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.limit.optimize.enable</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.auto.convert.sortmerge.join.to.mapjoin</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.txn.acid.dir.cache.duration</name>
- <value>0</value>
- </property>
- <property>
- <name>hive.optimize.bucketmapjoin</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.tez.container.size</name>
- <value>4096</value>
- </property>
- <property>
- <name>hive.stats.fetch.bitvector</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.convert.join.bucket.mapjoin.tez</name>
- <value>false</value>
- </property>
- <property>
- <name>hive.execution.engine</name>
- <value>tez</value>
- </property>
- <property>
- <name>hive.vectorized.execution.mapjoin.minmax.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hive.llap.auto.enforce.stats</name>
- <value>false</value>
- </property>
- <property>
- <name>hive.exec.max.dynamic.partitions.pernode</name>
- <value>2000</value>
- </property>
- <property>
- <name>hive.llap.daemon.num.executors</name>
- <value>12</value>
- </property>
- <property>
- <name>hive.tez.dynamic.partition.pruning.max.event.size</name>
- <value>1048576</value>
- </property>
- <property>
- <name>hive.exec.max.dynamic.partitions</name>
- <value>5000</value>
- </property>
- <property>
- <name>hive.default.fileformat.managed</name>
- <value>ORC</value>
- </property>
- <property>
- <name>hive.llap.io.enabled</name>
- <value>true</value>
- </property>
- <property>
- <name>hive.auto.convert.join.noconditionaltask.size</name>
- <value>1431655765</value>
- </property>
- <property>
- <name>hive.tez.auto.reducer.parallelism</name>
- <value>true</value>
- </property>
- <!-- Metastore properties -->
- <property>
- <name>hive.metastore.db.type</name>
- <value>POSTGRES</value>
- </property>
- <property>
- <name>hive.metastore.rawstore.impl</name>
- <value>org.apache.hadoop.hive.metastore.cache.CachedStore</value>
- </property>
- <property>
- <name>hive.metastore.cache.pinobjtypes</name>
- <value>Table,Database,Type,FieldSchema,Order</value>
- </property>
- <property>
- <name>hive.metastore.warehouse.dir</name>
- <value>${test.warehouse.dir}</value>
- </property>
- <property>
- <name>hive.metastore.failure.retries</name>
- <value>2</value>
- </property>
- <property>
- <name>hive.metastore.transactional.event.listeners</name>
- <value>org.apache.hive.hcatalog.listener.DbNotificationListener</value>
- </property>
-</configuration>
\ No newline at end of file
diff --git a/data/conf/spark/local/hive-site.xml b/data/conf/spark/local/hive-site.xml
deleted file mode 100644
index 4841f8a5dfe..00000000000
--- a/data/conf/spark/local/hive-site.xml
+++ /dev/null
@@ -1,291 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
-
-<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files -->
-<!-- that are implied by Hadoop setup variables. -->
-<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive -->
-<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
-<!-- resource). -->
-
-<!-- Hive Execution Parameters -->
-<property>
- <name>hadoop.tmp.dir</name>
- <value>${test.tmp.dir}/hadoop-tmp</value>
- <description>A base for other temporary directories.</description>
-</property>
-
-<property>
- <name>hive.exec.scratchdir</name>
- <value>${test.tmp.dir}/scratchdir</value>
- <description>Scratch space for Hive jobs</description>
-</property>
-
-<property>
- <name>hive.exec.local.scratchdir</name>
- <value>${test.tmp.dir}/localscratchdir/</value>
- <description>Local scratch space for Hive jobs</description>
-</property>
-
-<property>
- <name>datanucleus.schema.autoCreateAll</name>
- <value>true</value>
-</property>
-
-<property>
- <name>datanucleus.connectionPool.maxPoolSize</name>
- <value>4</value>
-</property>
-
-<property>
- <name>hive.metastore.schema.verification</name>
- <value>false</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionDriverName</name>
- <value>org.apache.derby.jdbc.EmbeddedDriver</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionUserName</name>
- <value>APP</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionPassword</name>
- <value>mine</value>
-</property>
-
-<property>
- <!-- this should eventually be deprecated since the metastore should supply this -->
- <name>hive.metastore.warehouse.dir</name>
- <value>${test.warehouse.dir}</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.metastore.metadb.dir</name>
- <value>file://${test.tmp.dir}/metadb/</value>
- <description>
- Required by metastore server or if the uris argument below is not supplied
- </description>
-</property>
-
-<property>
- <name>test.log.dir</name>
- <value>${test.tmp.dir}/log/</value>
- <description></description>
-</property>
-
-<property>
- <name>test.data.files</name>
- <value>${hive.root}/data/files</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.jar.path</name>
- <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
- <description></description>
-</property>
-
-<property>
- <name>test.data.scripts</name>
- <value>${hive.root}/data/scripts</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.metastore.rawstore.impl</name>
- <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
- <description>Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database</description>
-</property>
-
-<property>
- <name>hive.querylog.location</name>
- <value>${test.tmp.dir}/tmp</value>
- <description>Location of the structured hive logs</description>
-</property>
-
-<property>
- <name>hive.exec.pre.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
- <description>Pre Execute Hook for Tests</description>
-</property>
-
-<property>
- <name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
- <description>Post Execute Hook for Tests</description>
-</property>
-
-<property>
- <name>hive.support.concurrency</name>
- <value>false</value>
- <description>Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks.</description>
-</property>
-
-<property>
- <name>fs.pfile.impl</name>
- <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
- <description>A proxy for local file system used for cross file system testing</description>
-</property>
-
-<property>
- <name>hive.exec.mode.local.auto</name>
- <value>false</value>
- <description>
- Let hive determine whether to run in local mode automatically
- Disabling this for tests so that minimr is not affected
- </description>
-</property>
-
-<property>
- <name>hive.merge.nway.joins</name>
- <value>true</value>
-</property>
-
-
-<property>
- <name>hive.auto.convert.join</name>
- <value>false</value>
- <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file size</description>
-</property>
-
-<property>
- <name>hive.ignore.mapjoin.hint</name>
- <value>false</value>
- <description>Whether Hive ignores the mapjoin hint</description>
-</property>
-
-<property>
- <name>io.sort.mb</name>
- <value>10</value>
-</property>
-
-<property>
- <name>hive.input.format</name>
- <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
- <description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>
-</property>
-
-<property>
- <name>hive.default.rcfile.serde</name>
- <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
- <description>The default SerDe hive will use for the rcfile format</description>
-</property>
-
-<property>
- <name>hive.stats.dbclass</name>
- <value>fs</value>
- <description>The default storatge that stores temporary hive statistics. Currently, fs type is supported</description>
-</property>
-
-<property>
- <name>hive.execution.engine</name>
- <value>spark</value>
- <description>Chooses execution engine. Options are: mr (Map reduce, default), tez (hadoop 2 only), spark</description>
-</property>
-
-<property>
- <name>spark.master</name>
- <value>local[*]</value>
-</property>
-
-<property>
- <name>hive.prewarm.enabled</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.prewarm.numcontainers</name>
- <value>1</value>
-</property>
-
-<property>
- <name>hive.prewarm.spark.timeout</name>
- <value>30s</value>
-</property>
-
-<property>
- <name>spark.serializer</name>
- <value>org.apache.spark.serializer.KryoSerializer</value>
-</property>
-
-<property>
- <name>spark.akka.logLifecycleEvents</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.spark.log.dir</name>
- <value>${spark.home}/logs/</value>
-</property>
-
-<property>
- <name>spark.driver.extraClassPath</name>
- <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar:${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar:${maven.local.repository}/org/antlr/antlr-runtime/${antlr.version}/antlr-runtime-${antlr.version}.jar</value>
-</property>
-
-<property>
- <name>spark.local.dir</name>
- <value>${test.tmp.dir}/spark-local-dir</value>
-</property>
-
-<property>
- <name>hive.aux.jars.path</name>
- <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
-</property>
-
-<property>
- <name>hive.users.in.admin.role</name>
- <value>hive_admin_user</value>
-</property>
-
-<property>
- <name>hive.in.test</name>
- <value>true</value>
- <description>Internal marker for test. Used for masking env-dependent values</description>
-</property>
-
-<!-- remove this after HIVE-18139 -->
-<property>
- <name>hive.stats.column.autogather</name>
- <value>false</value>
-</property>
-
-<!-- remove this after HIVE-18139 and HIVE-20573 -->
-<property>
- <name>hive.stats.fetch.column.stats</name>
- <value>false</value>
-</property>
-
-<property>
- <name>hive.query.results.cache.enabled</name>
- <value>false</value>
-</property>
-
-</configuration>
diff --git a/data/conf/spark/log4j2.properties b/data/conf/spark/log4j2.properties
deleted file mode 100644
index c2deb4f9f88..00000000000
--- a/data/conf/spark/log4j2.properties
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-status = INFO
-name = SparkLog4j2Test
-packages = org.apache.hadoop.hive.ql.log
-
-# list of properties
-property.spark.log.level = DEBUG
-property.spark.root.logger = DRFA
-property.spark.log.dir = target/tmp/log
-property.spark.log.file = spark.log
-
-# list of all appenders
-appenders = console, DRFA
-
-# console appender
-appender.console.type = Console
-appender.console.name = console
-appender.console.target = SYSTEM_ERR
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
-
-# daily rolling file appender
-appender.DRFA.type = RollingFile
-appender.DRFA.name = DRFA
-appender.DRFA.fileName = ${sys:spark.log.dir}/${sys:spark.log.file}
-appender.DRFA.filePattern = ${sys:spark.log.dir}/${sys:spark.log.file}.%d{yyyy-MM-dd}
-appender.DRFA.layout.type = PatternLayout
-appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
-appender.DRFA.policies.type = Policies
-appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
-appender.DRFA.policies.time.interval = 1
-appender.DRFA.policies.time.modulate = true
-appender.DRFA.strategy.type = DefaultRolloverStrategy
-appender.DRFA.strategy.max = 30
-
-# list of all loggers
-loggers = SparkIMain, SparkILoop, Jetty, AbstractLifeCycle
-
-logger.SparkIMain.name = org.apache.spark.repl.SparkIMain$exprTyper
-logger.SparkIMain.level = INFO
-
-logger.SparkILoop.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter
-logger.SparkILoop.level = INFO
-
-logger.Jetty.name = org.eclipse.jetty
-logger.Jetty.level = WARN
-
-logger.AbstractLifeCycle.name = org.eclipse.jetty.util.component.AbstractLifeCycle
-logger.AbstractLifeCycle.level = ERROR
-
-# root logger
-rootLogger.level = ${sys:spark.log.level}
-rootLogger.appenderRefs = root
-rootLogger.appenderRef.root.ref = ${sys:spark.root.logger}
diff --git a/data/conf/spark/standalone/hive-site.xml b/data/conf/spark/standalone/hive-site.xml
deleted file mode 100644
index e882cd695aa..00000000000
--- a/data/conf/spark/standalone/hive-site.xml
+++ /dev/null
@@ -1,300 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
-
-<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files -->
-<!-- that are implied by Hadoop setup variables. -->
-<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive -->
-<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
-<!-- resource). -->
-
-<!-- Hive Execution Parameters -->
-<property>
- <name>hadoop.tmp.dir</name>
- <value>${test.tmp.dir}/hadoop-tmp</value>
- <description>A base for other temporary directories.</description>
-</property>
-
-<property>
- <name>hive.exec.scratchdir</name>
- <value>${test.tmp.dir}/scratchdir</value>
- <description>Scratch space for Hive jobs</description>
-</property>
-
-<property>
- <name>hive.exec.local.scratchdir</name>
- <value>${test.tmp.dir}/localscratchdir/</value>
- <description>Local scratch space for Hive jobs</description>
-</property>
-
-<property>
- <name>datanucleus.schema.autoCreateAll</name>
- <value>true</value>
-</property>
-
-<property>
- <name>datanucleus.connectionPool.maxPoolSize</name>
- <value>4</value>
-</property>
-
-<property>
- <name>hive.metastore.schema.verification</name>
- <value>false</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionDriverName</name>
- <value>org.apache.derby.jdbc.EmbeddedDriver</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionUserName</name>
- <value>APP</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionPassword</name>
- <value>mine</value>
-</property>
-
-<property>
- <!-- this should eventually be deprecated since the metastore should supply this -->
- <name>hive.metastore.warehouse.dir</name>
- <value>${test.warehouse.dir}</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.metastore.metadb.dir</name>
- <value>file://${test.tmp.dir}/metadb/</value>
- <description>
- Required by metastore server or if the uris argument below is not supplied
- </description>
-</property>
-
-<property>
- <name>test.log.dir</name>
- <value>${test.tmp.dir}/log/</value>
- <description></description>
-</property>
-
-<property>
- <name>test.data.files</name>
- <value>${hive.root}/data/files</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.jar.path</name>
- <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
- <description></description>
-</property>
-
-<property>
- <name>test.data.scripts</name>
- <value>${hive.root}/data/scripts</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.metastore.rawstore.impl</name>
- <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
- <description>Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database</description>
-</property>
-
-<property>
- <name>hive.querylog.location</name>
- <value>${test.tmp.dir}/tmp</value>
- <description>Location of the structured hive logs</description>
-</property>
-
-<property>
- <name>hive.exec.pre.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
- <description>Pre Execute Hook for Tests</description>
-</property>
-
-<property>
- <name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
- <description>Post Execute Hook for Tests</description>
-</property>
-
-<property>
- <name>hive.support.concurrency</name>
- <value>false</value>
- <description>Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks.</description>
-</property>
-
-<property>
- <name>fs.pfile.impl</name>
- <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
- <description>A proxy for local file system used for cross file system testing</description>
-</property>
-
-<property>
- <name>hive.exec.mode.local.auto</name>
- <value>false</value>
- <description>
- Let hive determine whether to run in local mode automatically
- Disabling this for tests so that minimr is not affected
- </description>
-</property>
-
-<property>
- <name>hive.merge.nway.joins</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.auto.convert.join</name>
- <value>false</value>
- <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file size</description>
-</property>
-
-<property>
- <name>hive.ignore.mapjoin.hint</name>
- <value>false</value>
- <description>Whether Hive ignores the mapjoin hint</description>
-</property>
-
-<property>
- <name>io.sort.mb</name>
- <value>10</value>
-</property>
-
-<property>
- <name>hive.input.format</name>
- <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
- <description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>
-</property>
-
-<property>
- <name>hive.default.rcfile.serde</name>
- <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
- <description>The default SerDe hive will use for the rcfile format</description>
-</property>
-
-<property>
- <name>hive.stats.dbclass</name>
- <value>fs</value>
- <description>The default storatge that stores temporary hive statistics. Currently, fs type is supported</description>
-</property>
-
-<property>
- <name>hive.execution.engine</name>
- <value>spark</value>
- <description>Chooses execution engine. Options are: mr (Map reduce, default), tez (hadoop 2 only), spark</description>
-</property>
-
-<property>
- <name>spark.master</name>
- <value>local-cluster[1,2,1024]</value>
-</property>
-
-<property>
- <name>hive.prewarm.enabled</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.prewarm.numcontainers</name>
- <value>1</value>
-</property>
-
-<property>
- <name>hive.prewarm.spark.timeout</name>
- <value>30s</value>
-</property>
-
-<property>
- <name>spark.serializer</name>
- <value>org.apache.spark.serializer.KryoSerializer</value>
-</property>
-
-<property>
- <name>spark.akka.logLifecycleEvents</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.spark.log.dir</name>
- <value>${spark.home}/logs/</value>
-</property>
-
-<property>
- <name>spark.driver.extraClassPath</name>
- <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar:${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar:${maven.local.repository}/org/antlr/antlr-runtime/${antlr.version}/antlr-runtime-${antlr.version}.jar:${maven.local.repository}/org/apache/hadoop/hadoop-common/${hadoop.version}/hadoop-common-${hadoop.version}.jar</value>
-</property>
-
-<property>
- <name>spark.executor.extraClassPath</name>
- <value>${maven.local.repository}/org/apache/hadoop/hadoop-common/${hadoop.version}/hadoop-common-${hadoop.version}.jar</value>
-</property>
-
-<property>
- <name>spark.local.dir</name>
- <value>${test.tmp.dir}/spark-local-dir</value>
-</property>
-
-<property>
- <name>hive.aux.jars.path</name>
- <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
-</property>
-
-<property>
- <name>hive.users.in.admin.role</name>
- <value>hive_admin_user</value>
-</property>
-
-<property>
- <name>hive.in.test</name>
- <value>true</value>
- <description>Internal marker for test. Used for masking env-dependent values</description>
-</property>
-
-<property>
- <name>hive.spark.client.connect.timeout</name>
- <value>30000ms</value>
-</property>
-
-<!-- remove this after HIVE-18139 -->
-<property>
- <name>hive.stats.column.autogather</name>
- <value>false</value>
-</property>
-
-<!-- remove this after HIVE-18139 and HIVE-20573 -->
-<property>
- <name>hive.stats.fetch.column.stats</name>
- <value>false</value>
-</property>
-
-<property>
- <name>hive.query.results.cache.enabled</name>
- <value>false</value>
-</property>
-
-</configuration>
diff --git a/data/conf/spark/yarn-cluster/hive-site.xml b/data/conf/spark/yarn-cluster/hive-site.xml
deleted file mode 100644
index b39eb2ce40d..00000000000
--- a/data/conf/spark/yarn-cluster/hive-site.xml
+++ /dev/null
@@ -1,336 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<configuration>
-
-<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files -->
-<!-- that are implied by Hadoop setup variables. -->
-<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive -->
-<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
-<!-- resource). -->
-
-<!-- Hive Execution Parameters -->
-<property>
- <name>hadoop.tmp.dir</name>
- <value>${test.tmp.dir}/hadoop-tmp</value>
- <description>A base for other temporary directories.</description>
-</property>
-
-<property>
- <name>hive.exec.scratchdir</name>
- <value>${test.tmp.dir}/scratchdir</value>
- <description>Scratch space for Hive jobs</description>
-</property>
-
-<property>
- <name>hive.exec.local.scratchdir</name>
- <value>${test.tmp.dir}/localscratchdir/</value>
- <description>Local scratch space for Hive jobs</description>
-</property>
-
-<property>
- <name>datanucleus.schema.autoCreateAll</name>
- <value>true</value>
-</property>
-
-<property>
- <name>datanucleus.connectionPool.maxPoolSize</name>
- <value>4</value>
-</property>
-
-<property>
- <name>hive.metastore.schema.verification</name>
- <value>false</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionURL</name>
- <value>jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionDriverName</name>
- <value>org.apache.derby.jdbc.EmbeddedDriver</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionUserName</name>
- <value>APP</value>
-</property>
-
-<property>
- <name>javax.jdo.option.ConnectionPassword</name>
- <value>mine</value>
-</property>
-
-<property>
- <!-- this should eventually be deprecated since the metastore should supply this -->
- <name>hive.metastore.warehouse.dir</name>
- <value>${test.warehouse.dir}</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.metastore.metadb.dir</name>
- <value>file://${test.tmp.dir}/metadb/</value>
- <description>
- Required by metastore server or if the uris argument below is not supplied
- </description>
-</property>
-
-<property>
- <name>test.log.dir</name>
- <value>${test.tmp.dir}/log/</value>
- <description></description>
-</property>
-
-<property>
- <name>test.data.files</name>
- <value>${hive.root}/data/files</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.jar.path</name>
- <value>${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
- <description></description>
-</property>
-
-<property>
- <name>test.data.scripts</name>
- <value>${hive.root}/data/scripts</value>
- <description></description>
-</property>
-
-<property>
- <name>hive.metastore.rawstore.impl</name>
- <value>org.apache.hadoop.hive.metastore.ObjectStore</value>
- <description>Name of the class that implements org.apache.hadoop.hive.metastore.rawstore interface. This class is used to store and retrieval of raw metadata objects such as table, database</description>
-</property>
-
-<property>
- <name>hive.querylog.location</name>
- <value>${test.tmp.dir}/tmp</value>
- <description>Location of the structured hive logs</description>
-</property>
-
-<property>
- <name>hive.exec.pre.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PreExecutePrinter, org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables</value>
- <description>Pre Execute Hook for Tests</description>
-</property>
-
-<property>
- <name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
- <description>Post Execute Hook for Tests</description>
-</property>
-
-<property>
- <name>hive.support.concurrency</name>
- <value>false</value>
- <description>Whether hive supports concurrency or not. A zookeeper instance must be up and running for the default hive lock manager to support read-write locks.</description>
-</property>
-
-<property>
- <name>fs.pfile.impl</name>
- <value>org.apache.hadoop.fs.ProxyLocalFileSystem</value>
- <description>A proxy for local file system used for cross file system testing</description>
-</property>
-
-<property>
- <name>hive.exec.mode.local.auto</name>
- <value>false</value>
- <description>
- Let hive determine whether to run in local mode automatically
- Disabling this for tests so that minimr is not affected
- </description>
-</property>
-
-<property>
- <name>hive.merge.nway.joins</name>
- <value>true</value>
-</property>
-
-
-<property>
- <name>hive.auto.convert.join</name>
- <value>false</value>
- <description>Whether Hive enable the optimization about converting common join into mapjoin based on the input file size</description>
-</property>
-
-<property>
- <name>hive.ignore.mapjoin.hint</name>
- <value>false</value>
- <description>Whether Hive ignores the mapjoin hint</description>
-</property>
-
-<property>
- <name>io.sort.mb</name>
- <value>10</value>
-</property>
-
-<property>
- <name>hive.input.format</name>
- <value>org.apache.hadoop.hive.ql.io.CombineHiveInputFormat</value>
- <description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>
-</property>
-
-<property>
- <name>hive.default.rcfile.serde</name>
- <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
- <description>The default SerDe hive will use for the rcfile format</description>
-</property>
-
-<property>
- <name>hive.stats.dbclass</name>
- <value>fs</value>
- <description>The default storatge that stores temporary hive statistics. Currently, fs type is supported</description>
-</property>
-
-<property>
- <name>hive.execution.engine</name>
- <value>spark</value>
- <description>Chooses execution engine. Options are: mr (Map reduce, default), tez (hadoop 2 only), spark</description>
-</property>
-
-<property>
- <name>spark.master</name>
- <value>yarn</value>
-</property>
-
-<property>
- <name>spark.submit.deployMode</name>
- <value>cluster</value>
-</property>
-
-<property>
- <name>spark.serializer</name>
- <value>org.apache.spark.serializer.KryoSerializer</value>
-</property>
-
-<property>
- <name>spark.akka.logLifecycleEvents</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.spark.log.dir</name>
- <value>${spark.home}/logs/</value>
-</property>
-
-<property>
- <name>spark.driver.extraClassPath</name>
- <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar:${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar</value>
-</property>
-
-<property>
- <name>spark.executor.instances</name>
- <value>2</value>
-</property>
-
-<property>
- <name>spark.executor.cores</name>
- <value>2</value>
-</property>
-
-<property>
- <name>spark.executor.memory</name>
- <value>512m</value>
-</property>
-
-<property>
- <name>spark.executor.memoryOverhead</name>
- <value>0</value>
-</property>
-
-<property>
- <name>spark.driver.memory</name>
- <value>512m</value>
-</property>
-
-<property>
- <name>spark.driver.memoryOverhead</name>
- <value>0</value>
-</property>
-
-<property>
- <name>spark.local.dir</name>
- <value>${test.tmp.dir}/spark-local-dir</value>
-</property>
-
-<property>
- <name>hive.prewarm.enabled</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.prewarm.spark.timeout</name>
- <value>30s</value>
-</property>
-
-<property>
- <name>spark.testing</name>
- <value>true</value>
-</property>
-
-<property>
- <name>hive.aux.jars.path</name>
- <value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
-</property>
-
-<property>
- <name>hive.users.in.admin.role</name>
- <value>hive_admin_user</value>
-</property>
-
-<property>
- <name>hive.in.test</name>
- <value>true</value>
- <description>Internal marker for test. Used for masking env-dependent values</description>
-</property>
-
-<property>
- <name>hive.spark.client.connect.timeout</name>
- <value>30000ms</value>
-</property>
-
-<property>
- <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
- <value>99</value>
-</property>
-
-<!-- remove this after HIVE-18139 -->
-<property>
- <name>hive.stats.column.autogather</name>
- <value>false</value>
-</property>
-
-<!-- remove this after HIVE-18139 and HIVE-20573 -->
-<property>
- <name>hive.stats.fetch.column.stats</name>
- <value>false</value>
-</property>
-
-<property>
- <name>hive.query.results.cache.enabled</name>
- <value>false</value>
-</property>
-
-</configuration>
diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh
deleted file mode 100644
index e9ba9e3f83c..00000000000
--- a/dev-support/jenkins-common.sh
+++ /dev/null
@@ -1,222 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-JIRA_ROOT_URL="https://issues.apache.org"
-
-fail() {
- echo "$@" 1>&2
- exit 1
-}
-
-alias echoerr='>&2 echo'
-
-# Parses the patch filename, and returns the branch name.
-# If empty or branch is not part of the filename, it returns a default branch
-# Note: DEFAULT_BRANCH must exists
-get_branch_name() {
- local PATCH_NAME="$1"
- local branch="$2" # $2 is a default branch
-
- if [[ -n $PATCH_NAME ]]; then
- if [[ $PATCH_NAME =~ ^HIVE-[0-9]+(\.[0-9]+)?([-\.][A-Za-z0-9.-]+)?\.(patch|patch.\txt)$ ]]; then
- if [[ -n "${BASH_REMATCH[2]}" ]]; then
- branch=${BASH_REMATCH[2]:1}
- fi
- elif [[ $PATCH_NAME =~ ^(HIVE-[0-9]+\.)?D[0-9]+(\.[0-9]+)?\.(patch|patch.\txt)$ ]]; then
- # It will assume the default branch
- continue
- else
- echoerr "Patch '$PATCH_NAME' does not appear to be a patch"
- return 1
- fi
- fi
-
- echo $branch
- return 0
-}
-
-# Used during development for testing get_branch_name
-_test_get_branch_name() {
- do_test() {
- local PATCH_NAME="$1"
- local EXPECTED="$2"
-
- local ACTUAL=$(get_branch_name $PATCH_NAME master)
-
- [[ "$ACTUAL" == "$EXPECTED" ]] || echoerr "$PATCH_NAME failed ($ACTUAL != $EXPECTED)"
- }
-
- # Branch not spcified in patch name
- do_test "HIVE-123.patch" "master"
- do_test "HIVE-123.1.patch" "master"
-
- # Branch name separated by '-'
- do_test "HIVE-123-tez.patch" "tez"
- do_test "HIVE-123.1-tez.patch" "tez"
- do_test "HIVE-123-branch-2.patch" "branch-2"
- do_test "HIVE-123.1-branch-2.patch" "branch-2"
-
- # Branch name separated by '.'
- do_test "HIVE-123.tez.patch" "tez"
- do_test "HIVE-123.1.tez.patch" "tez"
- do_test "HIVE-123.branch-2.patch" "branch-2"
- do_test "HIVE-123.1.branch-2.patch" "branch-2"
-}
-
-# Gets the attachment identifier of a JIRA attachment file
-get_attachment_id() {
- local jira_attachment_url="$1"
-
- basename $(dirname $jira_attachment_url)
-}
-
-# Fetchs JIRA information, and save it in a file specified as a parameter
-initialize_jira_info() {
- local jira_issue="$1"
- local output_file="$2"
-
- curl -s -S --location --retry 3 "${JIRA_ROOT_URL}/jira/browse/${jira_issue}" > $output_file
-}
-
-# Checks if a JIRA is in 'Patch Available' status
-is_patch_available() {
- grep -q "Patch Available" $1
-}
-
-# Checks if a JIRA has 'NO PRECOMMIT TESTS' enabled
-is_check_no_precommit_tests_set() {
- grep -q "NO PRECOMMIT TESTS" $1
-}
-
-# Gets the URL for the JIRA patch attached
-get_jira_patch_url() {
- grep -o '"/jira/secure/attachment/[0-9]*/[^"]*\.patch*' $1 | sort | tail -1 | \
- grep -o '/jira/secure/attachment/[0-9]*/[^"]*\.patch*'
-}
-
-# Checks if the patch was already tested
-is_patch_already_tested() {
- local attachment_id="$1"
- local test_handle="$2"
- local jira_info_file="$3"
-
- grep -q "ATTACHMENT ID: $attachment_id $test_handle" $jira_info_file
-}
-
-# Gets the branch profile attached to the patch
-get_branch_profile() {
- local jira_patch_url="$1"
- local branch_name=`get_branch_name $(basename $jira_patch_url)`
-
- if [ -n "$branch_name" ]; then
- shopt -s nocasematch
- if [[ $branch_name =~ (mr1|mr2)$ ]]; then
- echo $branch_name
- else
- echo ${branch_name}-mr2
- fi
- fi
-}
-
-# Checks if a JIRA patch has 'CLEAR LIBRARY CACHE' enabled
-is_clear_cache_set() {
- grep -q "CLEAR LIBRARY CACHE" $1
-}
-
-# Parses the JIRA/patch to find relavent information.
-# Exports two variables of import:
-# * BUILD_PROFILE - the profile which the ptest server understands
-# * BUILD_OPTS - additional test options to be sent to ptest cli
-# * PATCH_URL - the URL to the patch file
-process_jira() {
- test -n "$BRANCH" || fail "BRANCH must be specified"
- test -n "$JIRA_ROOT_URL" || fail "JIRA_ROOT_URL must be specified"
- test -n "$JIRA_NAME" || fail "JIRA_NAME must be specified"
- JIRA_TEXT=$(mktemp)
- trap "rm -f $JIRA_TEXT" EXIT
- curl -s -S --location --retry 3 "${JIRA_ROOT_URL}/jira/browse/${JIRA_NAME}" > $JIRA_TEXT
- if [[ "${CHECK_NO_PRECOMMIT_TESTS}" == "true" ]] && grep -q "NO PRECOMMIT TESTS" $JIRA_TEXT
- then
- fail "Test $JIRA_NAME has tag NO PRECOMMIT TESTS"
- fi
- # ensure the patch is actually in the correct state
- if ! grep -q 'Patch Available' $JIRA_TEXT
- then
- fail "$JIRA_NAME is not \"Patch Available\". Exiting."
- fi
- # pull attachments from JIRA (hack stolen from hadoop since rest api doesn't show attachments)
- export PATCH_URL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' $JIRA_TEXT | \
- grep -v -e 'htm[l]*$' | sort | tail -1 | \
- grep -o '/jira/secure/attachment/[0-9]*/[^"]*')
- if [[ -z "$PATCH_URL" ]]
- then
- fail "Unable to find attachment for $JIRA_NAME"
- fi
- # ensure attachment has not already been tested
- ATTACHMENT_ID=$(basename $(dirname $PATCH_URL))
- if test -n "$BUILD_TAG"
- then
- build_postfix=" - ${BUILD_TAG%-*}"
- fi
- if grep -q "ATTACHMENT ID: $ATTACHMENT_ID $build_postfix" $JIRA_TEXT
- then
- fail "Attachment $ATTACHMENT_ID is already tested for $JIRA_NAME"
- fi
- # validate the patch name, parse branch if needed
- shopt -s nocasematch
-
- # Get branch name
- PATCH_NAME=$(basename $PATCH_URL)
- BRANCH=$(get_branch_name $PATCH_NAME $BRANCH)
- echo "Assuming branch $BRANCH"
-
- shopt -u nocasematch
- # append mr2 if needed
- if [[ $BRANCH =~ (mr1|mr2)$ ]]
- then
- profile=$BRANCH
- else
- profile=${BRANCH}-mr2
- fi
- export BUILD_PROFILE=$profile
- build_opts=""
- if grep -q "CLEAR LIBRARY CACHE" $JIRA_TEXT
- then
- echo "Clearing library cache before starting test"
- build_opts="--clearLibraryCache"
- fi
- export BUILD_OPTS=$build_opts
-}
-
-# Checks if a specified URL patch contains HMS upgrade changes
-# Returns 0 if there are changes; non-zero value otherwise.
-patch_contains_hms_upgrade() {
- curl -s "$1" | grep "^diff.*metastore/scripts/upgrade/" >/dev/null
-}
-
-string_to_upper_case() {
- local str="$1"
-
- echo "$str" | tr '[:lower:]' '[:upper:]'
-}
-
-get_jenkins_job_url() {
- local branch="$1"
- local varname=`string_to_upper_case $branch`_URL
- local joburl=`eval echo \\$${varname}`
-
- echo $joburl
-}
diff --git a/dev-support/jenkins-submit-build.sh b/dev-support/jenkins-submit-build.sh
deleted file mode 100644
index 8336f1ebb06..00000000000
--- a/dev-support/jenkins-submit-build.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -e
-. jenkins-common.sh
-export JIRA_NAME="HIVE-${ISSUE_NUM}"
-export JIRA_ROOT_URL="https://issues.apache.org"
-export BRANCH=trunk
-echo $JIRA_NAME
-
-process_jira
-
-# sanity check the profile
-case "$BUILD_PROFILE" in
- trunk-mr2)
- test -n "$TRUNK_URL" || fail "TRUNK_URL must be specified"
- url="$TRUNK_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- branch-1-mr1|branch-1-mr2)
- test -n "$BRANCH1_URL" || fail "BRANCH1_URL must be specified"
- url="$BRANCH1_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- spark-mr2)
- test -n "$SPARK_URL" || fail "SPARK_URL must be specified"
- url="$SPARK_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- encryption-mr2)
- test -n "$ENCRYPTION_URL" || fail "ENCRYPTION_URL must be specified"
- url="$ENCRYPTION_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- parquet-mr2)
- test -n "$PARQUET_URL" || fail "PARQUET_URL must be specified"
- url="$PARQUET_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- beeline-cli-mr2)
- test -n "$BEELINE_CLI_URL" || fail "BEELINE_CLI_URL must be specified"
- url="$BEELINE_CLI_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- java8-mr2)
- test -n "$JAVA8_URL" || fail "JAVA8_URL must be specified"
- url="$JAVA8_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- branch-2.1-mr2)
- test -n "$BRANCH21_URL" || fail "BRANCH21_URL must be specified"
- url="$BRANCH21_URL&ISSUE_NUM=$ISSUE_NUM"
- ;;
- *)
- echo "Unknown profile '$BUILD_PROFILE'"
- exit 1
- ;;
-esac
-
-# Execute jenkins job for HMS upgrade tests if needed
-if patch_contains_hms_upgrade "${JIRA_ROOT_URL}$PATCH_URL"; then
- test -n "$HMS_UPGRADE_URL" || fail "HMS_UPGRADE_URL must be specified"
- echo "Calling HMS upgrade testing job..."
- curl -v -i "${HMS_UPGRADE_URL}&ISSUE_NUM=${ISSUE_NUM}&BRANCH=${BRANCH}"
-fi
-
-# Execute jenkins job for specific profile
-echo "Calling Precommit $BRANCH Build..."
-curl -v -i "$url"
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerLocalScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerLocalScan.java
index fc005ab4bde..af868d9f9a0 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerLocalScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerLocalScan.java
@@ -103,8 +103,7 @@ public class TestHiveIcebergStorageHandlerLocalScan {
@Before
public void before() throws IOException {
testTables = HiveIcebergStorageHandlerTestUtils.testTables(shell, testTableType, temp);
- // Uses spark as an engine so we can detect if we unintentionally try to use any execution engines
- HiveIcebergStorageHandlerTestUtils.init(shell, testTables, temp, "spark");
+ HiveIcebergStorageHandlerTestUtils.init(shell, testTables, temp, "mr");
}
@After
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index 66657d98b64..f020a2131e1 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -154,8 +154,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
@Before
public void before() throws IOException {
testTables = HiveIcebergStorageHandlerTestUtils.testTables(shell, testTableType, temp);
- // Uses spark as an engine so we can detect if we unintentionally try to use any execution engines
- HiveIcebergStorageHandlerTestUtils.init(shell, testTables, temp, "spark");
+ HiveIcebergStorageHandlerTestUtils.init(shell, testTables, temp, "mr");
}
@After
diff --git a/itests/bin/download-spark.sh b/itests/bin/download-spark.sh
deleted file mode 100644
index c68d5362d10..00000000000
--- a/itests/bin/download-spark.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-download() {
- url=$1;
- finalName=$2
- tarName=$(basename $url)
- rm -rf $BASE_DIR/$finalName
- if [[ ! -f $DOWNLOAD_DIR/$tarName ]]
- then
- curl -Sso $DOWNLOAD_DIR/$tarName $url
- else
- local md5File="$tarName".md5sum
- curl -Sso $DOWNLOAD_DIR/$md5File "$url".md5sum
- cd $DOWNLOAD_DIR
- if type md5sum >/dev/null && ! md5sum -c $md5File; then
- curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1
- elif type md5 >/dev/null; then
- md5chksum=$(md5 -r $tarName)
- if [[ $(< $md5File) != $md5chksum ]]; then
- curl -Sso $DOWNLOAD_DIR/$tarName $url || return 1
- fi
- fi
-
- cd -
- fi
-
- tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR
- mv $BASE_DIR/spark-$SPARK_VERSION-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName
-}
-
-set -x
-/bin/pwd
-BASE_DIR=./target
-HIVE_ROOT=$BASE_DIR/../../../
-DOWNLOAD_DIR=./../thirdparty
-SPARK_VERSION=$1
-
-mkdir -p $DOWNLOAD_DIR
-download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-$SPARK_VERSION-bin-hadoop3-beta1-without-hive.tgz" "spark"
-cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/
-
diff --git a/itests/hive-jmh/src/main/resources/log4j2.properties b/itests/hive-jmh/src/main/resources/log4j2.properties
index 33572409842..6bbe9e966e8 100644
--- a/itests/hive-jmh/src/main/resources/log4j2.properties
+++ b/itests/hive-jmh/src/main/resources/log4j2.properties
@@ -48,13 +48,7 @@ appender.DRFA.strategy.type = DefaultRolloverStrategy
appender.DRFA.strategy.max = 30
# list of all loggers
-loggers = SparkIMain, SparkILoop, Jetty, AbstractLifeCycle
-
-logger.SparkIMain.name = org.apache.hive-jmh.repl.SparkIMain$exprTyper
-logger.SparkIMain.level = INFO
-
-logger.SparkILoop.name = org.apache.hive-jmh.repl.SparkILoop$SparkILoopInterpreter
-logger.SparkILoop.level = INFO
+loggers = Jetty, AbstractLifeCycle
logger.Jetty.name = org.eclipse.jetty
logger.Jetty.level = WARN
diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index f8598eb1d21..0b78e608fce 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -25,7 +25,6 @@
<name>Hive Integration - Unit Tests</name>
<properties>
<hive.path.to.root>../..</hive.path.to.root>
- <spark.home>${basedir}/${hive.path.to.root}/itests/hive-unit/target/spark</spark.home>
<testcontainers.version>1.15.2</testcontainers.version>
<htmlunit.version>2.45.0</htmlunit.version>
</properties>
@@ -445,57 +444,6 @@
<scope>test</scope>
</dependency>
</dependencies>
- <profiles>
- <profile>
- <id>spark-test</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.esotericsoftware.kryo</groupId>
- <artifactId>kryo</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>download-spark</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <exec executable="bash" dir="${basedir}" failonerror="true">
- <arg line="../bin/download-spark.sh ${spark.version}"/>
- </exec>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
<build>
<plugins>
<plugin>
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSmallTableCacheEviction.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSmallTableCacheEviction.java
deleted file mode 100644
index ef4dbe1bf4f..00000000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSmallTableCacheEviction.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.logging.log4j.core.util.ReflectionUtil;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.File;
-import java.nio.file.Paths;
-import java.util.StringJoiner;
-import java.util.UUID;
-
-/**
- * Test if the small table cache is evicted, when a new query is executed.
- */
-@Ignore("HIVE-22944: Kryo 5 upgrade conflicts with Spark, which is not supported anymore")
-public class TestSmallTableCacheEviction {
-
- private String smallTableName1;
- private String smallTableName2;
- private String largeTableName;
- private SmallTableCache.SmallTableLocalCache<String, MapJoinTableContainer> innerCache;
- private HiveConf conf;
-
- @Before
- public void setUp() throws Exception {
- String confDir = "../../data/conf/spark/local/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
- conf = new HiveConf();
- conf.setBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN_TOMAPJOIN, true);
- conf.setBoolVar(HiveConf.ConfVars.HIVECONVERTJOIN, true);
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestSmallTableCacheEviction-local-dir").toString());
-
- SessionState.start(conf);
-
- largeTableName = "large" + UUID.randomUUID().toString().replace("-", "");
- smallTableName1 = "small" + UUID.randomUUID().toString().replace("-", "");
- smallTableName2 = "small" + UUID.randomUUID().toString().replace("-", "");
-
- innerCache = (SmallTableCache.SmallTableLocalCache<String, MapJoinTableContainer>) ReflectionUtil
- .getStaticFieldValue(SmallTableCache.class.getDeclaredField("TABLE_CONTAINER_CACHE"));
-
- // Create test data
- Driver driver = createDriver();
- MockDataBuilder.builder(smallTableName1).numberOfRows(1).create(driver);
- MockDataBuilder.builder(smallTableName2).numberOfRows(2).create(driver);
- MockDataBuilder.builder(largeTableName).numberOfRows(3).create(driver);
- driver.destroy();
- }
-
- @After
- public void tearDown() throws CommandProcessorException {
- Driver driver = createDriver();
- driver.run("drop table if exists " + smallTableName1);
- driver.run("drop table if exists " + smallTableName2);
- driver.run("drop table if exists " + largeTableName);
- driver.destroy();
- }
-
- @Test
- public void testSmallTableEvictionIfNewQueryIsExecuted() throws Exception {
-
- for (int i = 0; i < 2; i++) {
- Driver driver = null;
- try {
- driver = createDriver();
- String simpleJoinQuery = "select large.col, s1.col, s2.col from " + largeTableName + " large join " +
- smallTableName1 + " s1 on s1.col = large.col join " + smallTableName2 + " s2 on s2.col = large.col";
- driver.run(simpleJoinQuery);
- Assert.assertEquals(2, innerCache.size());
- } finally {
- if (driver != null) {
- driver.destroy();
- }
- }
- }
- }
-
- private Driver createDriver() {
- return new Driver(new QueryState.Builder()
- .withGenerateNewQueryId(true)
- .withHiveConf(conf).build(),
- null);
- }
-
- /**
- * Helper class to create a simple table with n number of rows in it.
- */
- private static final class MockDataBuilder {
-
- private final String tableName;
- private int numberOfRows;
-
- private MockDataBuilder(String tableName) {
- this.tableName = tableName;
- }
-
- public MockDataBuilder numberOfRows(int numberOfRows) {
- this.numberOfRows = numberOfRows;
- return this;
- }
-
- public void create(Driver driver) throws CommandProcessorException {
- driver.run("create table " + tableName + " (col int)");
-
- if (numberOfRows > 0) {
- StringJoiner query = new StringJoiner(",", "insert into " + tableName + " values ", "");
- for (int i = 0; i < numberOfRows; i++) {
- query.add("(" + Integer.toString(i + 1) + ")");
- }
- driver.run(query.toString());
- }
- }
-
- public static MockDataBuilder builder(String tableName) {
- return new MockDataBuilder(tableName);
- }
-
- }
-}
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
deleted file mode 100644
index 842aa9f703a..00000000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
-import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.net.MalformedURLException;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-public class TestSparkSessionTimeout {
-
- @Test
- public void testSparkSessionTimeout()
- throws HiveException, InterruptedException, MalformedURLException, CommandProcessorException {
- String confDir = "../../data/conf/spark/standalone/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
-
- HiveConf conf = new HiveConf();
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestSparkSessionTimeout-testSparkSessionTimeout-local-dir").toString());
-
- SessionState.start(conf);
-
- runTestSparkSessionTimeout(conf, 1);
- }
-
- @Test
- public void testMultiSessionSparkSessionTimeout() throws InterruptedException,
- ExecutionException {
- List<Future<Void>> futures = new ArrayList<>();
- ExecutorService es = Executors.newFixedThreadPool(10);
- for (int i = 0; i < 10; i++) {
- futures.add(es.submit(() -> {
- String confDir = "../../data/conf/spark/local/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
-
- HiveConf conf = new HiveConf();
- conf.setBoolVar(HiveConf.ConfVars.SPARK_OPTIMIZE_SHUFFLE_SERDE, false);
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestSparkSessionTimeout-testMultiSessionSparkSessionTimeout-local-dir").toString());
-
- SessionState.start(conf);
-
- runTestSparkSessionTimeout(conf, 1);
- return null;
- }));
- }
- for (Future<Void> future : futures) {
- future.get();
- }
- }
-
- @Test
- public void testSparkSessionMultipleTimeout()
- throws HiveException, InterruptedException, MalformedURLException, CommandProcessorException {
- String confDir = "../../data/conf/spark/standalone/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
-
- HiveConf conf = new HiveConf();
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestSparkSessionTimeout-testSparkSessionMultipleTimeout-local-dir").toString());
-
- SessionState.start(conf);
-
- runTestSparkSessionTimeout(conf, 2);
- }
-
- private void runTestSparkSessionTimeout(HiveConf conf, int sleepRunIteration)
- throws HiveException, InterruptedException, CommandProcessorException {
- conf.setVar(HiveConf.ConfVars.SPARK_SESSION_TIMEOUT, "5s");
- conf.setVar(HiveConf.ConfVars.SPARK_SESSION_TIMEOUT_PERIOD, "1s");
-
- String tableName = "test" + UUID.randomUUID().toString().replace("-", "");
-
- Driver driver = null;
-
- try {
- driver = new Driver(new QueryState.Builder()
- .withGenerateNewQueryId(true)
- .withHiveConf(conf).build(),
- null);
-
- SparkSession sparkSession = SparkUtilities.getSparkSession(conf, SparkSessionManagerImpl
- .getInstance());
-
- driver.run("create table " + tableName + " (col int)");
- driver.run("select * from " + tableName + " order by col");
-
- for (int i = 0; i < sleepRunIteration; i++) {
- Thread.sleep(10000);
-
- Assert.assertFalse(sparkSession.isOpen());
-
- driver.run("select * from " + tableName + " order by col");
- }
- } finally {
- if (driver != null) {
- driver.run("drop table if exists " + tableName);
- driver.destroy();
- }
- }
- }
-}
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
deleted file mode 100644
index 9b433a2e1a6..00000000000
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkStatistics.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import com.google.common.collect.Lists;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Driver;
-import org.apache.hadoop.hive.ql.TaskQueue;
-import org.apache.hadoop.hive.ql.QueryState;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.spark.Statistic.SparkStatistic;
-import org.apache.hadoop.hive.ql.exec.spark.Statistic.SparkStatisticsNames;
-import org.apache.hadoop.hive.ql.processors.CommandProcessorException;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.File;
-import java.net.MalformedURLException;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-public class TestSparkStatistics {
-
- @Test
- public void testSparkStatistics() throws MalformedURLException, CommandProcessorException {
- String confDir = "../../data/conf/spark/standalone/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
- HiveConf conf = new HiveConf();
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestSparkStatistics-local-dir").toString());
-
- SessionState.start(conf);
-
- Driver driver = null;
-
- try {
- driver = new Driver(new QueryState.Builder()
- .withGenerateNewQueryId(true)
- .withHiveConf(conf).build(),
- null);
-
- driver.run("create table test (col int)");
- Assert.assertEquals(0, driver.compile("select * from test order by col", true));
-
- List<SparkTask> sparkTasks = Utilities.getSparkTasks(driver.getPlan().getRootTasks());
- Assert.assertEquals(1, sparkTasks.size());
-
- SparkTask sparkTask = sparkTasks.get(0);
-
- TaskQueue taskQueue = new TaskQueue(driver.getContext());
- taskQueue.prepare(driver.getPlan());
-
- sparkTask.initialize(driver.getQueryState(), driver.getPlan(), taskQueue, driver.getContext());
- Assert.assertEquals(0, sparkTask.execute());
-
- Assert.assertNotNull(sparkTask.getSparkStatistics());
-
- List<SparkStatistic> sparkStats = Lists.newArrayList(sparkTask.getSparkStatistics()
- .getStatisticGroup(SparkStatisticsNames.SPARK_GROUP_NAME).getStatistics());
-
- Assert.assertEquals(26, sparkStats.size());
-
- Map<String, String> statsMap = sparkStats.stream().collect(
- Collectors.toMap(SparkStatistic::getName, SparkStatistic::getValue));
-
- Assert.assertTrue(Long.parseLong(statsMap.get(SparkStatisticsNames.TASK_DURATION_TIME)) > 0);
- Assert.assertTrue(Long.parseLong(statsMap.get(SparkStatisticsNames.EXECUTOR_CPU_TIME)) > 0);
- Assert.assertTrue(
- Long.parseLong(statsMap.get(SparkStatisticsNames.EXECUTOR_DESERIALIZE_CPU_TIME)) > 0);
- Assert.assertTrue(
- Long.parseLong(statsMap.get(SparkStatisticsNames.EXECUTOR_DESERIALIZE_TIME)) > 0);
- Assert.assertTrue(Long.parseLong(statsMap.get(SparkStatisticsNames.EXECUTOR_RUN_TIME)) > 0);
- } finally {
- if (driver != null) {
- driver.run("drop table if exists test");
- driver.destroy();
- }
- }
- }
-}
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
deleted file mode 100644
index 341da33f1a4..00000000000
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithLocalClusterSpark.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.jdbc;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-import java.io.File;
-import java.net.MalformedURLException;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hive.jdbc.miniHS2.MiniHS2;
-import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.session.HiveSessionHook;
-import org.apache.hive.service.cli.session.HiveSessionHookContext;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * This class is cloned from TestJdbcWithMiniMR, except use Spark as the execution engine.
- */
-public class TestJdbcWithLocalClusterSpark {
- public static final String TEST_TAG = "miniHS2.localClusterSpark.tag";
- public static final String TEST_TAG_VALUE = "miniHS2.localClusterSpark.value";
- public static class LocalClusterSparkSessionHook implements HiveSessionHook {
- @Override
- public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
- sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
- }
- }
-
- private static MiniHS2 miniHS2 = null;
- private static HiveConf conf;
- private static Path dataFilePath;
- private static String dbName = "mrTestDb";
- private Connection hs2Conn = null;
- private Statement stmt;
-
- private static HiveConf createHiveConf() throws MalformedURLException {
- String confDir = "../../data/conf/spark/standalone/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
- HiveConf conf = new HiveConf();
- // FIXME: Hadoop3 made the incompatible change for dfs.client.datanode-restart.timeout
- // while spark2 is still using Hadoop2.
- // Spark requires Hive to support Hadoop3 first then Spark can start
- // working on Hadoop3 support. Remove this after Spark supports Hadoop3.
- conf.set("dfs.client.datanode-restart.timeout", "30");
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestJdbcWithLocalClusterSpark-local-dir").toString());
- return conf;
- }
-
- @BeforeClass
- public static void beforeTest() throws Exception {
- Class.forName(MiniHS2.getJdbcDriverName());
- conf = createHiveConf();
- conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- String dataFileDir = conf.get("test.data.files").replace('\\', '/')
- .replace("c:", "");
- dataFilePath = new Path(dataFileDir, "kv1.txt");
- DriverManager.setLoginTimeout(0);
- conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- miniHS2 = new MiniHS2(conf, MiniClusterType.MR);
- Map<String, String> overlayProps = new HashMap<String, String>();
- overlayProps.put(ConfVars.HIVE_SERVER2_SESSION_HOOK.varname,
- LocalClusterSparkSessionHook.class.getName());
- miniHS2.start(overlayProps);
- createDb();
- }
-
- // setup DB
- private static void createDb() throws Exception {
- Connection conn = DriverManager.
- getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
- Statement stmt2 = conn.createStatement();
- stmt2.execute("DROP DATABASE IF EXISTS " + dbName + " CASCADE");
- stmt2.execute("CREATE DATABASE " + dbName);
- stmt2.close();
- conn.close();
- }
-
- @Before
- public void setUp() throws Exception {
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
- System.getProperty("user.name"), "bar");
- stmt = hs2Conn.createStatement();
- stmt.execute("USE " + dbName);
- }
-
- @After
- public void tearDown() throws Exception {
- if (hs2Conn != null) {
- hs2Conn.close();
- }
- }
-
- @AfterClass
- public static void afterTest() throws Exception {
- if (miniHS2 != null && miniHS2.isStarted()) {
- miniHS2.stop();
- }
- }
-
- /**
- * Verify that the connection to HS2 with MiniMr is successful.
- * @throws Exception
- */
- @Test
- public void testConnection() throws Exception {
- // the session hook should set the property
- verifyProperty(TEST_TAG, TEST_TAG_VALUE);
- }
-
- /**
- * Run nonMr query.
- * @throws Exception
- */
- @Test
- public void testNonSparkQuery() throws Exception {
- String tableName = "testTab1";
- String resultVal = "val_238";
- String queryStr = "SELECT * FROM " + tableName;
-
- testKvQuery(tableName, queryStr, resultVal);
- }
-
- /**
- * Run nonMr query.
- * @throws Exception
- */
- @Test
- public void testSparkQuery() throws Exception {
- String tableName = "testTab2";
- String resultVal = "val_238";
- String queryStr = "SELECT * FROM " + tableName
- + " where value = '" + resultVal + "'";
-
- testKvQuery(tableName, queryStr, resultVal);
- }
-
- @Test
- public void testPermFunc() throws Exception {
-
- // This test assumes the hive-contrib JAR has been built as part of the Hive build.
- // Also dependent on the UDFExampleAdd class within that JAR.
- String udfClassName = "org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd";
- String mvnRepo = System.getProperty("maven.local.repository");
- String hiveVersion = System.getProperty("hive.version");
- String jarFileName = "hive-contrib-" + hiveVersion + ".jar";
- String[] pathParts = {
- "org", "apache", "hive",
- "hive-contrib", hiveVersion, jarFileName
- };
-
- // Create path to hive-contrib JAR on local filesystem
- Path contribJarPath = new Path(mvnRepo);
- for (String pathPart : pathParts) {
- contribJarPath = new Path(contribJarPath, pathPart);
- }
- FileSystem localFs = FileSystem.getLocal(conf);
- assertTrue("Hive contrib JAR exists at " + contribJarPath, localFs.exists(contribJarPath));
-
- String hdfsJarPathStr = "hdfs:///" + jarFileName;
- Path hdfsJarPath = new Path(hdfsJarPathStr);
-
- // Copy JAR to DFS
- FileSystem dfs = miniHS2.getDFS().getFileSystem();
- dfs.copyFromLocalFile(contribJarPath, hdfsJarPath);
- assertTrue("Verify contrib JAR copied to HDFS at " + hdfsJarPath, dfs.exists(hdfsJarPath));
-
- // Register function
- String queryStr = "CREATE FUNCTION example_add AS '" + udfClassName + "'"
- + " USING JAR '" + hdfsJarPathStr + "'";
- stmt.execute(queryStr);
-
- // Call describe
- ResultSet res;
- res = stmt.executeQuery("DESCRIBE FUNCTION " + dbName + ".example_add");
- checkForNotExist(res);
-
- // Use UDF in query
- String tableName = "testTab3";
- setupKv1Tabs(tableName);
- res = stmt.executeQuery("SELECT EXAMPLE_ADD(1, 2) FROM " + tableName + " LIMIT 1");
- assertTrue("query has results", res.next());
- assertEquals(3, res.getInt(1));
- assertFalse("no more results", res.next());
-
- // A new connection should be able to call describe/use function without issue
- Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
- System.getProperty("user.name"), "bar");
- Statement stmt2 = conn2.createStatement();
- stmt2.execute("USE " + dbName);
- res = stmt2.executeQuery("DESCRIBE FUNCTION " + dbName + ".example_add");
- checkForNotExist(res);
-
- res = stmt2.executeQuery("SELECT " + dbName + ".example_add(1, 1) FROM " + tableName + " LIMIT 1");
- assertTrue("query has results", res.next());
- assertEquals(2, res.getInt(1));
- assertFalse("no more results", res.next());
-
- stmt.execute("DROP TABLE " + tableName);
- }
-
- @Test
- public void testTempTable() throws Exception {
- // Create temp table with current connection
- String tempTableName = "tmp1";
- stmt.execute("CREATE TEMPORARY TABLE " + tempTableName + " (key string, value string)");
- stmt.execute("load data local inpath '"
- + dataFilePath.toString() + "' into table " + tempTableName);
-
- String resultVal = "val_238";
- String queryStr = "SELECT * FROM " + tempTableName
- + " where value = '" + resultVal + "'";
- verifyResult(queryStr, resultVal, 2);
-
- // A second connection should not be able to see the table
- Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
- System.getProperty("user.name"), "bar");
- Statement stmt2 = conn2.createStatement();
- stmt2.execute("USE " + dbName);
- boolean gotException = false;
- try {
- stmt2.executeQuery(queryStr);
- } catch (SQLException err) {
- // This is expected to fail.
- assertTrue("Expecting table not found error, instead got: " + err,
- err.getMessage().contains("Table not found"));
- gotException = true;
- }
- assertTrue("Exception while querying non-existing temp table", gotException);
- }
-
- private void checkForNotExist(ResultSet res) throws Exception {
- int numRows = 0;
- while (res.next()) {
- numRows++;
- String strVal = res.getString(1);
- assertEquals("Should not find 'not exist'", -1, strVal.toLowerCase().indexOf("not exist"));
- }
- assertTrue("Rows returned from describe function", numRows > 0);
- }
-
- /**
- * Verify if the given property contains the expected value.
- * @param propertyName
- * @param expectedValue
- * @throws Exception
- */
- private void verifyProperty(String propertyName, String expectedValue) throws Exception {
- Statement stmt = hs2Conn .createStatement();
- ResultSet res = stmt.executeQuery("set " + propertyName);
- assertTrue(res.next());
- String[] results = res.getString(1).split("=");
- assertEquals("Property should be set", results.length, 2);
- assertEquals("Property should be set", expectedValue, results[1]);
- }
-
- // create tables, verify query
- private void testKvQuery(String tableName, String queryStr, String resultVal)
- throws SQLException {
- setupKv1Tabs(tableName);
- verifyResult(queryStr, resultVal, 2);
- stmt.execute("DROP TABLE " + tableName);
- }
-
- // create table and pupulate with kv1.txt
- private void setupKv1Tabs(String tableName) throws SQLException {
- Statement stmt = hs2Conn.createStatement();
- // create table
- stmt.execute("CREATE TABLE " + tableName
- + " (under_col INT COMMENT 'the under column', value STRING)"
- + " COMMENT ' test table'");
-
- // load data
- stmt.execute("load data local inpath '"
- + dataFilePath.toString() + "' into table " + tableName);
- }
-
- // run given query and validate expecated result
- private void verifyResult(String queryStr, String expString, int colPos)
- throws SQLException {
- ResultSet res = stmt.executeQuery(queryStr);
- assertTrue(res.next());
- assertEquals(expString, res.getString(colPos));
- res.close();
- }
-}
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java
index ca77db0dd43..1923e262fa4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2ErasureCoding.java
@@ -18,12 +18,9 @@
package org.apache.hive.jdbc;
-import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
-import java.net.MalformedURLException;
-import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -64,27 +61,14 @@ public class TestJdbcWithMiniHS2ErasureCoding {
private static HiveConf conf;
private Connection hs2Conn = null;
- private static HiveConf createHiveOnSparkConf() throws MalformedURLException {
- String confDir = "../../data/conf/spark/standalone/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
- HiveConf hiveConf = new HiveConf();
- // Tell dfs not to consider load when choosing a datanode as this can cause failure as
- // in a test we do not have spare datanode capacity.
- hiveConf.setBoolean("dfs.namenode.redundancy.considerLoad", false);
- hiveConf.set("hive.spark.client.connect.timeout", "30000ms");
- hiveConf.set("spark.local.dir",
- Paths.get(System.getProperty("test.tmp.dir"), "TestJdbcWithMiniHS2ErasureCoding-local-dir")
- .toString());
- return hiveConf;
- }
-
/**
* Setup a mini HS2 with miniMR.
*/
@BeforeClass
public static void beforeTest() throws Exception {
Class.forName(MiniHS2.getJdbcDriverName());
- conf = createHiveOnSparkConf();
+ conf = new HiveConf();
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
DriverManager.setLoginTimeout(0);
miniHS2 = new MiniHS2.Builder()
.withConf(conf)
@@ -155,7 +139,6 @@ public class TestJdbcWithMiniHS2ErasureCoding {
stmt.execute("INSERT INTO TABLE " + tableName
+ " PARTITION (datestamp = '2014-09-24', i = 2)(userid,link) VALUES ('mac', 'superchunk.com')");
String explain = getExtendedExplain(stmt, "select userid from " + tableName);
- assertMatchAndCount(explain, " numFiles 4", 2);
assertMatchAndCount(explain, " numFilesErasureCoded 4", 2);
}
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
deleted file mode 100644
index f7586c108d9..00000000000
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestMultiSessionsHS2WithLocalClusterSpark.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.jdbc;
-
-import java.io.File;
-import java.net.MalformedURLException;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hive.jdbc.miniHS2.MiniHS2;
-import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.session.HiveSessionHook;
-import org.apache.hive.service.cli.session.HiveSessionHookContext;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestMultiSessionsHS2WithLocalClusterSpark {
- public static final String TEST_TAG = "miniHS2.localClusterSpark.tag";
- public static final String TEST_TAG_VALUE = "miniHS2.localClusterSpark.value";
- private static final int PARALLEL_NUMBER = 3;
-
- public static class LocalClusterSparkSessionHook implements HiveSessionHook {
- @Override
- public void run(HiveSessionHookContext sessionHookContext) throws HiveSQLException {
- sessionHookContext.getSessionConf().set(TEST_TAG, TEST_TAG_VALUE);
- }
- }
-
- private static MiniHS2 miniHS2 = null;
- private static HiveConf conf;
- private static Path dataFilePath;
- private static String dbName = "sparkTestDb";
- private ThreadLocal<Connection> localConnection = new ThreadLocal<Connection>();
- private ThreadLocal<Statement> localStatement = new ThreadLocal<Statement>();
- private ExecutorService pool = null;
-
-
- private static HiveConf createHiveConf() throws MalformedURLException {
- String confDir = "../../data/conf/spark/standalone/hive-site.xml";
- HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL());
- HiveConf conf = new HiveConf();
- conf.set("hive.exec.parallel", "true");
- conf.set("spark.deploy.defaultCores", "2");
- // FIXME: Hadoop3 made the incompatible change for dfs.client.datanode-restart.timeout
- // while spark2 is still using Hadoop2.
- // Spark requires Hive to support Hadoop3 first then Spark can start
- // working on Hadoop3 support. Remove this after Spark supports Hadoop3.
- conf.set("dfs.client.datanode-restart.timeout", "30");
- conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"),
- "TestMultiSessionsHS2WithLocalClusterSpark-local-dir").toString());
- return conf;
- }
-
- @BeforeClass
- public static void beforeTest() throws Exception {
- Class.forName(MiniHS2.getJdbcDriverName());
- conf = createHiveConf();
- conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- String dataFileDir = conf.get("test.data.files").replace('\\', '/')
- .replace("c:", "");
- dataFilePath = new Path(dataFileDir, "kv1.txt");
- DriverManager.setLoginTimeout(0);
- conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
- miniHS2 = new MiniHS2(conf, MiniClusterType.MR);
- Map<String, String> overlayProps = new HashMap<String, String>();
- overlayProps.put(ConfVars.HIVE_SERVER2_SESSION_HOOK.varname,
- LocalClusterSparkSessionHook.class.getName());
- miniHS2.start(overlayProps);
- createDb();
- }
-
- // setup DB
- private static void createDb() throws Exception {
- Connection conn = DriverManager.
- getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar");
- Statement stmt2 = conn.createStatement();
- stmt2.execute("DROP DATABASE IF EXISTS " + dbName + " CASCADE");
- stmt2.execute("CREATE DATABASE " + dbName);
- stmt2.close();
- conn.close();
- }
-
- @Before
- public void setUp() throws Exception {
- pool = Executors.newFixedThreadPool(PARALLEL_NUMBER,
- new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Test-Thread-%d").build());
- createConnection();
- }
-
- @After
- public void tearDown() throws Exception {
- pool.shutdownNow();
- closeConnection();
- }
-
- private void createConnection() throws Exception {
- Connection connection = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
- System.getProperty("user.name"), "bar");
- Statement statement = connection.createStatement();
- localConnection.set(connection);
- localStatement.set(statement);
- statement.execute("USE " + dbName);
- }
-
- private void closeConnection() throws SQLException {
- if (localStatement.get() != null) {
- localStatement.get().close();
- }
-
- if (localConnection.get() != null) {
- localConnection.get().close();
- }
- }
-
- @AfterClass
- public static void afterTest() throws Exception {
- if (miniHS2 != null && miniHS2.isStarted()) {
- miniHS2.stop();
- }
- }
-
- /**
- * Run nonSpark query
- *
- * @throws Exception
- */
- @Test
- public void testNonSparkQuery() throws Exception {
- String tableName = "kvTable1";
- setupTable(tableName);
- Callable<Void> runNonSparkQuery = getNonSparkQueryCallable(tableName);
- runInParallel(runNonSparkQuery);
- dropTable(tableName);
- }
-
- /**
- * Run spark query
- *
- * @throws Exception
- */
- @Test
- public void testSparkQuery() throws Exception {
- String tableName = "kvTable2";
- setupTable(tableName);
- Callable<Void> runSparkQuery = getSparkQueryCallable(tableName);
- runInParallel(runSparkQuery);
- dropTable(tableName);
- }
-
- private void runInParallel(Callable<Void> runNonSparkQuery) throws InterruptedException, ExecutionException {
- List<Future> futureList = new LinkedList<Future>();
- for (int i = 0; i < PARALLEL_NUMBER; i++) {
- Future future = pool.submit(runNonSparkQuery);
- futureList.add(future);
- }
-
- for (Future future : futureList) {
- future.get();
- }
- }
-
- private Callable<Void> getNonSparkQueryCallable(final String tableName) {
- return new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- String resultVal = "val_238";
- String queryStr = "SELECT * FROM " + tableName;
- testKvQuery(queryStr, resultVal);
- return null;
- }
- };
- }
-
- private Callable<Void> getSparkQueryCallable(final String tableName) {
- return new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- String resultVal = "val_238";
- String queryStr = "SELECT * FROM " + tableName +
- " where value = '" + resultVal + "'";
- testKvQuery(queryStr, resultVal);
- return null;
- }
- };
- }
-
- private void testKvQuery(String queryStr, String resultVal)
- throws Exception {
- createConnection();
- verifyResult(queryStr, resultVal, 2);
- closeConnection();
- }
-
- // create table and load kv1.txt
- private void setupTable(String tableName) throws SQLException {
- Statement statement = localStatement.get();
- // create table
- statement.execute("CREATE TABLE " + tableName
- + " (under_col INT COMMENT 'the under column', value STRING)"
- + " COMMENT ' test table'");
-
- // load data
- statement.execute("LOAD DATA LOCAL INPATH '"
- + dataFilePath.toString() + "' INTO TABLE " + tableName);
- }
-
- private void dropTable(String tableName) throws SQLException {
- localStatement.get().execute("DROP TABLE " + tableName);
- }
-
- // run given query and validate expected result
- private void verifyResult(String queryStr, String expString, int colPos)
- throws SQLException {
- ResultSet res = localStatement.get().executeQuery(queryStr);
- assertTrue(res.next());
- assertEquals(expString, res.getString(colPos));
- res.close();
- }
-}
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
index cbf25db5a93..72167d7db2c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
@@ -90,26 +90,15 @@ public class TestRestrictedList {
addToExpectedRestrictedMap("hive.server2.authentication.ldap.groupClassKey");
addToExpectedRestrictedMap("hive.server2.authentication.ldap.customLDAPQuery");
addToExpectedRestrictedMap("hive.server2.service.users");
- addToExpectedRestrictedMap("hive.spark.client.channel.log.level");
- addToExpectedRestrictedMap("hive.spark.client.secret.bits");
- addToExpectedRestrictedMap("hive.spark.client.rpc.server.address");
- addToExpectedRestrictedMap("hive.spark.client.rpc.server.port");
- addToExpectedRestrictedMap("hive.spark.client.rpc.sasl.mechanisms");
addToExpectedRestrictedMap("hive.query.max.length");
addToExpectedRestrictedMap("hive.druid.broker.address.default");
addToExpectedRestrictedMap("hive.druid.coordinator.address.default");
addToExpectedRestrictedMap("hikaricp.test");
addToExpectedRestrictedMap("hadoop.bin.path");
addToExpectedRestrictedMap("yarn.bin.path");
- addToExpectedRestrictedMap("hive.spark.client.connect.timeout");
- addToExpectedRestrictedMap("hive.spark.client.server.connect.timeout");
- addToExpectedRestrictedMap("hive.spark.client.rpc.max.size");
- addToExpectedRestrictedMap("hive.spark.client.rpc.threads");
addToExpectedRestrictedMap("_hive.local.session.path");
addToExpectedRestrictedMap("_hive.tmp_table_space");
addToExpectedRestrictedMap("_hive.hdfs.session.path");
- addToExpectedRestrictedMap("hive.spark.client.rpc.server.address");
- addToExpectedRestrictedMap("spark.home");
addToExpectedRestrictedMap("hive.privilege.synchronizer.interval");
addToExpectedRestrictedMap("hive.driver.parallel.compilation.global.limit");
addToExpectedRestrictedMap("hive.zookeeper.ssl.keystore.location");
diff --git a/itests/pom.xml b/itests/pom.xml
index 6b50c856970..c74ae7d9b0f 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -484,12 +484,6 @@
</dependencies>
</dependencyManagement>
<profiles>
- <profile>
- <id>spark-test</id>
- <modules>
- <module>qtest-spark</module>
- </modules>
- </profile>
<profile>
<id>iceberg</id>
<modules>
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
deleted file mode 100644
index ac02f06985c..00000000000
--- a/itests/qtest-spark/pom.xml
+++ /dev/null
@@ -1,456 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-it</artifactId>
- <version>4.0.0-alpha-2-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <artifactId>hive-it-qfile-spark</artifactId>
- <packaging>jar</packaging>
- <name>Hive Integration - QFile Spark Tests</name>
- <properties>
- <hive.path.to.root>../..</hive.path.to.root>
- <!-- Determines the log level of the console logger, hive.log is independent of this-->
- <test.console.log.level>OFF</test.console.log.level>
- <!-- The following are to match the latest in spark project, overriding hive's versions -->
- <spark.kryo.version>2.21</spark.kryo.version>
- <qfile/>
- <qfile_regex/>
- <run_disabled>false</run_disabled>
- <execute.beeline.tests>false</execute.beeline.tests>
- <active.hadoop.version>${hadoop.version}</active.hadoop.version>
- <test.dfs.mkdir>-mkdir -p</test.dfs.mkdir>
- <spark.home>${basedir}/${hive.path.to.root}/itests/qtest-spark/target/spark</spark.home>
- </properties>
- <dependencies>
- <!-- dependencies are always listed in sorted order by groupId, artifactId -->
- <!-- test intra-project spark -->
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- <version>${jetty.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-security</artifactId>
- <version>${jetty.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-plus</artifactId>
- <version>${jetty.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <version>${jetty.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- <version>${jetty.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.esotericsoftware.kryo</groupId>
- <artifactId>kryo</artifactId>
- <version>${spark.kryo.version}</version>
- <scope>test</scope>
- </dependency>
- <!-- test intra-project -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-contrib</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-standalone-metastore-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-standalone-metastore-common</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-standalone-metastore-server</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-standalone-metastore-server</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-it-custom-serde</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-it-util</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-it-druid</artifactId>
- </exclusion>
- </exclusions>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-udf</artifactId>
- <scope>test</scope>
- </dependency>
- <!-- test inter-project -->
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-servlet</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-archives</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-hs</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-tests</artifactId>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-client</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-common</artifactId>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop-compat</artifactId>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-hadoop2-compat</artifactId>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.glassfish.web</groupId>
- <artifactId>javax.servlet.jsp</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-mapreduce</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>properties-maven-plugin</artifactId>
- <executions>
- <execution>
- <phase>initialize</phase>
- <goals>
- <goal>read-project-properties</goal>
- </goals>
- <configuration>
- <files>
- <file>${basedir}/../src/test/resources/testconfiguration.properties</file>
- </files>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>download-spark</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <exec executable="bash" dir="${basedir}" failonerror="true">
- <arg line="../bin/download-spark.sh ${spark.version}"/>
- </exec>
- </target>
- </configuration>
- </execution>
- <execution>
- <id>generate-tests-sources</id>
- <phase>generate-test-sources</phase>
- <configuration>
- <target>
- <property name="test.classpath" refid="maven.test.classpath"/>
- <echo message="${test.classpath}"/>
- <taskdef resource="net/sf/antcontrib/antcontrib.properties" classpathref="maven.plugin.classpath"/>
- <mkdir dir="${project.build.directory}/qfile-results/clientpositive/spark"/>
- <mkdir dir="${project.build.directory}/qfile-results/clientpositive/miniSparkOnYarn"/>
- <mkdir dir="${project.build.directory}/qfile-results/clientnegative/spark"/>
- <mkdir dir="${project.build.directory}/qfile-results/clientnegative/miniSparkOnYarn"/>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-test-sources</id>
- <phase>generate-test-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-test-sources/java</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <profiles>
- <profile>
- <id>qsplits</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>generate-split-tests</id>
- <phase>generate-sources</phase>
- <configuration>
- <target>
- <exec executable="bash" failonerror="true">
- <arg value="${basedir}/../bin/generate-cli-splits.sh"/>
- <arg value="src/test/java"/>
- <arg value="target/generated-test-sources"/>
- </exec>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>build-helper-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>add-test-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>add-test-source</goal>
- </goals>
- <configuration>
- <sources>
- <source>target/generated-test-sources</source>
- </sources>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-</project>
diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestLocalSparkCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestLocalSparkCliDriver.java
deleted file mode 100644
index 41da5fe84fd..00000000000
--- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestLocalSparkCliDriver.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class TestLocalSparkCliDriver {
-
- static CliAdapter adapter = new CliConfigs.LocalSparkCliConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
- }
-
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private String name;
- private File qfile;
-
- public TestLocalSparkCliDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-}
diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java
deleted file mode 100644
index c19d4dbe5ae..00000000000
--- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestMiniSparkOnYarnCliDriver.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.apache.hadoop.hive.cli.control.SplitSupport;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class TestMiniSparkOnYarnCliDriver {
-
- private static final int N_SPLITS = 5;
-
- static CliAdapter adapter = new CliConfigs.SparkOnYarnCliConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return SplitSupport.process(adapter.getParameters(), TestMiniSparkOnYarnCliDriver.class, N_SPLITS);
- }
-
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private String name;
- private File qfile;
-
- public TestMiniSparkOnYarnCliDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-
-}
diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java
deleted file mode 100644
index bf200b67b50..00000000000
--- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkCliDriver.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.apache.hadoop.hive.cli.control.SplitSupport;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class TestSparkCliDriver {
-
- private static final int N_SPLITS = 11;
-
- static CliAdapter adapter = new CliConfigs.SparkCliConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return SplitSupport.process(adapter.getParameters(), TestSparkCliDriver.class, N_SPLITS);
- }
-
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private String name;
- private File qfile;
-
- public TestSparkCliDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-
-}
diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkNegativeCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkNegativeCliDriver.java
deleted file mode 100644
index 9f8db15f7de..00000000000
--- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkNegativeCliDriver.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-@RunWith(Parameterized.class)
-public class TestSparkNegativeCliDriver {
-
- static CliAdapter adapter = new CliConfigs.SparkNegativeCliConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
- }
-
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private String name;
- private File qfile;
-
- public TestSparkNegativeCliDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-
-}
diff --git a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkTPCDS30TBPerfCliDriver.java b/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkTPCDS30TBPerfCliDriver.java
deleted file mode 100644
index cf6f413f5fd..00000000000
--- a/itests/qtest-spark/src/test/java/org/apache/hadoop/hive/cli/TestSparkTPCDS30TBPerfCliDriver.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-/**
- * The test exists only as a replacement of the old SparkPerfDriver.
- *
- * The test was never executed since Spark tests are broken at the time being and there was relevant discussion
- * to entirely remove supoort for Spark. Currently, the test script calle by maven cannot locate the Spark artifacts:
- * http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-2.4.5-bin-hadoop3-beta1-without-hive.tgz
- */
-@RunWith(Parameterized.class)
-public class TestSparkTPCDS30TBPerfCliDriver {
-
- static CliAdapter adapter = new CliConfigs.SparkTPCDS30TBCliConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
- }
-
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- /**
- * Rule for calling only {@link CliAdapter#setUp()} and {@link CliAdapter#tearDown()} before/after running each test.
- *
- * At the moment of writing this class the rule is mostly necessary for calling {@link CliAdapter#tearDown()} to avoid
- * state from one test pass to other (e.g., disabling one test should not disable subsequent ones).
- *
- * {@link CliAdapter#buildTestRule()} cannot not used since it is doing more than necessary for this test case. For
- * instance, we do not want to create and destroy the metastore after each query.
- */
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private final String name;
- private final File qfile;
-
- public TestSparkTPCDS30TBPerfCliDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-
-}
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index d12eca6d03f..9c9e5db787d 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -352,753 +352,6 @@ llap.query.negative.files=\
table_nonprintable_negative.q,\
udf_local_resource.q
-# tests are sorted use: perl -pe 's@\\\s*\n@ @g' testconfiguration.properties \
-# | awk -F= '/spark.query.files/{print $2}' | perl -pe 's@.q *, *@\n@g' \
-# | egrep -v '^ *$' | sort -V | uniq | perl -pe 's@\n@.q, \\\n@g' | perl -pe 's@^@ @g'
-spark.query.files=\
- add_part_multiple.q,\
- alter_merge_orc.q,\
- alter_merge_stats_orc.q,\
- annotate_stats_join.q,\
- antijoin.q,\
- auto_join0.q,\
- auto_join1.q,\
- auto_join10.q,\
- auto_join11.q,\
- auto_join12.q,\
- auto_join13.q,\
- auto_join14.q,\
- auto_join15.q,\
- auto_join16.q,\
- auto_join17.q,\
- auto_join18.q,\
- auto_join18_multi_distinct.q,\
- auto_join19.q,\
- auto_join2.q,\
- auto_join20.q,\
- auto_join21.q,\
- auto_join22.q,\
- auto_join23.q,\
- auto_join24.q,\
- auto_join26.q,\
- auto_join27.q,\
- auto_join28.q,\
- auto_join29.q,\
- auto_join3.q,\
- auto_join30.q,\
- auto_join31.q,\
- auto_join32.q,\
- auto_join4.q,\
- auto_join5.q,\
- auto_join6.q,\
- auto_join7.q,\
- auto_join8.q,\
- auto_join9.q,\
- auto_join_filters.q,\
- auto_join_nulls.q,\
- auto_join_reordering_values.q,\
- auto_join_stats.q,\
- auto_join_stats2.q,\
- auto_join_without_localtask.q,\
- auto_smb_mapjoin_14.q,\
- auto_sortmerge_join_1.q,\
- auto_sortmerge_join_12.q,\
- auto_sortmerge_join_13.q,\
- auto_sortmerge_join_14.q,\
- auto_sortmerge_join_15.q,\
- auto_sortmerge_join_16.q,\
- auto_sortmerge_join_2.q,\
- auto_sortmerge_join_3.q,\
- auto_sortmerge_join_4.q,\
- auto_sortmerge_join_5.q,\
- auto_sortmerge_join_6.q,\
- auto_sortmerge_join_7.q,\
- auto_sortmerge_join_8.q,\
- auto_sortmerge_join_9.q,\
- avro_compression_enabled_native.q,\
- avro_decimal_native.q,\
- avro_joins.q,\
- avro_joins_native.q,\
- bucket2.q,\
- bucket3.q,\
- bucket4.q,\
- bucket7.q,\
- bucket_map_join_1.q,\
- bucket_map_join_2.q,\
- bucket_map_join_spark1.q,\
- bucket_map_join_spark2.q,\
- bucket_map_join_spark3.q,\
- bucket_map_join_spark4.q,\
- bucket_map_join_tez1.q,\
- bucket_map_join_tez2.q,\
- bucketmapjoin1.q,\
- bucketmapjoin10.q,\
- bucketmapjoin11.q,\
- bucketmapjoin12.q,\
- bucketmapjoin13.q,\
- bucketmapjoin2.q,\
- bucketmapjoin3.q,\
- bucketmapjoin4.q,\
- bucketmapjoin5.q,\
- bucketmapjoin7.q,\
- bucketmapjoin8.q,\
- bucketmapjoin9.q,\
- bucketmapjoin_negative.q,\
- bucketmapjoin_negative2.q,\
- bucketmapjoin_negative3.q,\
- bucketsortoptimize_insert_2.q,\
- bucketsortoptimize_insert_4.q,\
- bucketsortoptimize_insert_6.q,\
- bucketsortoptimize_insert_7.q,\
- bucketsortoptimize_insert_8.q,\
- cbo_gby.q,\
- cbo_gby_empty.q,\
- cbo_limit.q,\
- cbo_semijoin.q,\
- cbo_simple_select.q,\
- cbo_stats.q,\
- cbo_subq_in.q,\
- cbo_subq_not_in.q,\
- cbo_udf_udaf.q,\
- cbo_union.q,\
- column_access_stats.q,\
- count.q,\
- create_merge_compressed.q,\
- cross_join.q,\
- cross_product_check_1.q,\
- cross_product_check_2.q,\
- ctas.q,\
- custom_input_output_format.q,\
- date_join1.q,\
- date_udf.q,\
- decimal_1_1.q,\
- decimal_join.q,\
- disable_merge_for_bucketing.q,\
- dynpart_sort_optimization.q,\
- enforce_order.q,\
- escape_clusterby1.q,\
- escape_distributeby1.q,\
- escape_orderby1.q,\
- escape_sortby1.q,\
- explaindenpendencydiffengs.q,\
- filter_join_breaktask.q,\
- filter_join_breaktask2.q,\
- groupby1.q,\
- groupby10.q,\
- groupby11.q,\
- groupby1_map.q,\
- groupby1_map_nomap.q,\
- groupby1_map_skew.q,\
- groupby1_noskew.q,\
- groupby2.q,\
- groupby2_map.q,\
- groupby2_map_multi_distinct.q,\
- groupby2_map_skew.q,\
- groupby2_noskew.q,\
- groupby2_noskew_multi_distinct.q,\
- groupby3.q,\
- groupby3_map.q,\
- groupby3_map_multi_distinct.q,\
- groupby3_map_skew.q,\
- groupby3_noskew.q,\
- groupby3_noskew_multi_distinct.q,\
- groupby4.q,\
- groupby4_map.q,\
- groupby4_map_skew.q,\
- groupby4_noskew.q,\
- groupby5.q,\
- groupby5_map.q,\
- groupby5_map_skew.q,\
- groupby5_noskew.q,\
- groupby6.q,\
- groupby6_map.q,\
- groupby6_map_skew.q,\
- groupby6_noskew.q,\
- groupby7.q,\
- groupby7_map.q,\
- groupby7_map_multi_single_reducer.q,\
- groupby7_map_skew.q,\
- groupby7_noskew.q,\
- groupby7_noskew_multi_single_reducer.q,\
- groupby8.q,\
- groupby8_map.q,\
- groupby8_map_skew.q,\
- groupby8_noskew.q,\
- groupby9.q,\
- groupby_bigdata.q,\
- groupby_complex_types.q,\
- groupby_complex_types_multi_single_reducer.q,\
- groupby_cube1.q,\
- groupby_grouping_id2.q,\
- groupby_map_ppr.q,\
- groupby_map_ppr_multi_distinct.q,\
- groupby_multi_insert_common_distinct.q,\
- groupby_multi_single_reducer.q,\
- groupby_multi_single_reducer2.q,\
- groupby_multi_single_reducer3.q,\
- groupby_position.q,\
- groupby_ppr.q,\
- groupby_ppr_multi_distinct.q,\
- groupby_resolution.q,\
- groupby_rollup1.q,\
- groupby_sort_1_23.q,\
- groupby_sort_skew_1.q,\
- groupby_sort_skew_1_23.q,\
- having.q,\
- identity_project_remove_skip.q,\
- index_auto_self_join.q,\
- innerjoin.q,\
- input12.q,\
- input13.q,\
- input14.q,\
- input17.q,\
- input18.q,\
- input1_limit.q,\
- input_part2.q,\
- insert1.q,\
- insert_into1.q,\
- insert_into2.q,\
- insert_into3.q,\
- join0.q,\
- join1.q,\
- join10.q,\
- join11.q,\
- join12.q,\
- join13.q,\
- join14.q,\
- join15.q,\
- join16.q,\
- join17.q,\
- join18.q,\
- join18_multi_distinct.q,\
- join19.q,\
- join2.q,\
- join20.q,\
- join21.q,\
- join22.q,\
- join23.q,\
- join24.q,\
- join25.q,\
- join26.q,\
- join27.q,\
- join28.q,\
- join29.q,\
- join3.q,\
- join30.q,\
- join31.q,\
- join32.q,\
- join32_lessSize.q,\
- join33.q,\
- join34.q,\
- join35.q,\
- join36.q,\
- join37.q,\
- join38.q,\
- join39.q,\
- join4.q,\
- join40.q,\
- join41.q,\
- join5.q,\
- join6.q,\
- join7.q,\
- join8.q,\
- join9.q,\
- join_1to1.q,\
- join_alt_syntax.q,\
- join_array.q,\
- join_casesensitive.q,\
- join_cond_pushdown_1.q,\
- join_cond_pushdown_2.q,\
- join_cond_pushdown_3.q,\
- join_cond_pushdown_4.q,\
- join_cond_pushdown_unqual1.q,\
- join_cond_pushdown_unqual2.q,\
- join_cond_pushdown_unqual3.q,\
- join_cond_pushdown_unqual4.q,\
- join_empty.q,\
- join_filters_overlap.q,\
- join_hive_626.q,\
- join_literals.q,\
- join_map_ppr.q,\
- join_merge_multi_expressions.q,\
- join_merging.q,\
- join_nullsafe.q,\
- join_rc.q,\
- join_reorder.q,\
- join_reorder2.q,\
- join_reorder3.q,\
- join_reorder4.q,\
- join_star.q,\
- join_thrift.q,\
- join_vc.q,\
- join_view.q,\
- lateral_view_explode2.q,\
- lateral_view_multi_lateralviews.q,\
- leftsemijoin.q,\
- leftsemijoin_mr.q,\
- limit_pushdown.q,\
- limit_pushdown2.q,\
- list_bucket_dml_2.q,\
- load_dyn_part1.q,\
- load_dyn_part10.q,\
- load_dyn_part11.q,\
- load_dyn_part12.q,\
- load_dyn_part13.q,\
- load_dyn_part14.q,\
- load_dyn_part15.q,\
- load_dyn_part2.q,\
- load_dyn_part3.q,\
- load_dyn_part4.q,\
- load_dyn_part5.q,\
- load_dyn_part6.q,\
- load_dyn_part7.q,\
- load_dyn_part8.q,\
- load_dyn_part9.q,\
- louter_join_ppr.q,\
- mapjoin1.q,\
- mapjoin_addjar.q,\
- mapjoin_decimal.q,\
- mapjoin_distinct.q,\
- mapjoin_filter_on_outerjoin.q,\
- mapjoin_mapjoin.q,\
- mapjoin_memcheck.q,\
- mapjoin_subquery.q,\
- mapjoin_subquery2.q,\
- mapjoin_test_outer.q,\
- mapreduce1.q,\
- mapreduce2.q,\
- merge1.q,\
- merge2.q,\
- mergejoins.q,\
- mergejoins_mixed.q,\
- metadata_only_queries.q,\
- metadata_only_queries_with_filters.q,\
- multi_insert.q,\
- multi_insert_gby.q,\
- multi_insert_gby2.q,\
- multi_insert_gby3.q,\
- multi_insert_lateral_view.q,\
- multi_insert_mixed.q,\
- multi_insert_move_tasks_share_dependencies.q,\
- multi_insert_with_join.q,\
- multi_join_union.q,\
- multi_join_union_src.q,\
- multigroupby_singlemr.q,\
- nullgroup.q,\
- nullgroup2.q,\
- nullgroup4.q,\
- nullgroup4_multi_distinct.q,\
- optimize_nullscan.q,\
- order.q,\
- order2.q,\
- outer_join_ppr.q,\
- parallel.q,\
- parallel_join0.q,\
- parallel_join1.q,\
- parquet_join.q,\
- parquet_vectorization_0.q,\
- parquet_vectorization_1.q,\
- parquet_vectorization_10.q,\
- parquet_vectorization_11.q,\
- parquet_vectorization_12.q,\
- parquet_vectorization_13.q,\
- parquet_vectorization_14.q,\
- parquet_vectorization_15.q,\
- parquet_vectorization_16.q,\
- parquet_vectorization_17.q,\
- parquet_vectorization_2.q,\
- parquet_vectorization_3.q,\
- parquet_vectorization_4.q,\
- parquet_vectorization_5.q,\
- parquet_vectorization_6.q,\
- parquet_vectorization_7.q,\
- parquet_vectorization_8.q,\
- parquet_vectorization_9.q,\
- parquet_vectorization_decimal_date.q,\
- parquet_vectorization_div0.q,\
- parquet_vectorization_limit.q,\
- parquet_vectorization_nested_udf.q,\
- parquet_vectorization_not.q,\
- parquet_vectorization_offset_limit.q,\
- parquet_vectorization_part.q,\
- parquet_vectorization_part_project.q,\
- parquet_vectorization_part_varchar.q,\
- parquet_vectorization_pushdown.q,\
- pcr.q,\
- ppd_gby_join.q,\
- ppd_join.q,\
- ppd_join2.q,\
- ppd_join3.q,\
- ppd_join4.q,\
- ppd_join5.q,\
- ppd_join_filter.q,\
- ppd_multi_insert.q,\
- ppd_outer_join1.q,\
- ppd_outer_join2.q,\
- ppd_outer_join3.q,\
- ppd_outer_join4.q,\
- ppd_outer_join5.q,\
- ppd_transform.q,\
- ptf.q,\
- ptf_decimal.q,\
- ptf_general_queries.q,\
- ptf_matchpath.q,\
- ptf_rcfile.q,\
- ptf_register_tblfn.q,\
- ptf_seqfile.q,\
- ptf_streaming.q,\
- qroupby_limit_extrastep.q,\
- rcfile_bigdata.q,\
- reduce_deduplicate_exclude_join.q,\
- router_join_ppr.q,\
- row_deserialize_with_union.q,\
- runtime_skewjoin_mapjoin_spark.q,\
- sample1.q,\
- sample10.q,\
- sample2.q,\
- sample3.q,\
- sample4.q,\
- sample5.q,\
- sample6.q,\
- sample7.q,\
- sample8.q,\
- sample9.q,\
- script_env_var1.q,\
- script_env_var2.q,\
- script_pipe.q,\
- scriptfile1.q,\
- semijoin.q,\
- skewjoin.q,\
- skewjoin_noskew.q,\
- skewjoin_union_remove_1.q,\
- skewjoin_union_remove_2.q,\
- skewjoinopt1.q,\
- skewjoinopt10.q,\
- skewjoinopt11.q,\
- skewjoinopt12.q,\
- skewjoinopt13.q,\
- skewjoinopt14.q,\
- skewjoinopt15.q,\
- skewjoinopt16.q,\
- skewjoinopt17.q,\
- skewjoinopt18.q,\
- skewjoinopt19.q,\
- skewjoinopt2.q,\
- skewjoinopt20.q,\
- skewjoinopt3.q,\
- skewjoinopt4.q,\
- skewjoinopt5.q,\
- skewjoinopt6.q,\
- skewjoinopt7.q,\
- skewjoinopt8.q,\
- skewjoinopt9.q,\
- smb_mapjoin_1.q,\
- smb_mapjoin_10.q,\
- smb_mapjoin_11.q,\
- smb_mapjoin_12.q,\
- smb_mapjoin_13.q,\
- smb_mapjoin_14.q,\
- smb_mapjoin_15.q,\
- smb_mapjoin_16.q,\
- smb_mapjoin_17.q,\
- smb_mapjoin_18.q,\
- smb_mapjoin_19.q,\
- smb_mapjoin_2.q,\
- smb_mapjoin_20.q,\
- smb_mapjoin_21.q,\
- smb_mapjoin_22.q,\
- smb_mapjoin_25.q,\
- smb_mapjoin_3.q,\
- smb_mapjoin_4.q,\
- smb_mapjoin_5.q,\
- smb_mapjoin_6.q,\
- smb_mapjoin_7.q,\
- smb_mapjoin_8.q,\
- smb_mapjoin_9.q,\
- sort.q,\
- stats0.q,\
- stats1.q,\
- stats10.q,\
- stats12.q,\
- stats13.q,\
- stats14.q,\
- stats15.q,\
- stats16.q,\
- stats18.q,\
- stats2.q,\
- stats3.q,\
- stats5.q,\
- stats6.q,\
- stats7.q,\
- stats8.q,\
- stats9.q,\
- stats_noscan_1.q,\
- stats_noscan_2.q,\
- stats_only_null.q,\
- stats_partscan_1_23.q,\
- statsfs.q,\
- subquery_exists.q,\
- subquery_in.q,\
- subquery_multi.q,\
- subquery_multiinsert.q,\
- subquery_nested_subquery.q,\
- subquery_notin.q,\
- subquery_null_agg.q,\
- subquery_scalar.q,\
- subquery_select.q,\
- subquery_shared_alias.q,\
- subquery_views.q,\
- table_access_keys_stats.q,\
- temp_table.q,\
- temp_table_gb1.q,\
- temp_table_join1.q,\
- tez_join_tests.q,\
- tez_joins_explain.q,\
- timestamp_1.q,\
- timestamp_2.q,\
- timestamp_3.q,\
- timestamp_comparison.q,\
- timestamp_lazy.q,\
- timestamp_null.q,\
- timestamp_udf.q,\
- transform1.q,\
- transform2.q,\
- transform_ppr1.q,\
- transform_ppr2.q,\
- udaf_collect_set.q,\
- udf_example_add.q,\
- udf_in_file.q,\
- udf_max.q,\
- udf_min.q,\
- udf_percentile.q,\
- union.q,\
- union10.q,\
- union11.q,\
- union12.q,\
- union13.q,\
- union14.q,\
- union15.q,\
- union16.q,\
- union17.q,\
- union18.q,\
- union19.q,\
- union2.q,\
- union20.q,\
- union21.q,\
- union22.q,\
- union23.q,\
- union24.q,\
- union25.q,\
- union26.q,\
- union27.q,\
- union28.q,\
- union29.q,\
- union3.q,\
- union30.q,\
- union31.q,\
- union32.q,\
- union33.q,\
- union34.q,\
- union4.q,\
- union5.q,\
- union6.q,\
- union7.q,\
- union8.q,\
- union9.q,\
- union_date.q,\
- union_date_trim.q,\
- union_lateralview.q,\
- union_null.q,\
- union_ppr.q,\
- union_remove_1.q,\
- union_remove_10.q,\
- union_remove_11.q,\
- union_remove_12.q,\
- union_remove_13.q,\
- union_remove_14.q,\
- union_remove_15.q,\
- union_remove_16.q,\
- union_remove_17.q,\
- union_remove_18.q,\
- union_remove_19.q,\
- union_remove_2.q,\
- union_remove_20.q,\
- union_remove_21.q,\
- union_remove_22.q,\
- union_remove_23.q,\
- union_remove_24.q,\
- union_remove_25.q,\
- union_remove_3.q,\
- union_remove_4.q,\
- union_remove_5.q,\
- union_remove_6.q,\
- union_remove_6_subq.q,\
- union_remove_7.q,\
- union_remove_8.q,\
- union_remove_9.q,\
- union_script.q,\
- union_top_level.q,\
- union_view.q,\
- uniquejoin.q,\
- varchar_join1.q,\
- vector_between_in.q,\
- vector_cast_constant.q,\
- vector_char_4.q,\
- vector_count_distinct.q,\
- vector_data_types.q,\
- vector_decimal_aggregate.q,\
- vector_decimal_mapjoin.q,\
- vector_distinct_2.q,\
- vector_elt.q,\
- vector_groupby_3.q,\
- vector_left_outer_join.q,\
- vector_mapjoin_reduce.q,\
- vector_orderby_5.q,\
- vector_string_concat.q,\
- vector_varchar_4.q,\
- vectorization_0.q,\
- vectorization_1.q,\
- vectorization_10.q,\
- vectorization_11.q,\
- vectorization_12.q,\
- vectorization_13.q,\
- vectorization_14.q,\
- vectorization_15.q,\
- vectorization_16.q,\
- vectorization_17.q,\
- vectorization_2.q,\
- vectorization_3.q,\
- vectorization_4.q,\
- vectorization_5.q,\
- vectorization_6.q,\
- vectorization_9.q,\
- vectorization_decimal_date.q,\
- vectorization_div0.q,\
- vectorization_input_format_excludes.q,\
- vectorization_nested_udf.q,\
- vectorization_not.q,\
- vectorization_parquet_projection.q,\
- vectorization_part.q,\
- vectorization_part_project.q,\
- vectorization_pushdown.q,\
- vectorization_short_regress.q,\
- vectorized_case.q,\
- vectorized_mapjoin.q,\
- vectorized_math_funcs.q,\
- vectorized_nested_mapjoin.q,\
- vectorized_ptf.q,\
- vectorized_rcfile_columnar.q,\
- vectorized_shufflejoin.q,\
- vectorized_string_funcs.q,\
- vectorized_timestamp_funcs.q,\
- windowing.q
-
-# Unlike "spark.query.files" above, these tests only run
-# under Spark engine and only use TestSparkCliDriver.
-spark.only.query.files=\
- spark_combine_equivalent_work_2.q,\
- spark_union_merge.q
-
-# Unlike "miniSparkOnYarn.query.files" below, these tests only run
-# under Spark engine and only use TestMiniSparkOnYarnCliDriver.
-miniSparkOnYarn.only.query.files=\
- dynamic_rdd_cache.q,\
- spark_combine_equivalent_work.q,\
- spark_constprog_dpp.q,\
- spark_dynamic_partition_pruning.q,\
- spark_dynamic_partition_pruning_2.q,\
- spark_dynamic_partition_pruning_3.q,\
- spark_dynamic_partition_pruning_4.q,\
- spark_dynamic_partition_pruning_5.q,\
- spark_dynamic_partition_pruning_6.q,\
- spark_dynamic_partition_pruning_7.q,\
- spark_dynamic_partition_pruning_mapjoin_only.q,\
- spark_dynamic_partition_pruning_recursive_mapjoin.q,\
- spark_explain_groupbyshuffle.q,\
- spark_explainuser_1.q,\
- spark_in_process_launcher.q,\
- spark_multi_insert_parallel_orderby.q,\
- spark_opt_shuffle_serde.q,\
- spark_use_op_stats.q,\
- spark_use_ts_stats_for_mapjoin.q,\
- spark_vectorized_dynamic_partition_pruning.q
-
-miniSparkOnYarn.query.files=\
- auto_sortmerge_join_16.q,\
- bucket4.q,\
- bucket5.q,\
- bucket6.q,\
- bucketmapjoin6.q,\
- bucketmapjoin7.q,\
- constprog_partitioner.q,\
- constprog_semijoin.q,\
- disable_merge_for_bucketing.q,\
- empty_dir_in_table.q,\
- external_table_with_space_in_location_path.q,\
- file_with_header_footer.q,\
- gen_udf_example_add10.q,\
- import_exported_table.q,\
- index_bitmap3.q,\
- index_bitmap_auto.q,\
- infer_bucket_sort_bucketed_table.q,\
- infer_bucket_sort_map_operators.q,\
- infer_bucket_sort_merge.q,\
- infer_bucket_sort_num_buckets.q,\
- infer_bucket_sort_reducers_power_two.q,\
- input16_cc.q,\
- insert_overwrite_directory2.q,\
- leftsemijoin_mr.q,\
- list_bucket_dml_10.q,\
- load_fs2.q,\
- load_hdfs_file_with_space_in_the_name.q,\
- orc_merge1.q,\
- orc_merge2.q,\
- orc_merge3.q,\
- orc_merge4.q,\
- orc_merge5.q,\
- orc_merge6.q,\
- orc_merge7.q,\
- orc_merge8.q,\
- orc_merge9.q,\
- orc_merge_diff_fs.q,\
- orc_merge_incompat1.q,\
- orc_merge_incompat2.q,\
- parallel_orderby.q,\
- quotedid_smb.q,\
- reduce_deduplicate.q,\
- remote_script.q,\
- root_dir_external_table.q,\
- schemeAuthority.q,\
- schemeAuthority2.q,\
- scriptfile1.q,\
- temp_table_external.q,\
- truncate_column_buckets.q,\
- uber_reduce.q,\
- vector_inner_join.q,\
- vector_outer_join0.q,\
- vector_outer_join1.q,\
- vector_outer_join2.q,\
- vector_outer_join3.q,\
- vector_outer_join4.q,\
- vector_outer_join5.q
-
-localSpark.only.query.files=\
- spark_local_queries.q
-
-spark.query.negative.files=\
- groupby2_map_skew_multi_distinct.q,\
- groupby2_multi_distinct.q,\
- groupby3_map_skew_multi_distinct.q,\
- groupby3_multi_distinct.q,\
- groupby_grouping_sets7.q
-
-spark.only.query.negative.files=\
- spark_job_max_tasks.q,\
- spark_stage_max_tasks.q,\
- spark_submit_negative_executor_cores.q,\
- spark_submit_negative_executor_memory.q,\
- spark_task_failure.q
-
tez.perf.disabled.query.files=\
cbo_query_grouping_sets.q,\
mv_query30.q,\
@@ -1107,115 +360,6 @@ tez.perf.disabled.query.files=\
mv_query67.q,\
mv_query68.q
-spark.perf.disabled.query.files=\
- cbo_ext_query1.q,\
- cbo_query1.q,\
- cbo_query10.q,\
- cbo_query11.q,\
- cbo_query12.q,\
- cbo_query13.q,\
- cbo_query14.q,\
- cbo_query15.q,\
- cbo_query16.q,\
- cbo_query17.q,\
- cbo_query18.q,\
- cbo_query19.q,\
- cbo_query2.q,\
- cbo_query20.q,\
- cbo_query21.q,\
- cbo_query22.q,\
- cbo_query23.q,\
- cbo_query24.q,\
- cbo_query25.q,\
- cbo_query26.q,\
- cbo_query27.q,\
- cbo_query28.q,\
- cbo_query29.q,\
- cbo_query3.q,\
- cbo_query30.q,\
- cbo_query31.q,\
- cbo_query32.q,\
- cbo_query33.q,\
- cbo_query34.q,\
- cbo_query35.q,\
- cbo_query36.q,\
- cbo_query37.q,\
- cbo_query38.q,\
- cbo_query39.q,\
- cbo_query4.q,\
- cbo_query40.q,\
- cbo_query42.q,\
- cbo_query43.q,\
- cbo_query44.q,\
- cbo_query45.q,\
- cbo_query46.q,\
- cbo_query47.q,\
- cbo_query48.q,\
- cbo_query49.q,\
- cbo_query5.q,\
- cbo_query50.q,\
- cbo_query51.q,\
- cbo_query52.q,\
- cbo_query53.q,\
- cbo_query54.q,\
- cbo_query55.q,\
- cbo_query56.q,\
- cbo_query57.q,\
- cbo_query58.q,\
- cbo_query59.q,\
- cbo_query6.q,\
- cbo_query60.q,\
- cbo_query61.q,\
- cbo_query63.q,\
- cbo_query64.q,\
- cbo_query65.q,\
- cbo_query66.q,\
- cbo_query67.q,\
- cbo_query68.q,\
- cbo_query69.q,\
- cbo_query7.q,\
- cbo_query70.q,\
- cbo_query71.q,\
- cbo_query72.q,\
- cbo_query73.q,\
- cbo_query74.q,\
- cbo_query75.q,\
- cbo_query76.q,\
- cbo_query77.q,\
- cbo_query78.q,\
- cbo_query79.q,\
- cbo_query8.q,\
- cbo_query80.q,\
- cbo_query81.q,\
- cbo_query82.q,\
- cbo_query83.q,\
- cbo_query84.q,\
- cbo_query85.q,\
- cbo_query86.q,\
- cbo_query87.q,\
- cbo_query88.q,\
- cbo_query89.q,\
- cbo_query9.q,\
- cbo_query90.q,\
- cbo_query91.q,\
- cbo_query92.q,\
- cbo_query93.q,\
- cbo_query94.q,\
- cbo_query95.q,\
- cbo_query96.q,\
- cbo_query97.q,\
- cbo_query98.q,\
- cbo_query99.q,\
- cbo_query_grouping_sets.q,\
- mv_query30.q,\
- mv_query44.q,\
- mv_query45.q,\
- mv_query67.q,\
- mv_query68.q,\
- query14.q,\
- query1b.q,\
- query64.q
-
druid.query.files=\
druid_materialized_view_rewrite_ssb.q,\
druid_timeseries.q,\
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index b69465cd9ab..340899873e0 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -222,9 +222,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "hive.kafka.query.files");
excludesFrom(testConfigProps, "erasurecoding.only.query.files");
excludesFrom(testConfigProps, "beeline.positive.include");
- excludesFrom(testConfigProps, "spark.only.query.files");
- excludesFrom(testConfigProps, "localSpark.only.query.files");
- excludesFrom(testConfigProps, "miniSparkOnYarn.only.query.files");
setResultsDir("ql/src/test/results/clientpositive/llap");
setLogDir("itests/qtest/target/qfile-results/clientpositive");
@@ -308,26 +305,6 @@ public class CliConfigs {
}
}
- public static class SparkTPCDS30TBCliConfig extends AbstractCliConfig {
- public SparkTPCDS30TBCliConfig() {
- super(CorePerfCliDriver.class);
- try {
- setQueryDir("ql/src/test/queries/clientpositive/perf");
- setLogDir("itests/qtest/target/qfile-results/clientpositive/perf/tpcds30tb/spark");
- setResultsDir("ql/src/test/results/clientpositive/perf/tpcds30tb/spark");
- setHiveConfDir("data/conf/perf/tpcds30tb/spark");
- setClusterType(MiniClusterType.SPARK);
- setMetastoreType("postgres.tpcds");
- // The metastore in this configuration can be used only for reading statistics.
- // We cannot exploit the information for running queries so it is impossible to
- // create views or perform other similar operations.
- excludesFrom(testConfigProps, "spark.perf.disabled.query.files");
- } catch (Exception e) {
- throw new RuntimeException("can't construct cliconfig", e);
- }
- }
- }
-
public static class NegativeLlapLocalCliConfig extends AbstractCliConfig {
public NegativeLlapLocalCliConfig() {
super(CoreNegativeCliDriver.class);
@@ -335,7 +312,6 @@ public class CliConfigs {
setQueryDir("ql/src/test/queries/clientnegative");
excludesFrom(testConfigProps, "llap.query.negative.files");
- excludesFrom(testConfigProps, "spark.only.query.negative.files");
setResultsDir("ql/src/test/results/clientnegative");
setLogDir("itests/qtest/target/qfile-results/clientnegative");
@@ -476,97 +452,6 @@ public class CliConfigs {
}
}
- public static class SparkCliConfig extends AbstractCliConfig {
- public SparkCliConfig() {
- super(CoreCliDriver.class);
- try {
- setQueryDir("ql/src/test/queries/clientpositive");
-
- includesFrom(testConfigProps, "spark.query.files");
- includesFrom(testConfigProps, "spark.only.query.files");
-
- setResultsDir("ql/src/test/results/clientpositive/spark");
- setLogDir("itests/qtest-spark/target/qfile-results/clientpositive/spark");
-
- setInitScript("q_test_init.sql");
- setCleanupScript("q_test_cleanup.sql");
-
- setHiveConfDir("data/conf/spark/standalone");
- setClusterType(MiniClusterType.SPARK);
- } catch (Exception e) {
- throw new RuntimeException("can't construct cliconfig", e);
- }
- }
- }
-
- public static class LocalSparkCliConfig extends AbstractCliConfig {
- public LocalSparkCliConfig() {
- super(CoreCliDriver.class);
- try {
- setQueryDir("ql/src/test/queries/clientpositive");
-
- includesFrom(testConfigProps, "localSpark.only.query.files");
-
- setResultsDir("ql/src/test/results/clientpositive/spark");
- setLogDir("itests/qtest-spark/target/qfile-results/clientpositive/spark");
-
- setInitScript("q_test_init.sql");
- setCleanupScript("q_test_cleanup.sql");
-
- setHiveConfDir("data/conf/spark/local");
- setClusterType(MiniClusterType.SPARK);
- } catch (Exception e) {
- throw new RuntimeException("can't construct cliconfig", e);
- }
- }
- }
-
- public static class SparkOnYarnCliConfig extends AbstractCliConfig {
- public SparkOnYarnCliConfig() {
- super(CoreCliDriver.class);
- try {
- setQueryDir("ql/src/test/queries/clientpositive");
-
- includesFrom(testConfigProps, "miniSparkOnYarn.query.files");
- includesFrom(testConfigProps, "miniSparkOnYarn.only.query.files");
-
- setResultsDir("ql/src/test/results/clientpositive/spark");
- setLogDir("itests/qtest-spark/target/qfile-results/clientpositive/spark");
-
- setInitScript("q_test_init.sql");
- setCleanupScript("q_test_cleanup.sql");
-
- setHiveConfDir("data/conf/spark/yarn-cluster");
- setClusterType(MiniClusterType.MINI_SPARK_ON_YARN);
- } catch (Exception e) {
- throw new RuntimeException("can't construct cliconfig", e);
- }
- }
- }
-
- public static class SparkNegativeCliConfig extends AbstractCliConfig {
- public SparkNegativeCliConfig() {
- super(CoreNegativeCliDriver.class);
- try {
- setQueryDir("ql/src/test/queries/clientnegative");
-
- includesFrom(testConfigProps, "spark.query.negative.files");
- includesFrom(testConfigProps, "spark.only.query.negative.files");
-
- setResultsDir("ql/src/test/results/clientnegative/spark");
- setLogDir("itests/qtest-spark/target/qfile-results/clientnegative/spark");
-
- setInitScript("q_test_init.sql");
- setCleanupScript("q_test_cleanup.sql");
-
- setHiveConfDir("data/conf/spark/standalone");
- setClusterType(MiniClusterType.SPARK);
- } catch (Exception e) {
- throw new RuntimeException("can't construct cliconfig", e);
- }
- }
- }
-
public static class BlobstoreCliConfig extends AbstractCliConfig {
public BlobstoreCliConfig() {
super(CoreBlobstoreCliDriver.class);
@@ -640,12 +525,6 @@ public class CliConfigs {
case TEZ:
setHiveConfDir("data/conf/tez");
break;
- case SPARK:
- setHiveConfDir("data/conf/spark/standalone");
- break;
- case MINI_SPARK_ON_YARN:
- setHiveConfDir("data/conf/spark/yarn-cluster");
- break;
default:
setHiveConfDir("data/conf");
break;
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
index cdf599ba369..5d199ce0239 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java
@@ -336,8 +336,6 @@ public class QOutProcessor {
ppm.add(new PatternReplacementPair(Pattern.compile("attempt_[0-9_]+"), "attempt_#ID#"));
ppm.add(new PatternReplacementPair(Pattern.compile("vertex_[0-9_]+"), "vertex_#ID#"));
ppm.add(new PatternReplacementPair(Pattern.compile("task_[0-9_]+"), "task_#ID#"));
- ppm.add(new PatternReplacementPair(Pattern.compile("for Spark session.*?:"),
- "#SPARK_SESSION_ID#:"));
ppm.add(new PatternReplacementPair(Pattern.compile("rowcount = [0-9]+(\\.[0-9]+(E[0-9]+)?)?, cumulative cost = \\{.*\\}, id = [0-9]*"),
"rowcount = ###Masked###, cumulative cost = ###Masked###, id = ###Masked###"));
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java
index 46e2f64a83c..38530f047e3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestMiniClusters.java
@@ -50,8 +50,6 @@ import org.apache.hadoop.hive.llap.LlapItUtils;
import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
-import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
@@ -72,7 +70,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
/**
- * QTestMiniClusters: decouples cluster details from QTestUtil (kafka/druid/spark/llap/tez/mr, file
+ * QTestMiniClusters: decouples cluster details from QTestUtil (kafka/druid/llap/tez/mr, file
* system)
*/
public class QTestMiniClusters {
@@ -91,7 +89,6 @@ public class QTestMiniClusters {
private MiniClusterType clusterType;
private HadoopShims shims;
- private SparkSession sparkSession = null;
private FileSystem fs;
private HadoopShims.MiniMrShim mr = null;
private HadoopShims.MiniDFSShim dfs = null;
@@ -101,7 +98,7 @@ public class QTestMiniClusters {
private SingleNodeKafkaCluster kafkaCluster = null;
public enum CoreClusterType {
- MR, TEZ, SPARK
+ MR, TEZ
}
public enum FsType {
@@ -112,8 +109,6 @@ public class QTestMiniClusters {
MR(CoreClusterType.MR, FsType.HDFS),
TEZ(CoreClusterType.TEZ, FsType.HDFS),
TEZ_LOCAL(CoreClusterType.TEZ, FsType.LOCAL),
- SPARK(CoreClusterType.SPARK, FsType.LOCAL),
- MINI_SPARK_ON_YARN(CoreClusterType.SPARK, FsType.HDFS),
LLAP(CoreClusterType.TEZ, FsType.HDFS),
LLAP_LOCAL(CoreClusterType.TEZ, FsType.LOCAL),
NONE(CoreClusterType.MR,FsType.LOCAL),
@@ -147,10 +142,6 @@ public class QTestMiniClusters {
return TEZ;
} else if (type.equals("tez_local")) {
return TEZ_LOCAL;
- } else if (type.equals("spark")) {
- return SPARK;
- } else if (type.equals("miniSparkOnYarn")) {
- return MINI_SPARK_ON_YARN;
} else if (type.equals("llap")) {
return LLAP;
} else if (type.equals("llap_local")) {
@@ -304,8 +295,6 @@ public class QTestMiniClusters {
MiniClusterType.DRUID_KAFKA, MiniClusterType.DRUID, MiniClusterType.KAFKA)
.contains(clusterType));
}
- } else if (clusterType == MiniClusterType.MINI_SPARK_ON_YARN) {
- mr = shims.getMiniSparkCluster(conf, 2, uriString, 1);
} else if (clusterType == MiniClusterType.MR) {
mr = shims.getMiniMrCluster(conf, 2, uriString, 1);
}
@@ -374,13 +363,6 @@ public class QTestMiniClusters {
ss.setTezSession(tezSessionState);
oldSs.close();
}
-
- if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) {
- sparkSession = oldSs.getSparkSession();
- ss.setSparkSession(sparkSession);
- oldSs.setSparkSession(null);
- oldSs.close();
- }
}
public void shutDown() throws Exception {
@@ -399,15 +381,6 @@ public class QTestMiniClusters {
kafkaCluster = null;
}
setup.tearDown();
- if (sparkSession != null) {
- try {
- SparkSessionManagerImpl.getInstance().closeSession(sparkSession);
- } catch (Exception ex) {
- LOG.error("Error closing spark session.", ex);
- } finally {
- sparkSession = null;
- }
- }
if (mr != null) {
mr.shutdown();
mr = null;
@@ -419,14 +392,6 @@ public class QTestMiniClusters {
}
}
- public void setSparkSession(SparkSession sparkSession) {
- this.sparkSession = sparkSession;
- }
-
- public SparkSession getSparkSession() {
- return sparkSession;
- }
-
public HadoopShims.HdfsEncryptionShim getHdfsEncryptionShim() {
return hes;
}
diff --git a/kryo-registrator/pom.xml b/kryo-registrator/pom.xml
deleted file mode 100644
index 1e9aab3ac64..00000000000
--- a/kryo-registrator/pom.xml
+++ /dev/null
@@ -1,79 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>hive</artifactId>
- <groupId>org.apache.hive</groupId>
- <version>4.0.0-alpha-2-SNAPSHOT</version>
- </parent>
- <artifactId>hive-kryo-registrator</artifactId>
- <packaging>jar</packaging>
- <name>Hive Kryo Registrator</name>
- <properties>
- <hive.path.to.root>..</hive.path.to.root>
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${project.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils-core</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo-shaded</artifactId>
- <version>${kryo4.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs-client</artifactId>
- <version>${hadoop.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
-</project>
diff --git a/kryo-registrator/src/main/java/org/apache/hive/spark/HiveKryoRegistrator.java b/kryo-registrator/src/main/java/org/apache/hive/spark/HiveKryoRegistrator.java
deleted file mode 100644
index 54307ca1598..00000000000
--- a/kryo-registrator/src/main/java/org/apache/hive/spark/HiveKryoRegistrator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.spark;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.Serializer;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.spark.serializer.KryoRegistrator;
-
-/**
- * Kryo registrator for shuffle data, i.e. HiveKey and BytesWritable.
- *
- * Active use (e.g. reflection to get a class instance) of this class on hive side can cause
- * problems because kryo is relocated in hive-exec.
- */
-public class HiveKryoRegistrator implements KryoRegistrator {
- @Override
- public void registerClasses(Kryo kryo) {
- kryo.register(HiveKey.class, new HiveKeySerializer());
- kryo.register(BytesWritable.class, new BytesWritableSerializer());
- }
-
- private static class HiveKeySerializer extends Serializer<HiveKey> {
-
- public void write(Kryo kryo, Output output, HiveKey object) {
- output.writeVarInt(object.getLength(), true);
- output.write(object.getBytes(), 0, object.getLength());
- output.writeVarInt(object.hashCode(), false);
- }
-
- public HiveKey read(Kryo kryo, Input input, Class<? extends HiveKey> type) {
- int len = input.readVarInt(true);
- byte[] bytes = new byte[len];
- input.readBytes(bytes);
- return new HiveKey(bytes, input.readVarInt(false));
- }
- }
-
- static class BytesWritableSerializer extends Serializer<BytesWritable> {
-
- public void write(Kryo kryo, Output output, BytesWritable object) {
- output.writeVarInt(object.getLength(), true);
- output.write(object.getBytes(), 0, object.getLength());
- }
-
- public BytesWritable read(Kryo kryo, Input input, Class<? extends BytesWritable> type) {
- int len = input.readVarInt(true);
- byte[] bytes = new byte[len];
- input.readBytes(bytes);
- return new BytesWritable(bytes);
- }
-
- }
-}
diff --git a/kryo-registrator/src/main/java/org/apache/hive/spark/NoHashCodeKryoSerializer.java b/kryo-registrator/src/main/java/org/apache/hive/spark/NoHashCodeKryoSerializer.java
deleted file mode 100644
index a994a5bc83a..00000000000
--- a/kryo-registrator/src/main/java/org/apache/hive/spark/NoHashCodeKryoSerializer.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hive.spark;
-
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.Serializer;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.spark.SparkConf;
-import org.apache.spark.serializer.KryoSerializer;
-
-
-/**
- * A {@link KryoSerializer} that does not serialize hash codes while serializing a
- * {@link HiveKey}. This decreases the amount of data to be shuffled during a Spark shuffle.
- */
-public class NoHashCodeKryoSerializer extends KryoSerializer {
-
- private static final long serialVersionUID = 3350910170041648022L;
-
- public NoHashCodeKryoSerializer(SparkConf conf) {
- super(conf);
- }
-
- @Override
- public Kryo newKryo() {
- Kryo kryo = super.newKryo();
- kryo.register(HiveKey.class, new HiveKeySerializer());
- kryo.register(BytesWritable.class, new HiveKryoRegistrator.BytesWritableSerializer());
- return kryo;
- }
-
- private static class HiveKeySerializer extends Serializer<HiveKey> {
-
- public void write(Kryo kryo, Output output, HiveKey object) {
- output.writeVarInt(object.getLength(), true);
- output.write(object.getBytes(), 0, object.getLength());
- }
-
- public HiveKey read(Kryo kryo, Input input, Class<? extends HiveKey> type) {
- int len = input.readVarInt(true);
- byte[] bytes = new byte[len];
- input.readBytes(bytes);
- return new HiveKey(bytes);
- }
- }
-}
diff --git a/packaging/pom.xml b/packaging/pom.xml
index ddc24ec1b77..61987be809e 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -311,11 +311,6 @@
<artifactId>hive-webhcat-java-client</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-kryo-registrator</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml
index 9acb6b5582e..6cd94585cbf 100644
--- a/packaging/src/main/assembly/src.xml
+++ b/packaging/src/main/assembly/src.xml
@@ -102,7 +102,6 @@
<include>service-rpc/**/*</include>
<include>service/**/*</include>
<include>shims/**/*</include>
- <include>spark-client/**/*</include>
<include>storage-api/**/*</include>
<include>standalone-metastore/metastore-common/**/*</include>
<include>standalone-metastore/metastore-server/**/*</include>
@@ -113,7 +112,6 @@
<include>testutils/**/*</include>
<include>upgrade-acid/**/*</include>
<include>vector-code-gen/**/*</include>
- <include>kryo-registrator/**/*</include>
<include>kudu-handler/**/*</include>
<include>parser/**/*</include>
<include>udf/**/*</include>
diff --git a/pom.xml b/pom.xml
index e4000c44d05..9af2b33c1c7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,8 +54,6 @@
<module>llap-tez</module>
<module>llap-server</module>
<module>shims</module>
- <module>spark-client</module>
- <module>kryo-registrator</module>
<module>kudu-handler</module>
<module>testutils</module>
<module>packaging</module>
@@ -89,7 +87,6 @@
<test.warehouse.scheme>pfile://</test.warehouse.scheme>
<!-- To add additional exclude patterns set this property -->
<test.excludes.additional/>
- <skip.spark.files>**/TestSparkStatistics*,**/TestSparkSessionTimeout*,**/TestJdbcWithMiniHS2ErasureCoding*,**/TestLocalHiveSparkClient*</skip.spark.files>
<!-- Plugin and Plugin Dependency Versions -->
<ant.contrib.version>1.0b3</ant.contrib.version>
<maven.test.jvm.args>-Xmx2048m</maven.test.jvm.args>
@@ -154,8 +151,6 @@
<jettison.version>1.1</jettison.version>
<jetty.version>9.3.27.v20190418</jetty.version>
<jersey.version>1.19</jersey.version>
- <!-- Glassfish jersey is included for Spark client test only -->
- <glassfish.jersey.version>2.22.2</glassfish.jersey.version>
<jline.version>2.14.6</jline.version>
<jms.version>2.0.2</jms.version>
<joda.version>2.9.9</joda.version>
@@ -166,8 +161,6 @@
<junit.vintage.version>5.6.2</junit.vintage.version>
<kafka.version>2.5.0</kafka.version>
<kryo.version>5.0.3</kryo.version>
- <kryo4.version>4.0.2</kryo4.version>
- <!-- old kryo, for spark-client, might be removed in HIVE-23885 -->
<reflectasm.version>1.11.9</reflectasm.version>
<kudu.version>1.12.0</kudu.version>
<!-- Leaving libfb303 at 0.9.3 regardless of libthrift: As per THRIFT-4613 The Apache Thrift project does not publish items related to fb303 at this point -->
@@ -199,9 +192,6 @@
<storage-api.version>4.0.0-alpha-2-SNAPSHOT</storage-api.version>
<tez.version>0.10.1</tez.version>
<super-csv.version>2.2.0</super-csv.version>
- <spark.version>2.4.5</spark.version>
- <scala.binary.version>2.12</scala.binary.version>
- <scala.version>2.12.10</scala.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
<snappy.version>1.1.4</snappy.version>
<wadl-resourcedoc-doclet.version>1.4</wadl-resourcedoc-doclet.version>
@@ -1166,31 +1156,6 @@
<artifactId>hadoop-minicluster</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- <version>${scala.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo-shaded</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.jamon</groupId>
<artifactId>jamon-runtime</artifactId>
@@ -1547,8 +1512,6 @@
<reason>Do not use shaded imports</reason>
<bannedImports>
<bannedImport>**.shaded.**</bannedImport>
- <bannedImport>org.spark_project.guava.**</bannedImport>
- <bannedImport>org.apache.spark.api.java.Optional</bannedImport>
<bannedImport>jersey.repackaged.com.google.**</bannedImport>
<bannedImport>org.codehaus.jackson.**</bannedImport>
</bannedImports>
@@ -1585,7 +1548,6 @@
<exclude>**/ql/exec/vector/udf/generic/*.java</exclude>
<exclude>**/TestHiveServer2Concurrency.java</exclude>
<exclude>${test.excludes.additional}</exclude>
- <exclude>${skip.spark.files}</exclude>
</excludes>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<reuseForks>false</reuseForks>
@@ -1601,10 +1563,6 @@
<LANG>en_US.UTF-8</LANG>
<HADOOP_CLASSPATH>${test.conf.dir}:${basedir}/${hive.path.to.root}/conf</HADOOP_CLASSPATH>
<HIVE_HADOOP_TEST_CLASSPATH>${test.hive.hadoop.classpath}</HIVE_HADOOP_TEST_CLASSPATH>
- <SPARK_SUBMIT_CLASSPATH>${spark.home}/lib/spark-assembly-${spark.version}-hadoop2.4.0.jar:${test.hive.hadoop.classpath}</SPARK_SUBMIT_CLASSPATH>
- <SPARK_OSX_TEST_OPTS>-Dorg.xerial.snappy.tempdir=/tmp -Dorg.xerial.snappy.lib.name=libsnappyjava.jnilib</SPARK_OSX_TEST_OPTS>
- <SPARK_SCALA_VERSION>2.11</SPARK_SCALA_VERSION>
- <SPARK_HOME>${spark.home}</SPARK_HOME>
<PATH>${env.PATH}${test.extra.path}</PATH>
</environmentVariables>
<systemPropertyVariables>
@@ -1627,7 +1585,6 @@
<log4j.debug>true</log4j.debug>
<!-- don't diry up /tmp -->
<java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>
- <spark.home>${spark.home}</spark.home>
<!-- Hadoop's minidfs class uses this -->
<test.build.data>${test.tmp.dir}</test.build.data>
<!-- required by QTestUtil -->
@@ -1643,8 +1600,6 @@
<!-- EnforceReadOnlyTables hook and QTestUtil -->
<test.src.tables/>
<java.security.krb5.conf>${test.conf.dir}/krb5.conf</java.security.krb5.conf>
- <!-- Required by spark to work around SPARK-14958 -->
- <antlr.version>${antlr.version}</antlr.version>
<hadoop.version>${hadoop.version}</hadoop.version>
<qfile>${qfile}</qfile>
<initScript>${initScript}</initScript>
@@ -1915,14 +1870,6 @@
<test.log4j.scheme>file:/</test.log4j.scheme>
</properties>
</profile>
- <profile>
- <id>spark-test</id>
- <properties>
- <skip.spark.files>
- **/ql/exec/spark/session/TestSparkSessionManagerImpl.java,**/TestMultiSessionsHS2WithLocalClusterSpark.java,**/TestJdbcWithLocalClusterSpark.java
- </skip.spark.files>
- </properties>
- </profile>
<profile>
<id>itests</id>
<modules>
diff --git a/ql/if/queryplan.thrift b/ql/if/queryplan.thrift
index 80f127cd512..d39d4090fa0 100644
--- a/ql/if/queryplan.thrift
+++ b/ql/if/queryplan.thrift
@@ -60,7 +60,6 @@ enum OperatorType {
ORCFILEMERGE,
RCFILEMERGE,
MERGEJOIN,
- SPARKPRUNINGSINK,
TOPNKEY,
}
diff --git a/ql/pom.xml b/ql/pom.xml
index e98611d1fb4..018f32f77bf 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -143,17 +143,6 @@
<artifactId>hive-shims</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-spark-client</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.esotericsoftware</groupId>
- <artifactId>kryo-shaded</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
@@ -793,59 +782,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>com.esotericsoftware.kryo</groupId>
- <artifactId>kryo</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.java.dev.jets3t</groupId>
- <artifactId>jets3t</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.glassfish.jersey.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-servlet</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.core</groupId>
- <artifactId>jersey-server</artifactId>
- <version>${glassfish.jersey.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.glassfish.jersey.containers</groupId>
- <artifactId>jersey-container-servlet-core</artifactId>
- <version>${glassfish.jersey.version}</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
@@ -1113,7 +1049,6 @@
<include>com.fasterxml.jackson.dataformat:jackson-dataformat-yaml</include>
<include>com.google.guava:guava</include>
<include>net.sf.opencsv:opencsv</include>
- <include>org.apache.hive:hive-spark-client</include>
<include>org.apache.hive:hive-storage-api</include>
<include>org.threeten:threeten-extra</include>
<include>org.apache.orc:orc-core</include>
diff --git a/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp b/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
index 1d32a8ffa75..b4a637086e8 100644
--- a/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
+++ b/ql/src/gen/thrift/gen-cpp/queryplan_types.cpp
@@ -97,7 +97,6 @@ int _kOperatorTypeValues[] = {
OperatorType::ORCFILEMERGE,
OperatorType::RCFILEMERGE,
OperatorType::MERGEJOIN,
- OperatorType::SPARKPRUNINGSINK,
OperatorType::TOPNKEY
};
const char* _kOperatorTypeNames[] = {
@@ -126,10 +125,9 @@ const char* _kOperatorTypeNames[] = {
"ORCFILEMERGE",
"RCFILEMERGE",
"MERGEJOIN",
- "SPARKPRUNINGSINK",
"TOPNKEY"
};
-const std::map<int, const char*> _OperatorType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(27, _kOperatorTypeValues, _kOperatorTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
+const std::map<int, const char*> _OperatorType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(26, _kOperatorTypeValues, _kOperatorTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const OperatorType::type& val) {
std::map<int, const char*>::const_iterator it = _OperatorType_VALUES_TO_NAMES.find(val);
diff --git a/ql/src/gen/thrift/gen-cpp/queryplan_types.h b/ql/src/gen/thrift/gen-cpp/queryplan_types.h
index 8015bb32e80..fac2cdf7a2a 100644
--- a/ql/src/gen/thrift/gen-cpp/queryplan_types.h
+++ b/ql/src/gen/thrift/gen-cpp/queryplan_types.h
@@ -74,8 +74,7 @@ struct OperatorType {
ORCFILEMERGE = 22,
RCFILEMERGE = 23,
MERGEJOIN = 24,
- SPARKPRUNINGSINK = 25,
- TOPNKEY = 26
+ TOPNKEY = 25
};
};
diff --git a/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java b/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java
index 7d0317d8229..bf38b26b837 100644
--- a/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java
+++ b/ql/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/ql/plan/api/OperatorType.java
@@ -34,8 +34,7 @@ public enum OperatorType implements org.apache.thrift.TEnum {
ORCFILEMERGE(22),
RCFILEMERGE(23),
MERGEJOIN(24),
- SPARKPRUNINGSINK(25),
- TOPNKEY(26);
+ TOPNKEY(25);
private final int value;
@@ -108,8 +107,6 @@ public enum OperatorType implements org.apache.thrift.TEnum {
case 24:
return MERGEJOIN;
case 25:
- return SPARKPRUNINGSINK;
- case 26:
return TOPNKEY;
default:
return null;
diff --git a/ql/src/gen/thrift/gen-php/OperatorType.php b/ql/src/gen/thrift/gen-php/OperatorType.php
index 3f037652fbb..18f8bbc2d18 100644
--- a/ql/src/gen/thrift/gen-php/OperatorType.php
+++ b/ql/src/gen/thrift/gen-php/OperatorType.php
@@ -66,9 +66,7 @@ final class OperatorType
const MERGEJOIN = 24;
- const SPARKPRUNINGSINK = 25;
-
- const TOPNKEY = 26;
+ const TOPNKEY = 25;
static public $__names = array(
0 => 'JOIN',
@@ -96,8 +94,7 @@ final class OperatorType
22 => 'ORCFILEMERGE',
23 => 'RCFILEMERGE',
24 => 'MERGEJOIN',
- 25 => 'SPARKPRUNINGSINK',
- 26 => 'TOPNKEY',
+ 25 => 'TOPNKEY',
);
}
diff --git a/ql/src/gen/thrift/gen-py/queryplan/ttypes.py b/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
index 81933ff3611..88fd2e8b678 100644
--- a/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
+++ b/ql/src/gen/thrift/gen-py/queryplan/ttypes.py
@@ -72,8 +72,7 @@ class OperatorType(object):
ORCFILEMERGE = 22
RCFILEMERGE = 23
MERGEJOIN = 24
- SPARKPRUNINGSINK = 25
- TOPNKEY = 26
+ TOPNKEY = 25
_VALUES_TO_NAMES = {
0: "JOIN",
@@ -101,8 +100,7 @@ class OperatorType(object):
22: "ORCFILEMERGE",
23: "RCFILEMERGE",
24: "MERGEJOIN",
- 25: "SPARKPRUNINGSINK",
- 26: "TOPNKEY",
+ 25: "TOPNKEY",
}
_NAMES_TO_VALUES = {
@@ -131,8 +129,7 @@ class OperatorType(object):
"ORCFILEMERGE": 22,
"RCFILEMERGE": 23,
"MERGEJOIN": 24,
- "SPARKPRUNINGSINK": 25,
- "TOPNKEY": 26,
+ "TOPNKEY": 25,
}
diff --git a/ql/src/gen/thrift/gen-rb/queryplan_types.rb b/ql/src/gen/thrift/gen-rb/queryplan_types.rb
index abc03047476..407681963da 100644
--- a/ql/src/gen/thrift/gen-rb/queryplan_types.rb
+++ b/ql/src/gen/thrift/gen-rb/queryplan_types.rb
@@ -46,10 +46,9 @@ module OperatorType
ORCFILEMERGE = 22
RCFILEMERGE = 23
MERGEJOIN = 24
- SPARKPRUNINGSINK = 25
- TOPNKEY = 26
- VALUE_MAP = {0 => "JOIN", 1 => "MAPJOIN", 2 => "EXTRACT", 3 => "FILTER", 4 => "FORWARD", 5 => "GROUPBY", 6 => "LIMIT", 7 => "SCRIPT", 8 => "SELECT", 9 => "TABLESCAN", 10 => "FILESINK", 11 => "REDUCESINK", 12 => "UNION", 13 => "UDTF", 14 => "LATERALVIEWJOIN", 15 => "LATERALVIEWFORWARD", 16 => "HASHTABLESINK", 17 => "HASHTABLEDUMMY", 18 => "PTF", 19 => "MUX", 20 => "DEMUX", 21 => "EVENT", 22 => "ORCFILEMERGE", 23 => "RCFILEMERGE", 24 => "MERGEJOIN", 25 => "SPARKPRUNINGSINK", 26 => "TOPNKEY"}
- VALID_VALUES = Set.new([JOIN, MAPJOIN, EXTRACT, FILTER, FORWARD, GROUPBY, LIMIT, SCRIPT, SELECT, TABLESCAN, FILESINK, REDUCESINK, UNION, UDTF, LATERALVIEWJOIN, LATERALVIEWFORWARD, HASHTABLESINK, HASHTABLEDUMMY, PTF, MUX, DEMUX, EVENT, ORCFILEMERGE, RCFILEMERGE, MERGEJOIN, SPARKPRUNINGSINK, TOPNKEY]).freeze
+ TOPNKEY = 25
+ VALUE_MAP = {0 => "JOIN", 1 => "MAPJOIN", 2 => "EXTRACT", 3 => "FILTER", 4 => "FORWARD", 5 => "GROUPBY", 6 => "LIMIT", 7 => "SCRIPT", 8 => "SELECT", 9 => "TABLESCAN", 10 => "FILESINK", 11 => "REDUCESINK", 12 => "UNION", 13 => "UDTF", 14 => "LATERALVIEWJOIN", 15 => "LATERALVIEWFORWARD", 16 => "HASHTABLESINK", 17 => "HASHTABLEDUMMY", 18 => "PTF", 19 => "MUX", 20 => "DEMUX", 21 => "EVENT", 22 => "ORCFILEMERGE", 23 => "RCFILEMERGE", 24 => "MERGEJOIN", 25 => "TOPNKEY"}
+ VALID_VALUES = Set.new([JOIN, MAPJOIN, EXTRACT, FILTER, FORWARD, GROUPBY, LIMIT, SCRIPT, SELECT, TABLESCAN, FILESINK, REDUCESINK, UNION, UDTF, LATERALVIEWJOIN, LATERALVIEWFORWARD, HASHTABLESINK, HASHTABLEDUMMY, PTF, MUX, DEMUX, EVENT, ORCFILEMERGE, RCFILEMERGE, MERGEJOIN, TOPNKEY]).freeze
end
module TaskType
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index 91330ddc3d0..f7c7c67a614 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -1050,13 +1050,6 @@ public class Context {
* Today this translates into running hadoop jobs locally
*/
public boolean isLocalOnlyExecutionMode() {
- // Always allow spark to run in a cluster mode. Without this, depending on
- // user's local hadoop settings, true may be returned, which causes plan to be
- // stored in local path.
- if (HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
- return false;
- }
-
return ShimLoader.getHadoopShims().isLocalMode(conf);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 8d1657eae47..5417402a1a4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
import org.apache.hadoop.hive.ql.lock.CompileLock;
import org.apache.hadoop.hive.ql.lock.CompileLockFactory;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
@@ -538,10 +537,6 @@ public class Driver implements IDriver {
String queryId = Strings.isNullOrEmpty(driverContext.getQueryState().getQueryId()) ?
QueryPlan.makeQueryId() : driverContext.getQueryState().getQueryId();
- SparkSession ss = SessionState.get().getSparkSession();
- if (ss != null) {
- ss.onQuerySubmission(queryId);
- }
driverContext.getQueryDisplay().setQueryId(queryId);
setTriggerContext(queryId);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Executor.java b/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
index 14b235d7ad1..1e2140ed802 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Executor.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql;
-import java.net.InetAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -298,19 +297,13 @@ public class Executor {
}
private String getJobName() {
- int maxlen;
- if ("spark".equals(driverContext.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE))) {
- maxlen = driverContext.getConf().getIntVar(HiveConf.ConfVars.HIVESPARKJOBNAMELENGTH);
- } else {
- maxlen = driverContext.getConf().getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
- }
+ int maxlen = driverContext.getConf().getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
return Utilities.abbreviate(driverContext.getQueryString(), maxlen - 6);
}
private int getJobCount() {
int mrJobCount = Utilities.getMRTasks(driverContext.getPlan().getRootTasks()).size();
- int jobCount = mrJobCount + Utilities.getTezTasks(driverContext.getPlan().getRootTasks()).size()
- + Utilities.getSparkTasks(driverContext.getPlan().getRootTasks()).size();
+ int jobCount = mrJobCount + Utilities.getTezTasks(driverContext.getPlan().getRootTasks()).size();
if (jobCount > 0) {
if (mrJobCount > 0 && "mr".equals(HiveConf.getVar(driverContext.getConf(), ConfVars.HIVE_EXECUTION_ENGINE))) {
LOG.warn(HiveConf.generateMrDeprecationWarning());
@@ -545,10 +538,6 @@ public class Executor {
logExecutionResourceUsage();
- if (SessionState.get().getSparkSession() != null) {
- SessionState.get().getSparkSession().onQueryCompletion(driverContext.getQueryId());
- }
-
driverState.executionFinishedWithLocking(executionError);
if (driverState.isAborted()) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java
index 018f8b6a673..5d4e2428bc6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/HashTableLoaderFactory.java
@@ -34,8 +34,6 @@ public class HashTableLoaderFactory {
public static HashTableLoader getLoader(Configuration hconf) {
if (HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
return new org.apache.hadoop.hive.ql.exec.tez.HashTableLoader();
- } else if (HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
- return new org.apache.hadoop.hive.ql.exec.spark.HashTableLoader();
} else {
return new org.apache.hadoop.hive.ql.exec.mr.HashTableLoader();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index 14cffc20e48..0949d0a5b21 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -74,7 +74,6 @@ import org.apache.hadoop.hive.ql.plan.ExplainWork;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
-import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.TezWork;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.ql.security.authorization.AuthorizationFactory;
@@ -751,36 +750,6 @@ public class ExplainTask extends Task<ExplainWork> implements Serializable {
json.accumulate(ent.getKey().toString(), jsonDep);
}
}
- } else if (ent.getValue() != null && !((List<?>) ent.getValue()).isEmpty()
- && ((List<?>) ent.getValue()).get(0) != null &&
- ((List<?>) ent.getValue()).get(0) instanceof SparkWork.Dependency) {
- if (out != null) {
- boolean isFirst = true;
- for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
- if (!isFirst) {
- out.print(", ");
- } else {
- out.print("<- ");
- isFirst = false;
- }
- out.print(dep.getName());
- out.print(" (");
- out.print(dep.getShuffleType());
- out.print(", ");
- out.print(dep.getNumPartitions());
- out.print(")");
- }
- out.println();
- }
- if (jsonOutput) {
- for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
- JSONObject jsonDep = new JSONObject(new LinkedHashMap<>());
- jsonDep.put("parent", dep.getName());
- jsonDep.put("type", dep.getShuffleType());
- jsonDep.put("partitions", dep.getNumPartitions());
- json.accumulate(ent.getKey().toString(), jsonDep);
- }
- }
} else {
if (out != null) {
out.print(ent.getValue().toString());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
index 13cc8fcfbbb..91cf0c3c03e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.Utilities.MissingBucketsContext;
-import org.apache.hadoop.hive.ql.exec.spark.SparkMetricUtils;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
@@ -1446,10 +1445,6 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
row_count.set(numRows);
LOG.info(toString() + ": records written - " + numRows);
- if ("spark".equalsIgnoreCase(HiveConf.getVar(hconf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
- SparkMetricUtils.updateSparkRecordsWrittenMetrics(runTimeNumRows);
- }
-
if (!bDynParts && !filesCreated) {
boolean isTez = "tez".equalsIgnoreCase(
HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE));
@@ -1529,9 +1524,6 @@ public class FileSinkOperator extends TerminalOperator<FileSinkDesc> implements
if (isNativeTable()) {
fsp.commit(fs, commitPaths, deleteDeltas);
}
- if ("spark".equals(HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE))) {
- SparkMetricUtils.updateSparkBytesWrittenMetrics(LOG, fs, fsp.finalPaths);
- }
}
if (conf.isMmTable() || conf.isDirectInsert()) {
boolean isDelete = AcidUtils.Operation.DELETE.equals(conf.getAcidOperation());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java
index 338e495ef82..8de600a7f3f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/GlobalWorkMapFactory.java
@@ -100,8 +100,7 @@ public class GlobalWorkMapFactory {
public Map<Path, BaseWork> get(Configuration conf) {
if (LlapProxy.isDaemon()
- || (SessionState.get() != null && SessionState.get().isHiveServerQuery())
- || HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
+ || (SessionState.get() != null && SessionState.get().isHiveServerQuery())) {
if (threadLocalWorkMap == null) {
threadLocalWorkMap = new ThreadLocal<Map<Path, BaseWork>>() {
@Override
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
index a6f05df2538..1897413de4a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
@@ -52,8 +52,6 @@ import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
import org.apache.hadoop.hive.ql.exec.persistence.MatchTracker;
import org.apache.hadoop.hive.ql.exec.persistence.ObjectContainer;
import org.apache.hadoop.hive.ql.exec.persistence.UnwrapRowContainer;
-import org.apache.hadoop.hive.ql.exec.spark.SmallTableCache;
-import org.apache.hadoop.hive.ql.exec.spark.SparkUtilities;
import org.apache.hadoop.hive.ql.exec.tez.LlapObjectCache;
import org.apache.hadoop.hive.ql.exec.tez.LlapObjectSubCache;
import org.apache.hadoop.hive.ql.io.HiveKey;
@@ -727,21 +725,6 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem
@Override
public void closeOp(boolean abort) throws HiveException {
- // Call the small table cache cache method, this way when a task finishes, we still keep the small table around
- // for at least 30 seconds, which gives any tasks scheduled in the future a chance to re-use the small table.
- if (HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark") &&
- SparkUtilities.isDedicatedCluster(hconf)) {
-
- for (byte pos = 0; pos < mapJoinTables.length; pos++) {
- if (pos != conf.getPosBigTable()) {
- MapJoinTableContainer container = mapJoinTables[pos];
- if (container != null && container.getKey() != null) {
- SmallTableCache.cache(container.getKey(), container);
- }
- }
- }
- }
-
if (isFullOuterMapJoin) {
// FULL OUTER MapJoin: After matching the Big Table row keys against the Small Table, we now
@@ -836,9 +819,7 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem
// in mapreduce case, we need to always clear up as mapreduce doesn't have object registry.
if ((this.getExecContext() != null) && (this.getExecContext().getLocalWork() != null)
- && (this.getExecContext().getLocalWork().getInputFileChangeSensitive())
- && !(HiveConf.getVar(hconf, ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")
- && SparkUtilities.isDedicatedCluster(hconf))) {
+ && (this.getExecContext().getLocalWork().getInputFileChangeSensitive())) {
LOG.info("MR: Clearing all map join table containers.");
clearAllTableContainers();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryExhaustionCheckerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryExhaustionCheckerFactory.java
index 086df68c8d4..aecaaeee3a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryExhaustionCheckerFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MemoryExhaustionCheckerFactory.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.plan.HashTableSinkDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -30,10 +29,6 @@ class MemoryExhaustionCheckerFactory {
static MemoryExhaustionChecker getChecker(SessionState.LogHelper console, Configuration conf,
HashTableSinkDesc hashTableSinkDesc) {
- if ("spark".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE))) {
- return SparkMemoryExhaustionChecker.get(conf);
- } else {
- return new DefaultMemoryExhaustionChecker(console, hashTableSinkDesc);
- }
+ return new DefaultMemoryExhaustionChecker(console, hashTableSinkDesc);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java
index 9a6d160a5ff..12644885467 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ObjectCacheFactory.java
@@ -77,7 +77,7 @@ public class ObjectCacheFactory {
return null;
}
}
- } else { // mr or spark
+ } else { // mr
return new ObjectCacheWrapper(
new org.apache.hadoop.hive.ql.exec.mr.ObjectCache(), queryId);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
index da26e4f9928..7f6a8bfed1d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java
@@ -36,14 +36,10 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorSMBMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorSelectOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorSparkHashTableSinkOperator;
-import org.apache.hadoop.hive.ql.exec.vector.VectorSparkPartitionPruningSinkOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorTopNKeyOperator;
import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFOperator;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.optimizer.spark.SparkPartitionPruningSinkDesc;
-import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator;
import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc;
import org.apache.hadoop.hive.ql.plan.CollectDesc;
import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
@@ -72,7 +68,6 @@ import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
import org.apache.hadoop.hive.ql.plan.ScriptDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
-import org.apache.hadoop.hive.ql.plan.SparkHashTableSinkDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.plan.TopNKeyDesc;
import org.apache.hadoop.hive.ql.plan.UDTFDesc;
@@ -114,13 +109,11 @@ public final class OperatorFactory {
opvec.put(LateralViewForwardDesc.class, LateralViewForwardOperator.class);
opvec.put(HashTableDummyDesc.class, HashTableDummyOperator.class);
opvec.put(HashTableSinkDesc.class, HashTableSinkOperator.class);
- opvec.put(SparkHashTableSinkDesc.class, SparkHashTableSinkOperator.class);
opvec.put(DummyStoreDesc.class, DummyStoreOperator.class);
opvec.put(DemuxDesc.class, DemuxOperator.class);
opvec.put(MuxDesc.class, MuxOperator.class);
opvec.put(AppMasterEventDesc.class, AppMasterEventOperator.class);
opvec.put(DynamicPruningEventDesc.class, AppMasterEventOperator.class);
- opvec.put(SparkPartitionPruningSinkDesc.class, SparkPartitionPruningSinkOperator.class);
opvec.put(RCFileMergeDesc.class, RCFileMergeOperator.class);
opvec.put(OrcFileMergeDesc.class, OrcFileMergeOperator.class);
opvec.put(CommonMergeJoinDesc.class, CommonMergeJoinOperator.class);
@@ -131,8 +124,6 @@ public final class OperatorFactory {
static {
vectorOpvec.put(AppMasterEventDesc.class, VectorAppMasterEventOperator.class);
vectorOpvec.put(DynamicPruningEventDesc.class, VectorAppMasterEventOperator.class);
- vectorOpvec.put(
- SparkPartitionPruningSinkDesc.class, VectorSparkPartitionPruningSinkOperator.class);
vectorOpvec.put(SelectDesc.class, VectorSelectOperator.class);
vectorOpvec.put(GroupByDesc.class, VectorGroupByOperator.class);
vectorOpvec.put(MapJoinDesc.class, VectorMapJoinOperator.class);
@@ -142,7 +133,6 @@ public final class OperatorFactory {
vectorOpvec.put(FilterDesc.class, VectorFilterOperator.class);
vectorOpvec.put(LimitDesc.class, VectorLimitOperator.class);
vectorOpvec.put(PTFDesc.class, VectorPTFOperator.class);
- vectorOpvec.put(SparkHashTableSinkDesc.class, VectorSparkHashTableSinkOperator.class);
vectorOpvec.put(TopNKeyDesc.class, VectorTopNKeyOperator.class);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
index aa9a44c40be..3924164166b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorUtils.java
@@ -33,7 +33,6 @@ import java.util.Stack;
import org.apache.hadoop.hive.ql.exec.NodeUtils.Function;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.SemiJoinBranchInfo;
-import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator;
import org.apache.hadoop.hive.ql.plan.BaseWork;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
@@ -359,40 +358,6 @@ public class OperatorUtils {
}
}
- /**
- * Given the input operator 'op', walk up the operator tree from 'op', and collect all the
- * roots that can be reached from it. The results are stored in 'roots'.
- */
- public static void findRoots(Operator<?> op, Collection<Operator<?>> roots) {
- List<Operator<?>> parents = op.getParentOperators();
- if (parents == null || parents.isEmpty()) {
- roots.add(op);
- return;
- }
- for (Operator<?> p : parents) {
- findRoots(p, roots);
- }
- }
-
- /**
- * Remove the branch that contains the specified operator. Do nothing if there's no branching,
- * i.e. all the upstream operators have only one child.
- */
- public static void removeBranch(SparkPartitionPruningSinkOperator op) {
- Operator<?> child = op;
- Operator<?> curr = op;
-
- while (curr.getChildOperators().size() <= 1) {
- child = curr;
- if (curr.getParentOperators() == null || curr.getParentOperators().isEmpty()) {
- return;
- }
- curr = curr.getParentOperators().get(0);
- }
-
- curr.removeChild(child);
- }
-
/**
* Remove operator from the tree, disconnecting it from its
* parents and children.
@@ -421,20 +386,6 @@ public class OperatorUtils {
return op.toString();
}
- /**
- * Return true if contain branch otherwise return false
- */
- public static boolean isInBranch(SparkPartitionPruningSinkOperator op) {
- Operator<?> curr = op;
- while (curr.getChildOperators().size() <= 1) {
- if (curr.getParentOperators() == null || curr.getParentOperators().isEmpty()) {
- return false;
- }
- curr = curr.getParentOperators().get(0);
- }
- return true;
- }
-
public static Set<Operator<?>> getOp(BaseWork work, Class<?> clazz) {
Set<Operator<?>> ops = new HashSet<Operator<?>>();
if (work instanceof MapWork) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ParsedOutputFileName.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ParsedOutputFileName.java
index 4fc827eaa12..713c7e56f84 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ParsedOutputFileName.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ParsedOutputFileName.java
@@ -36,9 +36,6 @@ import java.util.regex.Pattern;
* <p>
* All the components are here:
* tmp_(taskPrefix)00001_02_copy_1.zlib.gz
- *
- * Spark output file:
- * part-00026-23003837-facb-49ec-b1c4-eeda902cacf3.c000.zlib.orc
*/
public class ParsedOutputFileName {
private static final Pattern COPY_FILE_NAME_TO_TASK_ID_REGEX = Pattern.compile(
@@ -49,9 +46,6 @@ public class ParsedOutputFileName {
"(?:_copy_([0-9]{1,6}))?" + // copy file index
"(\\..*)?$"); // any suffix/file extension
- private static final Pattern SPARK_FILE_NAME =
- Pattern.compile("^part-(\\d+)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\.[a-f0-9]+(.*)$");
-
public static ParsedOutputFileName parse(String fileName) {
return new ParsedOutputFileName(fileName);
}
@@ -65,34 +59,22 @@ public class ParsedOutputFileName {
private final CharSequence filePrefixForCopy;
private ParsedOutputFileName(CharSequence fileName) {
- Matcher m = SPARK_FILE_NAME.matcher(fileName);
- if (m.matches()) {
- matches = true;
+ Matcher m = COPY_FILE_NAME_TO_TASK_ID_REGEX.matcher(fileName);
+ matches = m.matches();
+ if (matches) {
+ taskIdPrefix = m.group(2);
+ taskId = m.group(3);
+ attemptId = m.group(4);
+ copyIndex = m.group(5);
+ suffix = m.group(6);
+ filePrefixForCopy = m.end(4) >= 0 ? fileName.subSequence(0, m.end(4)) : null;
+ } else {
taskIdPrefix = null;
- taskId = m.group(1);
- attemptId = "1";
+ taskId = null;
+ attemptId = null;
copyIndex = null;
- String s = m.group(2);
- suffix = (s == null || s.isEmpty() ? null : s);
+ suffix = null;
filePrefixForCopy = null;
- } else {
- m = COPY_FILE_NAME_TO_TASK_ID_REGEX.matcher(fileName);
- matches = m.matches();
- if (matches) {
- taskIdPrefix = m.group(2);
- taskId = m.group(3);
- attemptId = m.group(4);
- copyIndex = m.group(5);
- suffix = m.group(6);
- filePrefixForCopy = m.end(4) >= 0 ? fileName.subSequence(0, m.end(4)) : null;
- } else {
- taskIdPrefix = null;
- taskId = null;
- attemptId = null;
- copyIndex = null;
- suffix = null;
- filePrefixForCopy = null;
- }
}
}
@@ -144,7 +126,7 @@ public class ParsedOutputFileName {
*/
public String makeFilenameWithCopyIndex(int idx) throws HiveException {
if (filePrefixForCopy == null) {
- throw new HiveException("Not expected to make copy files of spark output files.");
+ throw new HiveException("Not expected format for copying files.");
}
return filePrefixForCopy + "_copy_" + idx;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
index f2fb39f8382..af22e5e95e8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
@@ -53,10 +53,6 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.Shell;
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkEnv;
-import org.apache.spark.SparkFiles;
/**
* ScriptOperator.
@@ -318,7 +314,6 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
// initialize the user's process only when you receive the first row
if (firstRow) {
firstRow = false;
- SparkConf sparkConf = null;
try {
String[] cmdArgs = splitArgs(conf.getScriptCmd());
@@ -328,12 +323,6 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
if (!new File(prog).isAbsolute()) {
PathFinder finder = new PathFinder("PATH");
finder.prependPathComponent(currentDir.toString());
-
- // In spark local mode, we need to search added files in root directory.
- if (HiveConf.getVar(hconf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("spark")) {
- sparkConf = SparkEnv.get().conf();
- finder.prependPathComponent(SparkFiles.getRootDirectory());
- }
File f = finder.getAbsolutePath(prog);
if (f != null) {
cmdArgs[0] = f.getAbsolutePath();
@@ -360,17 +349,6 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
String idEnvVarVal = getOperatorId();
env.put(safeEnvVarName(idEnvVarName), idEnvVarVal);
- // For spark, in non-local mode, any added dependencies are stored at
- // SparkFiles::getRootDirectory, which is the executor's working directory.
- // In local mode, we need to manually point the process's working directory to it,
- // in order to make the dependencies accessible.
- if (sparkConf != null) {
- String master = sparkConf.get("spark.master");
- if (master.equals("local") || master.startsWith("local[")) {
- pb.directory(new File(SparkFiles.getRootDirectory()));
- }
- }
-
scriptPid = pb.start(); // Runtime.getRuntime().exec(wrappedCmdArgs);
DataOutputStream scriptOut = new DataOutputStream(
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
deleted file mode 100644
index 3b04e2bb7c9..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec;
-
-import java.io.BufferedOutputStream;
-import java.io.ObjectOutputStream;
-import java.io.OutputStream;
-import java.io.Serializable;
-import java.util.Set;
-import java.util.concurrent.ThreadLocalRandom;
-
-import org.apache.commons.io.FileExistsException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinPersistableTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
-import org.apache.hadoop.hive.ql.log.PerfLogger;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.BucketMapJoinContext;
-import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
-import org.apache.hadoop.hive.ql.plan.SparkBucketMapJoinContext;
-import org.apache.hadoop.hive.ql.plan.SparkHashTableSinkDesc;
-import org.apache.hadoop.hive.ql.plan.api.OperatorType;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SparkHashTableSinkOperator
- extends TerminalOperator<SparkHashTableSinkDesc> implements Serializable {
- private static final long serialVersionUID = 1L;
- private final String CLASS_NAME = this.getClass().getName();
- private final transient PerfLogger perfLogger = SessionState.getPerfLogger();
- protected static final Logger LOG = LoggerFactory.getLogger(SparkHashTableSinkOperator.class.getName());
- private static final String MAPRED_FILE_REPLICATION = "mapreduce.client.submit.file.replication";
- private static final int DEFAULT_REPLICATION = 10;
-
- private final HashTableSinkOperator htsOperator;
-
- private short numReplication;
-
- /** Kryo ctor. */
- protected SparkHashTableSinkOperator() {
- super();
- htsOperator = null; // Kryo will set this; or so we hope.
- }
-
- public SparkHashTableSinkOperator(CompilationOpContext ctx) {
- super(ctx);
- htsOperator = new HashTableSinkOperator(ctx);
- }
-
- @Override
- protected void initializeOp(Configuration hconf) throws HiveException {
- super.initializeOp(hconf);
- ObjectInspector[] inputOIs = new ObjectInspector[conf.getTagLength()];
- byte tag = conf.getTag();
- inputOIs[tag] = inputObjInspectors[0];
- conf.setTagOrder(new Byte[]{ tag });
- numReplication = (short) hconf.getInt(MAPRED_FILE_REPLICATION, DEFAULT_REPLICATION);
- htsOperator.setConf(conf);
- htsOperator.initialize(hconf, inputOIs);
- }
-
- @Override
- public void process(Object row, int tag) throws HiveException {
- // Ignore the tag passed in, which should be 0, not what we want
- htsOperator.process(row, conf.getTag());
- }
-
- @Override
- public void closeOp(boolean abort) throws HiveException {
- try {
- MapJoinPersistableTableContainer[] mapJoinTables = htsOperator.mapJoinTables;
- byte tag = conf.getTag();
- if (mapJoinTables == null || mapJoinTables.length < tag
- || mapJoinTables[tag] == null) {
- LOG.debug("mapJoinTable is null");
- } else if (abort) {
- LOG.debug("Aborting, skip dumping side-table for tag: {}", tag);
- } else {
- String method = PerfLogger.SPARK_FLUSH_HASHTABLE + getName();
- perfLogger.perfLogBegin(CLASS_NAME, method);
- try {
- flushToFile(mapJoinTables[tag], tag);
- } finally {
- perfLogger.perfLogEnd(CLASS_NAME, method);
- }
- }
- super.closeOp(abort);
- } catch (HiveException e) {
- throw e;
- } catch (Exception e) {
- throw new HiveException(e);
- }
- }
-
- protected void flushToFile(MapJoinPersistableTableContainer tableContainer,
- byte tag) throws Exception {
- MapredLocalWork localWork = getExecContext().getLocalWork();
- BucketMapJoinContext mapJoinCtx = localWork.getBucketMapjoinContext();
- Path inputPath = getExecContext().getCurrentInputPath();
- String bigInputPath = null;
- if (inputPath != null && mapJoinCtx != null) {
- Set<String> aliases =
- ((SparkBucketMapJoinContext)mapJoinCtx).getPosToAliasMap().get((int)tag);
- bigInputPath = mapJoinCtx.getMappingBigFile(
- aliases.iterator().next(), inputPath.toString());
- }
-
- // get tmp file URI
- Path tmpURI = localWork.getTmpHDFSPath();
- LOG.info("Temp URI for side table: " + tmpURI);
- // get current bucket file name
- String fileName = localWork.getBucketFileName(bigInputPath);
- // get the tmp URI path; it will be a hdfs path if not local mode
- String dumpFilePrefix = conf.getDumpFilePrefix();
- Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName);
- FileSystem fs = path.getFileSystem(htsOperator.getConfiguration());
-
- fs.mkdirs(path); // Create the folder and its parents if not there
- while (true) {
- path = new Path(path, getOperatorId()
- + "-" + Math.abs(ThreadLocalRandom.current().nextInt()));
- try {
- // This will guarantee file name uniqueness.
- if (fs.createNewFile(path)) {
- break;
- }
- } catch (FileExistsException e) {
- // No problem, use a new name
- }
- }
-
- htsOperator.console.printInfo(Utilities.now() + "\tDump the side-table for tag: " + tag
- + " with group count: " + tableContainer.size() + " into file: " + path);
- try {
- // get the hashtable file and path
- OutputStream os = null;
- ObjectOutputStream out = null;
- MapJoinTableContainerSerDe mapJoinTableSerde = htsOperator.mapJoinTableSerdes[tag];
- try {
- os = fs.create(path, numReplication);
- out = new ObjectOutputStream(new BufferedOutputStream(os, 4096));
- mapJoinTableSerde.persist(out, tableContainer);
- } finally {
- if (out != null) {
- out.close();
- } else if (os != null) {
- os.close();
- }
- }
-
- FileStatus status = fs.getFileStatus(path);
- htsOperator.console.printInfo(Utilities.now() + "\tUploaded 1 File to: " + path
- + " (" + status.getLen() + " bytes)");
- } catch (Exception e) {
- // Failed to dump the side-table, remove the partial file
- try {
- fs.delete(path, false);
- } catch (Exception ex) {
- LOG.warn("Got exception in deleting partial side-table dump for tag: "
- + tag + ", file " + path, ex);
- }
- throw e;
- }
- tableContainer.clear();
- }
-
- /**
- * Implements the getName function for the Node Interface.
- *
- * @return the name of the operator
- */
- @Override
- public String getName() {
- return SparkHashTableSinkOperator.getOperatorName();
- }
-
- public static String getOperatorName() {
- return HashTableSinkOperator.getOperatorName();
- }
-
- @Override
- public OperatorType getType() {
- return OperatorType.HASHTABLESINK;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkMemoryExhaustionChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkMemoryExhaustionChecker.java
deleted file mode 100644
index b1b3e093c3b..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkMemoryExhaustionChecker.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.GcTimeMonitor;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.mapjoin.MapJoinMemoryExhaustionError;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A {@link MemoryExhaustionChecker} specific to Hive-on-Spark. Unlike the
- * {@link DefaultMemoryExhaustionChecker} it uses a {@link GcTimeMonitor}
- * to monitor how much time (what percentage of run time within the last
- * minute or so) is spent in GC. If this value exceeds the configured value
- * in {@link HiveConf.ConfVars#HIVEHASHTABLEMAXGCTIMEPERCENTAGE}, a
- * {@link MapJoinMemoryExhaustionError} is thrown.
- */
-class SparkMemoryExhaustionChecker implements MemoryExhaustionChecker {
-
- private static final Logger LOG = LoggerFactory.getLogger(SparkMemoryExhaustionChecker.class);
-
- private static SparkMemoryExhaustionChecker INSTANCE;
-
- // The GC time alert functionality below is used by the checkGcOverhead() method.
- // This method may be called very frequently, and if it called
- // GcTimeMonitor.getLatestGcData() every time, it could result in unnecessary
- // overhead due to synchronization and new object creation. So instead,
- // GcTimeMonitor itself sets the "red flag" in lastAlertGcTimePercentage,
- // and checkGcOverhead() may check it as frequently as needed.
- private volatile int lastAlertGcTimePercentage;
- private final int criticalGcTimePercentage;
-
- private SparkMemoryExhaustionChecker(Configuration conf) {
- super();
- criticalGcTimePercentage = (int) (HiveConf.getFloatVar(
- conf, HiveConf.ConfVars.HIVEHASHTABLEMAXGCTIMEPERCENTAGE) * 100);
- GcTimeMonitor hiveGcTimeMonitor = new HiveGcTimeMonitor(criticalGcTimePercentage);
- hiveGcTimeMonitor.start();
- }
-
- static synchronized SparkMemoryExhaustionChecker get(Configuration conf) {
- if (INSTANCE == null) {
- INSTANCE = new SparkMemoryExhaustionChecker(conf);
- }
- return INSTANCE;
- }
-
- @Override
- public void checkMemoryOverhead(long rowNumber, long hashTableScale, int tableContainerSize) {
- if (lastAlertGcTimePercentage >= criticalGcTimePercentage) {
- String msg = "GC time percentage = " + lastAlertGcTimePercentage + "% exceeded threshold "
- + criticalGcTimePercentage + "%";
- throw new MapJoinMemoryExhaustionError(msg);
- }
- }
-
- // GC time monitoring
- private class HiveGcTimeMonitor extends GcTimeMonitor {
-
- HiveGcTimeMonitor(int criticalGcTimePercentage) {
- super(45 * 1000, 200, criticalGcTimePercentage, new HiveGcTimeAlertHandler());
- }
- }
-
- private class HiveGcTimeAlertHandler implements GcTimeMonitor.GcTimeAlertHandler {
-
- @Override
- public void alert(GcTimeMonitor.GcData gcData) {
- lastAlertGcTimePercentage = gcData.getGcTimePercentage();
- LOG.warn("GcTimeMonitor alert called. Current GC time = " + lastAlertGcTimePercentage + "%");
- }
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
index acea877d6f4..625300fb234 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
@@ -48,7 +48,6 @@ import org.apache.hadoop.hive.ql.exec.repl.RangerDumpTask;
import org.apache.hadoop.hive.ql.exec.repl.RangerDenyTask;
import org.apache.hadoop.hive.ql.exec.repl.RangerDenyWork;
import org.apache.hadoop.hive.ql.exec.schq.ScheduledQueryMaintenanceTask;
-import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.io.merge.MergeFileTask;
import org.apache.hadoop.hive.ql.io.merge.MergeFileWork;
@@ -65,7 +64,6 @@ import org.apache.hadoop.hive.ql.plan.MapredWork;
import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.ReplCopyWork;
import org.apache.hadoop.hive.ql.plan.ReplTxnWork;
-import org.apache.hadoop.hive.ql.plan.SparkWork;
import org.apache.hadoop.hive.ql.plan.StatsWork;
import org.apache.hadoop.hive.ql.plan.TezWork;
import org.apache.hadoop.hive.ql.scheduled.ScheduledQueryMaintenanceWork;
@@ -119,7 +117,6 @@ public final class TaskFactory {
taskvec.add(new TaskTuple<DependencyCollectionWork>(DependencyCollectionWork.class,
DependencyCollectionTask.class));
taskvec.add(new TaskTuple<TezWork>(TezWork.class, TezTask.class));
- taskvec.add(new TaskTuple<SparkWork>(SparkWork.class, SparkTask.class));
taskvec.add(new TaskTuple<>(ReplDumpWork.class, ReplDumpTask.class));
taskvec.add(new TaskTuple<>(ReplLoadWork.class, ReplLoadTask.class));
taskvec.add(new TaskTuple<>(ReplStateLogWork.class, ReplStateLogTask.class));
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index 09ff2129237..95affc4b64e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -37,7 +37,6 @@ import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
import java.net.URLDecoder;
-import java.security.AccessController;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@@ -120,7 +119,6 @@ import org.apache.hadoop.hive.ql.exec.mr.ExecDriver;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
import org.apache.hadoop.hive.ql.exec.mr.ExecReducer;
import org.apache.hadoop.hive.ql.exec.mr.MapRedTask;
-import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
import org.apache.hadoop.hive.ql.exec.tez.DagUtils;
import org.apache.hadoop.hive.ql.exec.tez.TezTask;
import org.apache.hadoop.hive.ql.exec.util.DAGTraversal;
@@ -459,19 +457,6 @@ public final class Utilities {
Kryo kryo = SerializationUtilities.borrowKryo();
try {
String engine = HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE);
- if (engine.equals("spark")) {
- // TODO Add jar into current thread context classloader as it may be invoked by Spark driver inside
- // threads, should be unnecessary while SPARK-5377 is resolved.
- String addedJars = conf.get(HIVE_ADDED_JARS);
- if (StringUtils.isNotEmpty(addedJars)) {
- AddToClassPathAction addAction = new AddToClassPathAction(
- Thread.currentThread().getContextClassLoader(), Arrays.asList(addedJars.split(";"))
- );
- ClassLoader newLoader = AccessController.doPrivileged(addAction);
- Thread.currentThread().setContextClassLoader(newLoader);
- kryo.setClassLoader(newLoader);
- }
- }
Path localPath = path;
LOG.debug("local path = {}", localPath);
final long serializedSize;
@@ -2827,16 +2812,12 @@ public final class Utilities {
return getTasks(tasks, new TaskFilterFunction<>(TezTask.class));
}
- public static List<SparkTask> getSparkTasks(List<Task<?>> tasks) {
- return getTasks(tasks, new TaskFilterFunction<>(SparkTask.class));
- }
-
public static List<ExecDriver> getMRTasks(List<Task<?>> tasks) {
return getTasks(tasks, new TaskFilterFunction<>(ExecDriver.class));
}
public static int getNumClusterJobs(List<Task<?>> tasks) {
- return getMRTasks(tasks).size() + getTezTasks(tasks).size() + getSparkTasks(tasks).size();
+ return getMRTasks(tasks).size() + getTezTasks(tasks).size();
}
static class TaskFilterFunction<T> implements DAGTraversal.Function {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
index 79c31a578a4..514a8c92fb5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinTableContainerSerDe.java
@@ -97,84 +97,6 @@ public class MapJoinTableContainerSerDe {
}
}
- /**
- * Loads the table container from a folder. Only used on Spark path.
- * @param fs FileSystem of the folder.
- * @param folder The folder to load table container.
- * @param hconf The hive configuration
- * @return Loaded table.
- */
- @SuppressWarnings("unchecked")
- public MapJoinTableContainer load(
- FileSystem fs, Path folder, Configuration hconf) throws HiveException {
- try {
-
- if (!fs.exists(folder)) {
- return getDefaultEmptyContainer(hconf, keyContext, valueContext);
- }
- if (!fs.isDirectory(folder)) {
- throw new HiveException("Error, not a directory: " + folder);
- }
- FileStatus[] fileStatuses = fs.listStatus(folder);
- if (fileStatuses == null || fileStatuses.length == 0) {
- return getDefaultEmptyContainer(hconf, keyContext, valueContext);
- }
-
- AbstractSerDe keySerDe = keyContext.getSerDe();
- AbstractSerDe valueSerDe = valueContext.getSerDe();
- Writable keyContainer = keySerDe.getSerializedClass().newInstance();
- Writable valueContainer = valueSerDe.getSerializedClass().newInstance();
-
- MapJoinTableContainer tableContainer = null;
-
- boolean useOptimizedContainer = HiveConf.getBoolVar(
- hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
-
- for (FileStatus fileStatus: fileStatuses) {
- Path filePath = fileStatus.getPath();
- if (ShimLoader.getHadoopShims().isDirectory(fileStatus)) {
- throw new HiveException("Error, not a file: " + filePath);
- }
- InputStream is = null;
- ObjectInputStream in = null;
- try {
- is = fs.open(filePath);
- in = new ObjectInputStream(is);
- String name = in.readUTF();
- Map<String, String> metaData = (Map<String, String>) in.readObject();
- if (tableContainer == null) {
- tableContainer = useOptimizedContainer ?
- new MapJoinBytesTableContainer(hconf, valueContext, -1, 0) :
- create(name, metaData);
- }
- tableContainer.setSerde(keyContext, valueContext);
- if (useOptimizedContainer) {
- loadOptimized((MapJoinBytesTableContainer) tableContainer,
- in, keyContainer, valueContainer);
- } else {
- loadNormal((MapJoinPersistableTableContainer) tableContainer,
- in, keyContainer, valueContainer);
- }
- } finally {
- if (in != null) {
- in.close();
- } else if (is != null) {
- is.close();
- }
- }
- }
- if (tableContainer != null) {
- tableContainer.setKey(folder.toString());
- tableContainer.seal();
- }
- return tableContainer;
- } catch (IOException e) {
- throw new HiveException("IO error while trying to create table container", e);
- } catch (Exception e) {
- throw new HiveException("Error while trying to create table container", e);
- }
- }
-
private void loadNormal(MapJoinPersistableTableContainer container,
ObjectInputStream in, Writable keyContainer, Writable valueContainer) throws Exception {
int numKeys = in.readInt();
@@ -203,76 +125,6 @@ public class MapJoinTableContainerSerDe {
}
}
- /**
- * Loads the small table into a VectorMapJoinFastTableContainer. Only used on Spark path.
- * @param mapJoinDesc The descriptor for the map join
- * @param fs FileSystem of the folder.
- * @param folder The folder to load table container.
- * @param hconf The hive configuration
- * @return Loaded table.
- */
- @SuppressWarnings("unchecked")
- public MapJoinTableContainer loadFastContainer(MapJoinDesc mapJoinDesc,
- FileSystem fs, Path folder, Configuration hconf) throws HiveException {
- try {
- VectorMapJoinFastTableContainer tableContainer =
- new VectorMapJoinFastTableContainer(mapJoinDesc, hconf, -1, 1);
- tableContainer.setSerde(keyContext, valueContext);
-
- if (fs.exists(folder)) {
- if (!fs.isDirectory(folder)) {
- throw new HiveException("Error, not a directory: " + folder);
- }
-
- FileStatus[] fileStatuses = fs.listStatus(folder);
- if (fileStatuses != null && fileStatuses.length > 0) {
- AbstractSerDe keySerDe = keyContext.getSerDe();
- AbstractSerDe valueSerDe = valueContext.getSerDe();
- Writable key = keySerDe.getSerializedClass().newInstance();
- Writable value = valueSerDe.getSerializedClass().newInstance();
-
- for (FileStatus fileStatus : fileStatuses) {
- Path filePath = fileStatus.getPath();
- if (ShimLoader.getHadoopShims().isDirectory(fileStatus)) {
- throw new HiveException("Error, not a file: " + filePath);
- }
- InputStream is = null;
- ObjectInputStream in = null;
- try {
- is = fs.open(filePath);
- in = new ObjectInputStream(is);
- // skip the name and metadata
- in.readUTF();
- in.readObject();
- int numKeys = in.readInt();
- for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) {
- key.readFields(in);
- long numRows = in.readLong();
- for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) {
- value.readFields(in);
- tableContainer.putRow(key, value);
- }
- }
- } finally {
- if (in != null) {
- in.close();
- } else if (is != null) {
- is.close();
- }
- }
- }
- }
- tableContainer.setKey(folder.toString());
- }
- tableContainer.seal();
- return tableContainer;
- } catch (IOException e) {
- throw new HiveException("IO error while trying to create table container", e);
- } catch (Exception e) {
- throw new HiveException("Error while trying to create table container", e);
- }
- }
-
public void persist(ObjectOutputStream out, MapJoinPersistableTableContainer tableContainer)
throws HiveException {
int numKeys = tableContainer.size();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
deleted file mode 100644
index 770a1000a16..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/CacheTran.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import org.apache.hadoop.hive.ql.plan.BaseWork;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.spark.api.java.JavaPairRDD;
-import org.apache.spark.storage.StorageLevel;
-
-public abstract class CacheTran<KI extends WritableComparable, VI, KO extends WritableComparable, VO>
- implements SparkTran<KI, VI, KO, VO> {
- // whether to cache current RDD.
- private boolean caching = false;
- private JavaPairRDD<KO, VO> cachedRDD;
- protected final String name;
- private final BaseWork baseWork;
-
- protected CacheTran(boolean cache, String name, BaseWork baseWork) {
- this.caching = cache;
- this.name = name;
- this.baseWork = baseWork;
- }
-
- @Override
- public JavaPairRDD<KO, VO> transform(
- JavaPairRDD<KI, VI> input) {
- if (caching) {
- if (cachedRDD == null) {
- cachedRDD = doTransform(input);
- cachedRDD.persist(StorageLevel.MEMORY_AND_DISK());
- }
- return cachedRDD.setName(this.name + " (" + cachedRDD.getNumPartitions() + ", cached)");
- } else {
- JavaPairRDD<KO, VO> rdd = doTransform(input);
- return rdd.setName(this.name + " (" + rdd.getNumPartitions() + ")");
- }
- }
-
- public Boolean isCacheEnable() {
- return caching;
- }
-
- protected abstract JavaPairRDD<KO, VO> doTransform(JavaPairRDD<KI, VI> input);
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public BaseWork getBaseWork() {
- return baseWork;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
deleted file mode 100644
index 4bfde25f15d..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/GroupByShuffler.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.spark.api.java.JavaPairRDD;
-
-public class GroupByShuffler implements SparkShuffler<Iterable<BytesWritable>> {
-
- @Override
- public JavaPairRDD<HiveKey, Iterable<BytesWritable>> shuffle(
- JavaPairRDD<HiveKey, BytesWritable> input, int numPartitions) {
- if (numPartitions > 0) {
- return input.groupByKey(numPartitions);
- }
- return input.groupByKey();
- }
-
- @Override
- public String getName() {
- return "GroupBy";
- }
-
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
deleted file mode 100644
index 4e6a8eed655..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.exec.HashTableSinkOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.MapredContext;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.TemporaryHashSinkOperator;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
-import org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
-import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizationOperator;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.BucketMapJoinContext;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
-import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.SparkBucketMapJoinContext;
-import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.mapred.JobConf;
-
-/**
- * HashTableLoader for Spark to load the hashtable for MapJoins.
- */
-public class HashTableLoader implements org.apache.hadoop.hive.ql.exec.HashTableLoader {
-
- private static final Logger LOG = LoggerFactory.getLogger(HashTableLoader.class.getName());
-
- private ExecMapperContext context;
- private Configuration hconf;
-
- private MapJoinOperator joinOp;
- private MapJoinDesc desc;
-
- private boolean useFastContainer = false;
-
- @Override
- public void init(ExecMapperContext context, MapredContext mrContext, Configuration hconf,
- MapJoinOperator joinOp) {
- this.context = context;
- this.hconf = hconf;
- this.joinOp = joinOp;
- this.desc = joinOp.getConf();
- if (desc.getVectorMode() && HiveConf.getBoolVar(
- hconf, HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_FAST_HASHTABLE_ENABLED)) {
- if (joinOp instanceof VectorizationOperator) {
- VectorMapJoinDesc vectorDesc = (VectorMapJoinDesc) ((VectorizationOperator) joinOp).getVectorDesc();
- useFastContainer = vectorDesc != null && vectorDesc.getHashTableImplementationType() ==
- VectorMapJoinDesc.HashTableImplementationType.FAST;
- }
- }
- }
-
- @Override
- public void load(MapJoinTableContainer[] mapJoinTables,
- MapJoinTableContainerSerDe[] mapJoinTableSerdes)
- throws HiveException {
-
- // Note: it's possible that a MJ operator is in a ReduceWork, in which case the
- // currentInputPath will be null. But, since currentInputPath is only interesting
- // for bucket join case, and for bucket join the MJ operator will always be in
- // a MapWork, this should be OK.
- String currentInputPath =
- context.getCurrentInputPath() == null ? null : context.getCurrentInputPath().toString();
-
- LOG.info("******* Load from HashTable for input file: " + currentInputPath);
- MapredLocalWork localWork = context.getLocalWork();
- try {
- if (localWork.getDirectFetchOp() != null) {
- loadDirectly(mapJoinTables, currentInputPath);
- }
- // All HashTables share the same base dir,
- // which is passed in as the tmp path
- Path baseDir = localWork.getTmpPath();
- if (baseDir == null) {
- return;
- }
- FileSystem fs = FileSystem.get(baseDir.toUri(), hconf);
- BucketMapJoinContext mapJoinCtx = localWork.getBucketMapjoinContext();
- boolean firstContainer = true;
- boolean useOptimizedContainer = !useFastContainer && HiveConf.getBoolVar(
- hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
- for (int pos = 0; pos < mapJoinTables.length; pos++) {
- if (pos == desc.getPosBigTable() || mapJoinTables[pos] != null) {
- continue;
- }
- if (useOptimizedContainer) {
- MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext();
- ObjectInspector keyOI = keyCtx.getSerDe().getObjectInspector();
- if (!MapJoinBytesTableContainer.isSupportedKey(keyOI)) {
- if (firstContainer) {
- LOG.warn("Not using optimized table container." +
- "Only a subset of mapjoin keys is supported.");
- useOptimizedContainer = false;
- HiveConf.setBoolVar(hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE, false);
- } else {
- throw new HiveException("Only a subset of mapjoin keys is supported.");
- }
- }
- }
- firstContainer = false;
- String bigInputPath = currentInputPath;
- if (currentInputPath != null && mapJoinCtx != null) {
- if (!desc.isBucketMapJoin()) {
- bigInputPath = null;
- } else {
- Set<String> aliases =
- ((SparkBucketMapJoinContext) mapJoinCtx).getPosToAliasMap().get(pos);
- String alias = aliases.iterator().next();
- // Any one small table input path
- String smallInputPath =
- mapJoinCtx.getAliasBucketFileNameMapping().get(alias).get(bigInputPath).get(0);
- bigInputPath = mapJoinCtx.getMappingBigFile(alias, smallInputPath);
- }
- }
- String fileName = localWork.getBucketFileName(bigInputPath);
- Path path = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(), (byte) pos, fileName);
- mapJoinTables[pos] = load(fs, path, mapJoinTableSerdes[pos]);
- }
- } catch (Exception e) {
- throw new HiveException(e);
- }
- }
-
- private MapJoinTableContainer load(FileSystem fs, Path path,
- MapJoinTableContainerSerDe mapJoinTableSerde) throws HiveException {
- LOG.info("\tLoad back all hashtable files from tmp folder uri:" + path);
- if (!SparkUtilities.isDedicatedCluster(hconf)) {
- return loadMapJoinTableContainer(fs, path, mapJoinTableSerde);
- }
-
- try {
- return SmallTableCache.get(path.toString(), () -> loadMapJoinTableContainer(fs, path, mapJoinTableSerde));
- } catch (ExecutionException e) {
- throw new HiveException(e);
- }
- }
-
- private MapJoinTableContainer loadMapJoinTableContainer(
- FileSystem fs, Path path, MapJoinTableContainerSerDe mapJoinTableSerde) throws HiveException {
- return useFastContainer ? mapJoinTableSerde.loadFastContainer(desc, fs, path, hconf) :
- mapJoinTableSerde.load(fs, path, hconf);
- }
-
- private void loadDirectly(MapJoinTableContainer[] mapJoinTables, String inputFileName)
- throws Exception {
- MapredLocalWork localWork = context.getLocalWork();
- List<Operator<?>> directWorks = localWork.getDirectFetchOp().get(joinOp);
- if (directWorks == null || directWorks.isEmpty()) {
- return;
- }
- JobConf job = new JobConf(hconf);
- MapredLocalTask localTask = new MapredLocalTask(localWork, job, false);
-
- HashTableSinkOperator sink = new TemporaryHashSinkOperator(new CompilationOpContext(), desc);
- sink.setParentOperators(new ArrayList<Operator<? extends OperatorDesc>>(directWorks));
-
- for (Operator<?> operator : directWorks) {
- if (operator != null) {
- operator.setChildOperators(Arrays.<Operator<? extends OperatorDesc>>asList(sink));
- }
- }
- localTask.setExecContext(context);
- localTask.startForward(inputFileName);
-
- MapJoinTableContainer[] tables = sink.getMapJoinTables();
- for (int i = 0; i < sink.getNumParent(); i++) {
- if (sink.getParentOperators().get(i) != null) {
- mapJoinTables[i] = tables[i];
- }
- }
-
- Arrays.fill(tables, null);
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveBaseFunctionResultList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveBaseFunctionResultList.java
deleted file mode 100644
index c75158b3c79..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveBaseFunctionResultList.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.mapred.OutputCollector;
-
-import scala.Tuple2;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Base class for
- * - collecting Map/Reduce function output and
- * - providing an Iterable interface for fetching output records. Input records
- * are processed in lazy fashion i.e when output records are requested
- * through Iterator interface.
- */
-@SuppressWarnings("rawtypes")
-public abstract class HiveBaseFunctionResultList<T>
- implements Iterator, OutputCollector<HiveKey, BytesWritable>, Serializable {
- private static final long serialVersionUID = -1L;
- private final Iterator<T> inputIterator;
- private boolean isClosed = false;
-
- // Contains results from last processed input record.
- private final HiveKVResultCache lastRecordOutput;
-
- public HiveBaseFunctionResultList(Iterator<T> inputIterator) {
- this.inputIterator = inputIterator;
- this.lastRecordOutput = new HiveKVResultCache();
- }
-
- @Override
- public void collect(HiveKey key, BytesWritable value) throws IOException {
- lastRecordOutput.add(SparkUtilities.copyHiveKey(key),
- SparkUtilities.copyBytesWritable(value));
- }
-
- /** Process the given record. */
- protected abstract void processNextRecord(T inputRecord) throws IOException;
-
- /**
- * @return true if current state of the record processor is done.
- */
- protected abstract boolean processingDone();
-
- /** Close the record processor. */
- protected abstract void closeRecordProcessor();
-
- @Override
- public boolean hasNext() {
- // Return remaining records (if any) from last processed input record.
- if (lastRecordOutput.hasNext()) {
- return true;
- }
-
- // Process the records in the input iterator until
- // - new output records are available for serving downstream operator,
- // - input records are exhausted or
- // - processing is completed.
- while (inputIterator.hasNext() && !processingDone()) {
- try {
- processNextRecord(inputIterator.next());
- if (lastRecordOutput.hasNext()) {
- return true;
- }
- } catch (IOException ex) {
- throw new IllegalStateException("Error while processing input.", ex);
- }
- }
-
- // At this point we are done processing the input. Close the record processor
- if (!isClosed) {
- closeRecordProcessor();
- isClosed = true;
- }
-
- // It is possible that some operators add records after closing the processor, so make sure
- // to check the lastRecordOutput
- if (lastRecordOutput.hasNext()) {
- return true;
- }
-
- lastRecordOutput.clear();
- return false;
- }
-
- @Override
- public Tuple2<HiveKey, BytesWritable> next() {
- if (hasNext()) {
- return lastRecordOutput.next();
- }
- throw new NoSuchElementException("There are no more elements");
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("Iterator.remove() is not supported");
- }
-
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java
deleted file mode 100644
index ee78bf77e33..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.commons.lang3.tuple.MutablePair;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-
-import scala.Tuple2;
-
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-/**
- * A cache with fixed buffer. If the buffer is full, new entries will
- * be written to disk. This class is thread safe since multiple threads
- * could access it (doesn't have to be concurrently), for example,
- * the StreamThread in ScriptOperator.
- */
-@SuppressWarnings("unchecked")
-class HiveKVResultCache {
- private static final Logger LOG = LoggerFactory.getLogger(HiveKVResultCache.class);
-
- @VisibleForTesting
- static final int IN_MEMORY_NUM_ROWS = 1024;
-
- private MutablePair<HiveKey, BytesWritable>[] writeBuffer;
- private MutablePair<HiveKey, BytesWritable>[] readBuffer;
-
- private File parentFile;
- private File tmpFile;
-
- private int readCursor = 0;
- private int writeCursor = 0;
-
- // Indicate if the read buffer has data, for example,
- // when in reading, data on disk could be pull in
- private boolean readBufferUsed = false;
- private int rowsInReadBuffer = 0;
-
- private Input input;
- private Output output;
-
- public HiveKVResultCache() {
- writeBuffer = new MutablePair[IN_MEMORY_NUM_ROWS];
- readBuffer = new MutablePair[IN_MEMORY_NUM_ROWS];
- for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) {
- writeBuffer[i] = new MutablePair<HiveKey, BytesWritable>();
- readBuffer[i] = new MutablePair<HiveKey, BytesWritable>();
- }
- }
-
- private void switchBufferAndResetCursor() {
- MutablePair<HiveKey, BytesWritable>[] tmp = readBuffer;
- rowsInReadBuffer = writeCursor;
- readBuffer = writeBuffer;
- readBufferUsed = true;
- readCursor = 0;
- writeBuffer = tmp;
- writeCursor = 0;
- }
-
- private void setupOutput() throws IOException {
- if (parentFile == null) {
- while (true) {
- parentFile = File.createTempFile("hive-resultcache", "");
- if (parentFile.delete() && parentFile.mkdir()) {
- parentFile.deleteOnExit();
- break;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retry creating tmp result-cache directory...");
- }
- }
- }
-
- if (tmpFile == null || input != null) {
- tmpFile = File.createTempFile("ResultCache", ".tmp", parentFile);
- LOG.info("ResultCache created temp file " + tmpFile.getAbsolutePath());
- tmpFile.deleteOnExit();
- }
-
- FileOutputStream fos = null;
- try {
- fos = new FileOutputStream(tmpFile);
- output = new Output(fos);
- } finally {
- if (output == null && fos != null) {
- fos.close();
- }
- }
- }
-
- private BytesWritable readValue(Input input) {
- return new BytesWritable(input.readBytes(input.readInt()));
- }
-
- private void writeValue(Output output, BytesWritable bytesWritable) {
- int size = bytesWritable.getLength();
- output.writeInt(size);
- output.writeBytes(bytesWritable.getBytes(), 0, size);
- }
-
- private HiveKey readHiveKey(Input input) {
- HiveKey hiveKey = new HiveKey(
- input.readBytes(input.readInt()), input.readInt());
- hiveKey.setDistKeyLength(input.readInt());
- return hiveKey;
- }
-
- private void writeHiveKey(Output output, HiveKey hiveKey) {
- int size = hiveKey.getLength();
- output.writeInt(size);
- output.writeBytes(hiveKey.getBytes(), 0, size);
- output.writeInt(hiveKey.hashCode());
- output.writeInt(hiveKey.getDistKeyLength());
- }
-
- public synchronized void add(HiveKey key, BytesWritable value) {
- if (writeCursor >= IN_MEMORY_NUM_ROWS) { // Write buffer is full
- if (!readBufferUsed) { // Read buffer isn't used, switch buffer
- switchBufferAndResetCursor();
- } else {
- // Need to spill from write buffer to disk
- try {
- if (output == null) {
- setupOutput();
- }
- for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) {
- MutablePair<HiveKey, BytesWritable> pair = writeBuffer[i];
- writeHiveKey(output, pair.getLeft());
- writeValue(output, pair.getRight());
- pair.setLeft(null);
- pair.setRight(null);
- }
- writeCursor = 0;
- } catch (Exception e) {
- clear(); // Clean up the cache
- throw new RuntimeException("Failed to spill rows to disk", e);
- }
- }
- }
- MutablePair<HiveKey, BytesWritable> pair = writeBuffer[writeCursor++];
- pair.setLeft(key);
- pair.setRight(value);
- }
-
- public synchronized void clear() {
- writeCursor = readCursor = rowsInReadBuffer = 0;
- readBufferUsed = false;
-
- if (parentFile != null) {
- if (input != null) {
- try {
- input.close();
- } catch (Throwable ignored) {
- }
- input = null;
- }
- if (output != null) {
- try {
- output.close();
- } catch (Throwable ignored) {
- }
- output = null;
- }
- try {
- FileUtil.fullyDelete(parentFile);
- } catch (Throwable ignored) {
- }
- parentFile = null;
- tmpFile = null;
- }
- }
-
- public synchronized boolean hasNext() {
- return readBufferUsed || writeCursor > 0;
- }
-
- public synchronized Tuple2<HiveKey, BytesWritable> next() {
- Preconditions.checkState(hasNext());
- if (!readBufferUsed) {
- try {
- if (input == null && output != null) {
- // Close output stream if open
- output.close();
- output = null;
-
- FileInputStream fis = null;
- try {
- fis = new FileInputStream(tmpFile);
- input = new Input(fis);
- } finally {
- if (input == null && fis != null) {
- fis.close();
- }
- }
- }
- if (input != null) {
- // Load next batch from disk
- for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) {
- MutablePair<HiveKey, BytesWritable> pair = readBuffer[i];
- pair.setLeft(readHiveKey(input));
- pair.setRight(readValue(input));
- }
- if (input.end()) {
- input.close();
- input = null;
- }
- rowsInReadBuffer = IN_MEMORY_NUM_ROWS;
- readBufferUsed = true;
- readCursor = 0;
- } else if (writeCursor == 1) {
- MutablePair<HiveKey, BytesWritable> pair = writeBuffer[0];
- Tuple2<HiveKey, BytesWritable> row = new Tuple2<HiveKey, BytesWritable>(pair.getLeft(), pair.getRight());
- pair.setLeft(null);
- pair.setRight(null);
- writeCursor = 0;
- return row;
- } else {
- // No record on disk, more data in write buffer
- switchBufferAndResetCursor();
- }
- } catch (Exception e) {
- clear(); // Clean up the cache
- throw new RuntimeException("Failed to load rows from disk", e);
- }
- }
- MutablePair<HiveKey, BytesWritable> pair = readBuffer[readCursor];
- Tuple2<HiveKey, BytesWritable> row = new Tuple2<HiveKey, BytesWritable>(pair.getLeft(), pair.getRight());
- pair.setLeft(null);
- pair.setRight(null);
- if (++readCursor >= rowsInReadBuffer) {
- readBufferUsed = false;
- rowsInReadBuffer = 0;
- readCursor = 0;
- }
- return row;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveMapFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveMapFunction.java
deleted file mode 100644
index 9296aaad922..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveMapFunction.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.hive.ql.io.merge.MergeFileMapper;
-import org.apache.hadoop.io.BytesWritable;
-
-import scala.Tuple2;
-
-public class HiveMapFunction extends HivePairFlatMapFunction<
- Iterator<Tuple2<BytesWritable, BytesWritable>>, HiveKey, BytesWritable> {
-
- private static final long serialVersionUID = 1L;
-
- public HiveMapFunction(byte[] jobConfBuffer, SparkReporter sparkReporter) {
- super(jobConfBuffer, sparkReporter);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public Iterator<Tuple2<HiveKey, BytesWritable>>
- call(Iterator<Tuple2<BytesWritable, BytesWritable>> it) throws Exception {
- initJobConf();
-
- SparkRecordHandler mapRecordHandler;
-
- // need different record handler for MergeFileWork
- if (MergeFileMapper.class.getName().equals(jobConf.get(Utilities.MAPRED_MAPPER_CLASS))) {
- mapRecordHandler = new SparkMergeFileRecordHandler();
- } else {
- mapRecordHandler = new SparkMapRecordHandler();
- }
-
- HiveMapFunctionResultList result = new HiveMapFunctionResultList(it, mapRecordHandler);
- mapRecordHandler.init(jobConf, result, sparkReporter);
-
- return result;
- }
-
- @Override
- protected boolean isMap() {
- return true;
- }
-
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveMapFunctionResultList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveMapFunctionResultList.java
deleted file mode 100644
index 8ba9e18a024..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveMapFunctionResultList.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.io.BytesWritable;
-
-import scala.Tuple2;
-
-public class HiveMapFunctionResultList extends
- HiveBaseFunctionResultList<Tuple2<BytesWritable, BytesWritable>> {
- private static final long serialVersionUID = 1L;
- private final SparkRecordHandler recordHandler;
-
- /**
- * Instantiate result set Iterable for Map function output.
- *
- * @param inputIterator Input record iterator.
- * @param handler Initialized {@link SparkMapRecordHandler} instance.
- */
- public HiveMapFunctionResultList(
- Iterator<Tuple2<BytesWritable, BytesWritable>> inputIterator,
- SparkRecordHandler handler) {
- super(inputIterator);
- recordHandler = handler;
- }
-
- @Override
- protected void processNextRecord(Tuple2<BytesWritable, BytesWritable> inputRecord)
- throws IOException {
- recordHandler.processRow(inputRecord._1(), inputRecord._2());
- }
-
- @Override
- protected boolean processingDone() {
- return recordHandler.getDone();
- }
-
- @Override
- protected void closeRecordProcessor() {
- recordHandler.close();
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
deleted file mode 100644
index 7b019b9c4d9..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HivePairFlatMapFunction.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.text.NumberFormat;
-
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.spark.TaskContext;
-import org.apache.spark.api.java.function.PairFlatMapFunction;
-
-
-public abstract class HivePairFlatMapFunction<T, K, V> implements PairFlatMapFunction<T, K, V> {
- private final NumberFormat taskIdFormat = NumberFormat.getInstance();
- private final NumberFormat stageIdFormat = NumberFormat.getInstance();
- {
- taskIdFormat.setGroupingUsed(false);
- taskIdFormat.setMinimumIntegerDigits(6);
- stageIdFormat.setGroupingUsed(false);
- stageIdFormat.setMinimumIntegerDigits(4);
- }
-
- protected transient JobConf jobConf;
- protected SparkReporter sparkReporter;
-
- private byte[] buffer;
-
- public HivePairFlatMapFunction(byte[] buffer, SparkReporter sparkReporter) {
- this.buffer = buffer;
- this.sparkReporter = sparkReporter;
- }
-
- protected void initJobConf() {
- if (jobConf == null) {
- jobConf = KryoSerializer.deserializeJobConf(this.buffer);
- SmallTableCache.initialize(jobConf);
- setupMRLegacyConfigs();
- }
- }
-
- protected abstract boolean isMap();
-
- // Some Hive features depends on several MR configuration legacy, build and add
- // these configuration to JobConf here.
- private void setupMRLegacyConfigs() {
- StringBuilder taskAttemptIdBuilder = new StringBuilder("attempt_");
- taskAttemptIdBuilder.append(System.currentTimeMillis())
- .append("_")
- .append(stageIdFormat.format(TaskContext.get().stageId()))
- .append("_");
-
- if (isMap()) {
- taskAttemptIdBuilder.append("m_");
- } else {
- taskAttemptIdBuilder.append("r_");
- }
-
- // Hive requires this TaskAttemptId to be unique. MR's TaskAttemptId is composed
- // of "attempt_timestamp_jobNum_m/r_taskNum_attemptNum". The counterpart for
- // Spark should be "attempt_timestamp_stageNum_m/r_partitionId_attemptNum".
- // When there're multiple attempts for a task, Hive will rely on the partitionId
- // to figure out if the data are duplicate or not when collecting the final outputs
- // (see org.apache.hadoop.hive.ql.exec.Utils.removeTempOrDuplicateFiles)
- taskAttemptIdBuilder.append(taskIdFormat.format(TaskContext.get().partitionId()))
- .append("_").append(TaskContext.get().attemptNumber());
-
- String taskAttemptIdStr = taskAttemptIdBuilder.toString();
- jobConf.set("mapred.task.id", taskAttemptIdStr);
- jobConf.set("mapreduce.task.attempt.id", taskAttemptIdStr);
- jobConf.setInt("mapred.task.partition", TaskContext.get().partitionId());
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveReduceFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveReduceFunction.java
deleted file mode 100644
index 9c7d15a3e30..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveReduceFunction.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.util.Iterator;
-
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-
-import scala.Tuple2;
-
-public class HiveReduceFunction<V> extends HivePairFlatMapFunction<
- Iterator<Tuple2<HiveKey, V>>, HiveKey, BytesWritable> {
-
- private static final long serialVersionUID = 1L;
-
- public HiveReduceFunction(byte[] buffer, SparkReporter sparkReporter) {
- super(buffer, sparkReporter);
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public Iterator<Tuple2<HiveKey, BytesWritable>>
- call(Iterator<Tuple2<HiveKey, V>> it) throws Exception {
- initJobConf();
-
- SparkReduceRecordHandler reducerRecordhandler = new SparkReduceRecordHandler();
- HiveReduceFunctionResultList<V> result =
- new HiveReduceFunctionResultList<V>(it, reducerRecordhandler);
- reducerRecordhandler.init(jobConf, result, sparkReporter);
-
- return result;
- }
-
- @Override
- protected boolean isMap() {
- return false;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveReduceFunctionResultList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveReduceFunctionResultList.java
deleted file mode 100644
index 7cc2e9af6ed..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveReduceFunctionResultList.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-
-import scala.Tuple2;
-
-public class HiveReduceFunctionResultList<V> extends
- HiveBaseFunctionResultList<Tuple2<HiveKey, V>> {
- private static final long serialVersionUID = 1L;
- private final SparkReduceRecordHandler reduceRecordHandler;
-
- /**
- * Instantiate result set Iterable for Reduce function output.
- *
- * @param inputIterator Input record iterator.
- * @param reducer Initialized {@link org.apache.hadoop.hive.ql.exec.mr.ExecReducer} instance.
- */
- public HiveReduceFunctionResultList(
- Iterator<Tuple2<HiveKey, V>> inputIterator,
- SparkReduceRecordHandler reducer) {
- super(inputIterator);
- this.reduceRecordHandler = reducer;
- }
-
- @Override
- protected void processNextRecord(Tuple2<HiveKey, V> inputRecord)
- throws IOException {
- HiveKey key = inputRecord._1();
- V value = inputRecord._2();
- if (value instanceof Iterable) {
- @SuppressWarnings("unchecked")
- Iterable<BytesWritable> values = (Iterable<BytesWritable>)value;
- reduceRecordHandler.<BytesWritable>processRow(key, values.iterator());
- } else {
- reduceRecordHandler.processRow(key, value);
- }
- }
-
- @Override
- protected boolean processingDone() {
- return false;
- }
-
- @Override
- protected void closeRecordProcessor() {
- reduceRecordHandler.close();
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClient.java
deleted file mode 100644
index 454e407e481..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClient.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.io.Closeable;
-import java.io.Serializable;
-
-import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.TaskQueue;
-import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef;
-import org.apache.hadoop.hive.ql.plan.SparkWork;
-import org.apache.spark.SparkConf;
-
-public interface HiveSparkClient extends Serializable, Closeable {
- /**
- * HiveSparkClient should generate Spark RDD graph by given sparkWork and taskQueue,
- * and submit RDD graph to Spark cluster.
- * @param taskQueue
- * @param sparkWork
- * @return SparkJobRef could be used to track spark job progress and metrics.
- * @throws Exception
- */
- SparkJobRef execute(TaskQueue taskQueue, Context context, SparkWork sparkWork) throws Exception;
-
- /**
- * @return spark configuration
- */
- SparkConf getSparkConf();
-
- /**
- * @return the number of executors
- */
- int getExecutorCount() throws Exception;
-
- /**
- * For standalone mode, this can be used to get total number of cores.
- * @return default parallelism.
- */
- int getDefaultParallelism() throws Exception;
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
deleted file mode 100644
index 75b378d10fb..00000000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveSparkClientFactory.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.spark;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.compress.utils.CharsetNames;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.hive.common.LogUtils;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hive.spark.client.SparkClientUtilities;
-import org.apache.hive.spark.client.rpc.RpcConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConfUtil;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.io.HiveKey;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.spark.SparkConf;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Sets;
-
-public class HiveSparkClientFactory {
- protected static final transient Logger LOG = LoggerFactory.getLogger(HiveSparkClientFactory.class);
-
- private static final String SPARK_DEFAULT_CONF_FILE = "spark-defaults.conf";
- private static final String SPARK_DEFAULT_MASTER = "yarn";
- private static final String SPARK_DEFAULT_DEPLOY_MODE = "cluster";
- private static final String SPARK_DEFAULT_APP_NAME = "Hive on Spark";
- private static final String SPARK_DEFAULT_SERIALIZER = "org.apache.spark.serializer.KryoSerializer";
- private static final String SPARK_DEFAULT_REFERENCE_TRACKING = "false";
- private static final String SPARK_WAIT_APP_COMPLETE = "spark.yarn.submit.waitAppCompletion";
- private static final String SPARK_DEPLOY_MODE = "spark.submit.deployMode";
- @VisibleForTesting
- public static final String SPARK_CLONE_CONFIGURATION = "spark.hadoop.cloneConf";
-
- public static HiveSparkClient createHiveSparkClient(HiveConf hiveconf, String sparkSessionId,
- String hiveSessionId) throws Exception {
- Map<String, String> sparkConf = initiateSparkConf(hiveconf, hiveSessionId);
-
- // Submit spark job through local spark context while spark master is local mode, otherwise submit
- // spark job through remote spark context.
- String master = sparkConf.get("spark.master");
- if (master.equals("local") || master.startsWith("local[")) {
- // With local spark context, all user sessions share the same spark context.
- return LocalHiveSparkClient.getInstance(generateSparkConf(sparkConf), hiveconf);
- } else {
- return new RemoteHiveSparkClient(hiveconf, sparkConf, sparkSessionId);
- }
- }
-
- public static Map<String, String> initiateSparkConf(HiveConf hiveConf, String hiveSessionId) {
- Map<String, String> sparkConf = new HashMap<String, String>();
- try {
- HBaseConfiguration.addHbaseResources(hiveConf);
- } catch (Exception e) {
- LOG.warn("Could not add HBase resources to Configuration", e);
- }
-
- // set default spark configurations.
- sparkConf.put("spark.master", SPARK_DEFAULT_MASTER);
- final String appNameKey = "spark.app.name";
- String appName = hiveConf.get(appNameKey);
- final String sessionIdString = " (hiveSessionId = " + hiveSessionId + ")";
- if (appName == null) {
- if (hiveSessionId == null) {
- appName = SPARK_DEFAULT_APP_NAME;
- } else {
- appName = SPARK_DEFAULT_APP_NAME + sessionIdString;
- }
- } else {
- appName = appName + sessionIdString;
- }
- sparkConf.put(appNameKey, appName);
- sparkConf.put("spark.serializer", SPARK_DEFAULT_SERIALIZER);
- sparkConf.put("spark.kryo.referenceTracking", SPARK_DEFAULT_REFERENCE_TRACKING);
-
- // load properties from spark-defaults.conf.
- InputStream inputStream = null;
- try {
- inputStream = HiveSparkClientFactory.class.getClassLoader()
- .getResourceAsStream(SPARK_DEFAULT_CONF_FILE);
- if (inputStream != null) {
- LOG.info("Loading Spark properties from: " + SPARK_DEFAULT_CONF_FILE);
- Properties properties = new Properties();
- properties.load(new InputStreamReader(inputStream, CharsetNames.UTF_8));
- for (String propertyName : properties.stringPropertyNames()) {
- if (propertyName.startsWith("spark")) {
- String value = properties.getProperty(propertyName);
- sparkConf.put(propertyName, properties.getProperty(propertyName));
- LOG.debug(String.format(
- "load spark property from %s (%s -> %s).",
- SPARK_DEFAULT_CONF_FILE, propertyName, LogUtils.maskIfPassword(propertyName,value)));
- }
- }
- }
- } catch (IOException e) {
- LOG.info("Failed to open Spark configuration file: "
- + SPARK_DEFAULT_CONF_FILE, e);
- } finally {
- if (inputStream != null) {
- try {
- inputStream.close();
- } catch (IOException e) {
- LOG.debug("Failed to close inputstream.", e);
- }
- }
- }
-
- // load properties from hive configurations, including both spark.* properties,
- // properties for remote driver RPC, and yarn properties for Spark on YARN mode.
- String sparkMaster = hiveConf.get("spark.master");
- if (sparkMaster == null) {
- sparkMaster = sparkConf.get("spark.master");
- hiveConf.set("spark.master", sparkMaster);
- }
- String deployMode = null;
- if (!SparkClientUtilities.isLocalMaster(sparkMaster)) {
- deployMode = hiveConf.get(SPARK_DEPLOY_MODE);
- if (deployMode == null) {
- deployMode = sparkConf.get(SPARK_DEPLOY_MODE);
- if (deployMode == null) {
- deployMode = SparkClientUtilities.getDeployModeFromMaster(sparkMaster);
- }
- if (deployMode == null) {
- deployMode = SPARK_DEFAULT_DEPLOY_MODE;
- }
- hiveConf.set(SPARK_DEPLOY_MODE, deployMode);
- }
- }
- if (SessionState.get() != null && SessionState.get().getConf() != null) {
- SessionState.get().getConf().set("spark.master", sparkMaster);
- if (deployMode != null) {
- SessionState.get().getConf().set(SPARK_DEPLOY_MODE, deployMode);
- }
- }
- if (SparkClientUtilities.isYarnClusterMode(sparkMaster, deployMode)) {
- sparkConf.put("spark.yarn.maxAppAttempts", "1");
- }
- for (Map.Entry<String, String> entry : hiveConf) {
- String propertyName = entry.getKey();
- if (propertyName.startsWith("spark")) {
- String value = hiveConf.get(propertyName);
- sparkConf.put(propertyName, value);
- LOG.debug(String.format(
- "load spark property from hive configuration (%s -> %s).",
- propertyName, LogUtils.maskIfPassword(propertyName,value)));
- } else if (propertyName.startsWith("yarn") &&
- SparkClientUtilities.isYarnMaster(sparkMaster)) {
- String value = hiveConf.get(propertyName);
- // Add spark.hadoop prefix for yarn properties as SparkConf only accept properties
- // started with spark prefix, Spark would remove spark.hadoop prefix lately and add
- // it to its hadoop configuration.
- sparkConf.put("spark.hadoop." + propertyName, value);
- LOG.debug(String.format(
- "load yarn property from hive configuration in %s mode (%s -> %s).",
- sparkMaster, propertyName, LogUtils.maskIfPassword(propertyName,value)));
- } else if (propertyName.equals(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)) {
- String value = hiveConf.get(propertyName);
- if (value != null && !value.isEmpty()) {
- sparkConf.put("spark.hadoop." + propertyName, value);
- }
- } else if (propertyName.startsWith("hbase") || propertyName.startsWith("zookeeper.znode")) {
- // Add HBase related configuration to Spark because in security mode, Spark needs it
- // to generate hbase delegation token for Spark. This is a temp solution to deal with
- // Spark problem.
- String value = hiveConf.get(propertyName);
- sparkConf.put("spark.hadoop." + propertyName, value);
- LOG.debug(String.format(
- "load HBase configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value)));
- } else if (propertyName.startsWith("oozie")) {
- String value = hiveConf.get(propertyName);
- sparkConf.put("spark." + propertyName, value);
- LOG.debug(String.format(
- "Pass Oozie configuration (%s -> %s).", propertyName, LogUtils.maskIfPassword(propertyName,value)));
- }
- if (RpcConfiguration.HIVE_SPARK_RSC_CONFIGS.contains(propertyName)) {
- String value = RpcConfiguration.getValue(hiveConf, propertyName);
- sparkConf.put(propertyName, value);
- LOG.debug(String.format("load RPC property from hive configuration (%s -> %s).", propertyName,
- LogUtils.maskIfPassword(propertyName, value)));
- }
- }
-
- final boolean optShuffleSerDe = hiveConf.getBoolVar(
- HiveConf.ConfVars.SPARK_OPTIMIZE_SHUFFLE_SERDE);
-
- Set<String> classes = Sets.newHashSet(
- Splitter.on(",").trimResults().omitEmptyStrings().split(
- Strings.nullToEmpty(sparkConf.get("spark.kryo.classesToRegister"))));
- classes.add(Writable.class.getName());
- classes.add(VectorizedRowBatch.class.getName());
- if (!optShuffleSerDe) {
- classes.add(HiveKey.class.getName());
- classes.add(BytesWritable.class.getName());
- } else {
- sparkConf.put("spark.kryo.registrator", SparkClientUtilities.HIVE_KRYO_REG_NAME);
- }
- sparkConf.put("spark.kryo.classesToRegister", Joiner.on(",").join(classes));
-
- // set yarn queue name
- final String sparkQueueNameKey = "spark.yarn.queue";
- if (SparkClientUtilities.isYarnMaster(sparkMaster) && hiveConf.get(sparkQueueNameKey) == null) {
- String queueName = hiveConf.get("mapreduce.job.queuename");
- if (queueName != null) {
- sparkConf.put(sparkQueueNameKey, queueName);
- }
- }
-
- // Disable it to avoid verbose app state report in yarn-cluster mode
- if (SparkClientUtilities.isYarnClusterMode(sparkMaster, deployMode) &&
- sparkConf.get(SPARK_WAIT_APP_COMPLETE) == null) {
- sparkConf.put(SPARK_WAIT_APP_COMPLETE, "false");
- }
-
- // Force Spark configs to be cloned by default
- sparkConf.putIfAbsent(SPARK_CLONE_CONFIGURATION, "true");
-
-
- // Set the credential provider passwords if found, if there is job specific password
- // the credential provider location is set directly in the execute method of LocalSparkClient
- // and submit method of RemoteHiveSparkClient when the job config is created
- String password = HiveConfUtil.getJobCredentialProviderPassword(hiveConf);
- if(password != null) {
- addCredentialProviderPassword(sparkConf, password);
- }
- return sparkConf;
- }
-
- private static void addCredentialProviderPassword(Map<String, String> sparkConf,
- String jobCredstorePassword) {
- sparkConf.put("spark.yarn.appMasterEnv.HADOOP_CREDSTORE_PASSWORD", jobCredstorePassword);
- sparkConf.put("spark.executorEnv.HADOOP_CREDSTORE_PASSWORD", jobCredstorePassword);
- }
-
- static SparkConf generateSparkConf(Map<String, String> conf) {
- SparkConf sparkConf = new SparkConf(false);
- for (Map.Entry<String, String> entry : conf.entrySet()) {
- sparkConf.set(entry.getKey(), entry.getValue());
- }
- return sparkConf;
- }
-}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveVoidFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveVoidFunction.java
... 627293 lines suppressed ...