You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/08/02 05:59:34 UTC
[49/50] hive git commit: HIVE-9152 - Dynamic Partition Pruning [Spark
Branch] (Chao Sun, reviewed by Xuefu Zhang and Chengxiang Li)
HIVE-9152 - Dynamic Partition Pruning [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang and Chengxiang Li)
Conflicts:
ql/src/test/results/clientpositive/spark/vectorized_case.q.out
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d7413e4b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d7413e4b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d7413e4b
Branch: refs/heads/branch-1
Commit: d7413e4ba379dab90abf63a38e1d1edb18862c66
Parents: 4bedc3a
Author: Chao Sun <su...@apache.org>
Authored: Fri Jul 17 12:51:27 2015 -0700
Committer: xzhang <xz...@xzdt>
Committed: Sat Aug 1 20:56:49 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 7 +
itests/qtest/pom.xml | 2 +-
.../test/resources/testconfiguration.properties | 6 +
.../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 4708 ++++-
.../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 7807 +++----
.../thrift/gen-cpp/hive_metastore_constants.cpp | 2 +-
.../thrift/gen-cpp/hive_metastore_constants.h | 2 +-
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 6044 +++++-
.../gen/thrift/gen-cpp/hive_metastore_types.h | 2619 ++-
.../hive/metastore/api/AbortTxnRequest.java | 24 +-
.../metastore/api/AddDynamicPartitions.java | 69 +-
.../metastore/api/AddPartitionsRequest.java | 80 +-
.../hive/metastore/api/AddPartitionsResult.java | 48 +-
.../hadoop/hive/metastore/api/AggrStats.java | 54 +-
.../metastore/api/AlreadyExistsException.java | 24 +-
.../metastore/api/BinaryColumnStatsData.java | 40 +-
.../metastore/api/BooleanColumnStatsData.java | 40 +-
.../hive/metastore/api/CheckLockRequest.java | 24 +-
.../hive/metastore/api/ColumnStatistics.java | 54 +-
.../metastore/api/ColumnStatisticsData.java | 20 +-
.../metastore/api/ColumnStatisticsDesc.java | 58 +-
.../hive/metastore/api/ColumnStatisticsObj.java | 40 +-
.../hive/metastore/api/CommitTxnRequest.java | 24 +-
.../hive/metastore/api/CompactionRequest.java | 62 +-
.../hive/metastore/api/CompactionType.java | 2 +-
.../api/ConfigValSecurityException.java | 24 +-
.../api/CurrentNotificationEventId.java | 24 +-
.../hadoop/hive/metastore/api/Database.java | 115 +-
.../apache/hadoop/hive/metastore/api/Date.java | 24 +-
.../hive/metastore/api/DateColumnStatsData.java | 50 +-
.../hadoop/hive/metastore/api/Decimal.java | 41 +-
.../metastore/api/DecimalColumnStatsData.java | 50 +-
.../metastore/api/DoubleColumnStatsData.java | 50 +-
.../hive/metastore/api/DropPartitionsExpr.java | 43 +-
.../metastore/api/DropPartitionsRequest.java | 82 +-
.../metastore/api/DropPartitionsResult.java | 48 +-
.../hive/metastore/api/EnvironmentContext.java | 61 +-
.../hive/metastore/api/EventRequestType.java | 2 +-
.../hadoop/hive/metastore/api/FieldSchema.java | 58 +-
.../hive/metastore/api/FireEventRequest.java | 79 +-
.../metastore/api/FireEventRequestData.java | 20 +-
.../hive/metastore/api/FireEventResponse.java | 16 +-
.../hadoop/hive/metastore/api/Function.java | 110 +-
.../hadoop/hive/metastore/api/FunctionType.java | 2 +-
.../metastore/api/GetOpenTxnsInfoResponse.java | 54 +-
.../hive/metastore/api/GetOpenTxnsResponse.java | 53 +-
.../api/GetPrincipalsInRoleRequest.java | 24 +-
.../api/GetPrincipalsInRoleResponse.java | 46 +-
.../api/GetRoleGrantsForPrincipalRequest.java | 36 +-
.../api/GetRoleGrantsForPrincipalResponse.java | 46 +-
.../api/GrantRevokePrivilegeRequest.java | 46 +-
.../api/GrantRevokePrivilegeResponse.java | 26 +-
.../metastore/api/GrantRevokeRoleRequest.java | 86 +-
.../metastore/api/GrantRevokeRoleResponse.java | 26 +-
.../hive/metastore/api/GrantRevokeType.java | 2 +-
.../hive/metastore/api/HeartbeatRequest.java | 34 +-
.../metastore/api/HeartbeatTxnRangeRequest.java | 32 +-
.../api/HeartbeatTxnRangeResponse.java | 74 +-
.../hive/metastore/api/HiveObjectPrivilege.java | 52 +-
.../hive/metastore/api/HiveObjectRef.java | 81 +-
.../hive/metastore/api/HiveObjectType.java | 2 +-
.../apache/hadoop/hive/metastore/api/Index.java | 133 +-
.../api/IndexAlreadyExistsException.java | 24 +-
.../metastore/api/InsertEventRequestData.java | 45 +-
.../metastore/api/InvalidInputException.java | 24 +-
.../metastore/api/InvalidObjectException.java | 24 +-
.../api/InvalidOperationException.java | 24 +-
.../api/InvalidPartitionException.java | 24 +-
.../hive/metastore/api/LockComponent.java | 66 +-
.../hadoop/hive/metastore/api/LockLevel.java | 2 +-
.../hadoop/hive/metastore/api/LockRequest.java | 72 +-
.../hadoop/hive/metastore/api/LockResponse.java | 36 +-
.../hadoop/hive/metastore/api/LockState.java | 2 +-
.../hadoop/hive/metastore/api/LockType.java | 2 +-
.../hive/metastore/api/LongColumnStatsData.java | 50 +-
.../hive/metastore/api/MetaException.java | 24 +-
.../hive/metastore/api/NoSuchLockException.java | 24 +-
.../metastore/api/NoSuchObjectException.java | 24 +-
.../hive/metastore/api/NoSuchTxnException.java | 24 +-
.../hive/metastore/api/NotificationEvent.java | 66 +-
.../metastore/api/NotificationEventRequest.java | 34 +-
.../api/NotificationEventResponse.java | 46 +-
.../hive/metastore/api/OpenTxnRequest.java | 40 +-
.../hive/metastore/api/OpenTxnsResponse.java | 45 +-
.../apache/hadoop/hive/metastore/api/Order.java | 32 +-
.../hadoop/hive/metastore/api/Partition.java | 156 +-
.../hive/metastore/api/PartitionEventType.java | 2 +-
.../api/PartitionListComposingSpec.java | 46 +-
.../hive/metastore/api/PartitionSpec.java | 58 +-
.../api/PartitionSpecWithSharedSD.java | 54 +-
.../hive/metastore/api/PartitionWithoutSD.java | 124 +-
.../metastore/api/PartitionsByExprRequest.java | 67 +-
.../metastore/api/PartitionsByExprResult.java | 54 +-
.../metastore/api/PartitionsStatsRequest.java | 90 +-
.../metastore/api/PartitionsStatsResult.java | 72 +-
.../metastore/api/PrincipalPrivilegeSet.java | 184 +-
.../hive/metastore/api/PrincipalType.java | 2 +-
.../hadoop/hive/metastore/api/PrivilegeBag.java | 46 +-
.../hive/metastore/api/PrivilegeGrantInfo.java | 60 +-
.../hive/metastore/api/RequestPartsSpec.java | 56 +-
.../hadoop/hive/metastore/api/ResourceType.java | 2 +-
.../hadoop/hive/metastore/api/ResourceUri.java | 36 +-
.../apache/hadoop/hive/metastore/api/Role.java | 40 +-
.../hive/metastore/api/RolePrincipalGrant.java | 80 +-
.../hadoop/hive/metastore/api/Schema.java | 91 +-
.../hadoop/hive/metastore/api/SerDeInfo.java | 93 +-
.../api/SetPartitionsStatsRequest.java | 46 +-
.../hive/metastore/api/ShowCompactRequest.java | 16 +-
.../hive/metastore/api/ShowCompactResponse.java | 46 +-
.../api/ShowCompactResponseElement.java | 86 +-
.../hive/metastore/api/ShowLocksRequest.java | 16 +-
.../hive/metastore/api/ShowLocksResponse.java | 46 +-
.../metastore/api/ShowLocksResponseElement.java | 114 +-
.../hadoop/hive/metastore/api/SkewedInfo.java | 147 +-
.../hive/metastore/api/StorageDescriptor.java | 242 +-
.../metastore/api/StringColumnStatsData.java | 48 +-
.../apache/hadoop/hive/metastore/api/Table.java | 189 +-
.../hive/metastore/api/TableStatsRequest.java | 61 +-
.../hive/metastore/api/TableStatsResult.java | 46 +-
.../hive/metastore/api/ThriftHiveMetastore.java | 18497 ++++++++++++-----
.../hive/metastore/api/TxnAbortedException.java | 24 +-
.../hadoop/hive/metastore/api/TxnInfo.java | 52 +-
.../hive/metastore/api/TxnOpenException.java | 24 +-
.../hadoop/hive/metastore/api/TxnState.java | 2 +-
.../apache/hadoop/hive/metastore/api/Type.java | 72 +-
.../hive/metastore/api/UnknownDBException.java | 24 +-
.../api/UnknownPartitionException.java | 24 +-
.../metastore/api/UnknownTableException.java | 24 +-
.../hive/metastore/api/UnlockRequest.java | 24 +-
.../hadoop/hive/metastore/api/Version.java | 32 +-
.../metastore/api/hive_metastoreConstants.java | 7 +-
.../gen-php/metastore/ThriftHiveMetastore.php | 3328 ++-
.../src/gen/thrift/gen-php/metastore/Types.php | 1081 +-
.../hive_metastore/ThriftHiveMetastore-remote | 609 +-
.../hive_metastore/ThriftHiveMetastore.py | 3726 +++-
.../thrift/gen-py/hive_metastore/constants.py | 2 +-
.../gen/thrift/gen-py/hive_metastore/ttypes.py | 782 +-
.../thrift/gen-rb/hive_metastore_constants.rb | 2 +-
.../gen/thrift/gen-rb/hive_metastore_types.rb | 2 +-
.../gen/thrift/gen-rb/thrift_hive_metastore.rb | 2 +-
ql/if/queryplan.thrift | 1 +
.../gen/thrift/gen-cpp/queryplan_constants.cpp | 2 +-
ql/src/gen/thrift/gen-cpp/queryplan_constants.h | 2 +-
ql/src/gen/thrift/gen-cpp/queryplan_types.cpp | 796 +-
ql/src/gen/thrift/gen-cpp/queryplan_types.h | 294 +-
.../hadoop/hive/ql/plan/api/Adjacency.java | 65 +-
.../hadoop/hive/ql/plan/api/AdjacencyType.java | 2 +-
.../apache/hadoop/hive/ql/plan/api/Graph.java | 87 +-
.../hadoop/hive/ql/plan/api/NodeType.java | 2 +-
.../hadoop/hive/ql/plan/api/Operator.java | 142 +-
.../hadoop/hive/ql/plan/api/OperatorType.java | 7 +-
.../apache/hadoop/hive/ql/plan/api/Query.java | 176 +-
.../hadoop/hive/ql/plan/api/QueryPlan.java | 62 +-
.../apache/hadoop/hive/ql/plan/api/Stage.java | 172 +-
.../hadoop/hive/ql/plan/api/StageType.java | 2 +-
.../apache/hadoop/hive/ql/plan/api/Task.java | 182 +-
.../hadoop/hive/ql/plan/api/TaskType.java | 2 +-
ql/src/gen/thrift/gen-php/Types.php | 119 +-
ql/src/gen/thrift/gen-py/queryplan/constants.py | 2 +-
ql/src/gen/thrift/gen-py/queryplan/ttypes.py | 87 +-
ql/src/gen/thrift/gen-rb/queryplan_constants.rb | 2 +-
ql/src/gen/thrift/gen-rb/queryplan_types.rb | 7 +-
.../hadoop/hive/ql/exec/OperatorFactory.java | 8 +
.../ql/exec/spark/HiveSparkClientFactory.java | 5 +-
.../exec/spark/SparkDynamicPartitionPruner.java | 268 +
.../hive/ql/exec/spark/SparkUtilities.java | 56 +
...VectorSparkPartitionPruningSinkOperator.java | 99 +
.../hive/ql/io/CombineHiveInputFormat.java | 59 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 46 +-
.../DynamicPartitionPruningOptimization.java | 44 +-
.../hadoop/hive/ql/optimizer/Optimizer.java | 2 +-
.../SparkRemoveDynamicPruningBySize.java | 73 +
.../hive/ql/optimizer/physical/Vectorizer.java | 2 +
.../spark/SparkPartitionPruningSinkDesc.java | 100 +
.../ql/parse/spark/GenSparkProcContext.java | 14 +-
.../hive/ql/parse/spark/GenSparkUtils.java | 111 +-
.../parse/spark/OptimizeSparkProcContext.java | 16 +-
.../hive/ql/parse/spark/SparkCompiler.java | 177 +-
.../SparkPartitionPruningSinkOperator.java | 142 +
.../hive/ql/parse/spark/SplitOpTreeForDPP.java | 151 +
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 10 +
.../hadoop/hive/ql/plan/TableScanDesc.java | 6 +-
.../hive/ql/ppd/SyntheticJoinPredicate.java | 14 +-
.../spark_dynamic_partition_pruning.q | 180 +
.../spark_dynamic_partition_pruning_2.q | 118 +
...spark_vectorized_dynamic_partition_pruning.q | 192 +
.../results/clientpositive/spark/bucket2.q.out | 3 -
.../results/clientpositive/spark/bucket3.q.out | 3 -
.../results/clientpositive/spark/bucket4.q.out | 3 -
.../spark/column_access_stats.q.out | 4 -
.../spark/limit_partition_metadataonly.q.out | 2 -
.../spark/list_bucket_dml_2.q.java1.7.out | 3 -
.../spark/optimize_nullscan.q.out | 3 -
.../test/results/clientpositive/spark/pcr.q.out | 6 -
.../results/clientpositive/spark/sample3.q.out | 3 -
.../results/clientpositive/spark/sample9.q.out | 3 -
.../clientpositive/spark/smb_mapjoin_11.q.out | 6 -
.../spark/spark_dynamic_partition_pruning.q.out | 5573 +++++
.../spark_dynamic_partition_pruning_2.q.out | 1015 +
...k_vectorized_dynamic_partition_pruning.q.out | 5822 ++++++
.../clientpositive/spark/udf_example_add.q.out | 3 -
.../clientpositive/spark/udf_in_file.q.out | 3 -
.../clientpositive/spark/union_view.q.out | 9 -
.../clientpositive/spark/vector_elt.q.out | 7 -
.../spark/vector_string_concat.q.out | 3 -
.../spark/vectorization_decimal_date.q.out | 4 -
.../spark/vectorization_div0.q.out | 3 -
.../clientpositive/spark/vectorized_case.q.out | 3 -
.../spark/vectorized_math_funcs.q.out | 3 -
.../spark/vectorized_string_funcs.q.out | 3 -
.../gen/thrift/gen-cpp/complex_constants.cpp | 2 +-
.../src/gen/thrift/gen-cpp/complex_constants.h | 2 +-
serde/src/gen/thrift/gen-cpp/complex_types.cpp | 442 +-
serde/src/gen/thrift/gen-cpp/complex_types.h | 174 +-
.../gen/thrift/gen-cpp/megastruct_constants.cpp | 2 +-
.../gen/thrift/gen-cpp/megastruct_constants.h | 2 +-
.../src/gen/thrift/gen-cpp/megastruct_types.cpp | 585 +-
serde/src/gen/thrift/gen-cpp/megastruct_types.h | 175 +-
.../src/gen/thrift/gen-cpp/serde_constants.cpp | 2 +-
serde/src/gen/thrift/gen-cpp/serde_constants.h | 2 +-
serde/src/gen/thrift/gen-cpp/serde_types.cpp | 5 +-
serde/src/gen/thrift/gen-cpp/serde_types.h | 5 +-
.../gen/thrift/gen-cpp/testthrift_constants.cpp | 2 +-
.../gen/thrift/gen-cpp/testthrift_constants.h | 2 +-
.../src/gen/thrift/gen-cpp/testthrift_types.cpp | 95 +-
serde/src/gen/thrift/gen-cpp/testthrift_types.h | 45 +-
.../hadoop/hive/serde/serdeConstants.java | 7 +-
.../hadoop/hive/serde/test/InnerStruct.java | 24 +-
.../hadoop/hive/serde/test/ThriftTestObj.java | 62 +-
.../hadoop/hive/serde2/thrift/test/Complex.java | 279 +-
.../hive/serde2/thrift/test/IntString.java | 40 +-
.../hive/serde2/thrift/test/MegaStruct.java | 521 +-
.../hive/serde2/thrift/test/MiniStruct.java | 38 +-
.../hadoop/hive/serde2/thrift/test/MyEnum.java | 2 +-
.../hive/serde2/thrift/test/PropValueUnion.java | 60 +-
.../hive/serde2/thrift/test/SetIntString.java | 54 +-
serde/src/gen/thrift/gen-php/Types.php | 15 +-
.../org/apache/hadoop/hive/serde/Types.php | 373 +-
.../src/gen/thrift/gen-py/complex/constants.py | 2 +-
serde/src/gen/thrift/gen-py/complex/ttypes.py | 50 +-
.../gen/thrift/gen-py/megastruct/constants.py | 2 +-
.../src/gen/thrift/gen-py/megastruct/ttypes.py | 44 +-
.../org_apache_hadoop_hive_serde/constants.py | 2 +-
.../org_apache_hadoop_hive_serde/ttypes.py | 2 +-
.../gen/thrift/gen-py/testthrift/constants.py | 2 +-
.../src/gen/thrift/gen-py/testthrift/ttypes.py | 14 +-
.../src/gen/thrift/gen-rb/complex_constants.rb | 2 +-
serde/src/gen/thrift/gen-rb/complex_types.rb | 2 +-
.../gen/thrift/gen-rb/megastruct_constants.rb | 2 +-
serde/src/gen/thrift/gen-rb/megastruct_types.rb | 2 +-
serde/src/gen/thrift/gen-rb/serde_constants.rb | 2 +-
serde/src/gen/thrift/gen-rb/serde_types.rb | 2 +-
.../gen/thrift/gen-rb/testthrift_constants.rb | 2 +-
serde/src/gen/thrift/gen-rb/testthrift_types.rb | 2 +-
service/src/gen/thrift/gen-cpp/TCLIService.cpp | 458 +-
service/src/gen/thrift/gen-cpp/TCLIService.h | 821 +-
.../thrift/gen-cpp/TCLIService_constants.cpp | 2 +-
.../gen/thrift/gen-cpp/TCLIService_constants.h | 2 +-
.../gen/thrift/gen-cpp/TCLIService_types.cpp | 3250 ++-
.../src/gen/thrift/gen-cpp/TCLIService_types.h | 1482 +-
service/src/gen/thrift/gen-cpp/ThriftHive.cpp | 286 +-
service/src/gen/thrift/gen-cpp/ThriftHive.h | 389 +-
.../thrift/gen-cpp/hive_service_constants.cpp | 2 +-
.../gen/thrift/gen-cpp/hive_service_constants.h | 2 +-
.../gen/thrift/gen-cpp/hive_service_types.cpp | 110 +-
.../src/gen/thrift/gen-cpp/hive_service_types.h | 75 +-
.../hadoop/hive/service/HiveClusterStatus.java | 68 +-
.../hive/service/HiveServerException.java | 40 +-
.../hadoop/hive/service/JobTrackerState.java | 2 +-
.../apache/hadoop/hive/service/ThriftHive.java | 914 +-
.../service/cli/thrift/TArrayTypeEntry.java | 24 +-
.../hive/service/cli/thrift/TBinaryColumn.java | 64 +-
.../hive/service/cli/thrift/TBoolColumn.java | 62 +-
.../hive/service/cli/thrift/TBoolValue.java | 26 +-
.../hive/service/cli/thrift/TByteColumn.java | 62 +-
.../hive/service/cli/thrift/TByteValue.java | 26 +-
.../hive/service/cli/thrift/TCLIService.java | 1734 +-
.../cli/thrift/TCLIServiceConstants.java | 7 +-
.../cli/thrift/TCancelDelegationTokenReq.java | 32 +-
.../cli/thrift/TCancelDelegationTokenResp.java | 24 +-
.../service/cli/thrift/TCancelOperationReq.java | 24 +-
.../cli/thrift/TCancelOperationResp.java | 24 +-
.../service/cli/thrift/TCloseOperationReq.java | 24 +-
.../service/cli/thrift/TCloseOperationResp.java | 24 +-
.../service/cli/thrift/TCloseSessionReq.java | 24 +-
.../service/cli/thrift/TCloseSessionResp.java | 24 +-
.../apache/hive/service/cli/thrift/TColumn.java | 20 +-
.../hive/service/cli/thrift/TColumnDesc.java | 50 +-
.../hive/service/cli/thrift/TColumnValue.java | 20 +-
.../hive/service/cli/thrift/TDoubleColumn.java | 62 +-
.../hive/service/cli/thrift/TDoubleValue.java | 26 +-
.../cli/thrift/TExecuteStatementReq.java | 87 +-
.../cli/thrift/TExecuteStatementResp.java | 34 +-
.../service/cli/thrift/TFetchOrientation.java | 2 +-
.../service/cli/thrift/TFetchResultsReq.java | 54 +-
.../service/cli/thrift/TFetchResultsResp.java | 42 +-
.../service/cli/thrift/TGetCatalogsReq.java | 24 +-
.../service/cli/thrift/TGetCatalogsResp.java | 34 +-
.../hive/service/cli/thrift/TGetColumnsReq.java | 58 +-
.../service/cli/thrift/TGetColumnsResp.java | 34 +-
.../cli/thrift/TGetDelegationTokenReq.java | 40 +-
.../cli/thrift/TGetDelegationTokenResp.java | 34 +-
.../service/cli/thrift/TGetFunctionsReq.java | 50 +-
.../service/cli/thrift/TGetFunctionsResp.java | 34 +-
.../hive/service/cli/thrift/TGetInfoReq.java | 36 +-
.../hive/service/cli/thrift/TGetInfoResp.java | 32 +-
.../hive/service/cli/thrift/TGetInfoType.java | 2 +-
.../hive/service/cli/thrift/TGetInfoValue.java | 20 +-
.../cli/thrift/TGetOperationStatusReq.java | 24 +-
.../cli/thrift/TGetOperationStatusResp.java | 62 +-
.../cli/thrift/TGetResultSetMetadataReq.java | 24 +-
.../cli/thrift/TGetResultSetMetadataResp.java | 34 +-
.../hive/service/cli/thrift/TGetSchemasReq.java | 42 +-
.../service/cli/thrift/TGetSchemasResp.java | 34 +-
.../service/cli/thrift/TGetTableTypesReq.java | 24 +-
.../service/cli/thrift/TGetTableTypesResp.java | 34 +-
.../hive/service/cli/thrift/TGetTablesReq.java | 79 +-
.../hive/service/cli/thrift/TGetTablesResp.java | 34 +-
.../service/cli/thrift/TGetTypeInfoReq.java | 24 +-
.../service/cli/thrift/TGetTypeInfoResp.java | 34 +-
.../service/cli/thrift/THandleIdentifier.java | 50 +-
.../hive/service/cli/thrift/TI16Column.java | 62 +-
.../hive/service/cli/thrift/TI16Value.java | 26 +-
.../hive/service/cli/thrift/TI32Column.java | 62 +-
.../hive/service/cli/thrift/TI32Value.java | 26 +-
.../hive/service/cli/thrift/TI64Column.java | 62 +-
.../hive/service/cli/thrift/TI64Value.java | 26 +-
.../hive/service/cli/thrift/TMapTypeEntry.java | 32 +-
.../service/cli/thrift/TOpenSessionReq.java | 91 +-
.../service/cli/thrift/TOpenSessionResp.java | 91 +-
.../service/cli/thrift/TOperationHandle.java | 54 +-
.../service/cli/thrift/TOperationState.java | 2 +-
.../hive/service/cli/thrift/TOperationType.java | 2 +-
.../service/cli/thrift/TPrimitiveTypeEntry.java | 38 +-
.../service/cli/thrift/TProtocolVersion.java | 2 +-
.../cli/thrift/TRenewDelegationTokenReq.java | 32 +-
.../cli/thrift/TRenewDelegationTokenResp.java | 24 +-
.../apache/hive/service/cli/thrift/TRow.java | 46 +-
.../apache/hive/service/cli/thrift/TRowSet.java | 86 +-
.../hive/service/cli/thrift/TSessionHandle.java | 24 +-
.../apache/hive/service/cli/thrift/TStatus.java | 83 +-
.../hive/service/cli/thrift/TStatusCode.java | 2 +-
.../hive/service/cli/thrift/TStringColumn.java | 62 +-
.../hive/service/cli/thrift/TStringValue.java | 26 +-
.../service/cli/thrift/TStructTypeEntry.java | 50 +-
.../hive/service/cli/thrift/TTableSchema.java | 46 +-
.../hive/service/cli/thrift/TTypeDesc.java | 46 +-
.../hive/service/cli/thrift/TTypeEntry.java | 20 +-
.../apache/hive/service/cli/thrift/TTypeId.java | 2 +-
.../service/cli/thrift/TTypeQualifierValue.java | 20 +-
.../service/cli/thrift/TTypeQualifiers.java | 54 +-
.../service/cli/thrift/TUnionTypeEntry.java | 50 +-
.../cli/thrift/TUserDefinedTypeEntry.java | 24 +-
service/src/gen/thrift/gen-php/TCLIService.php | 269 +-
service/src/gen/thrift/gen-php/ThriftHive.php | 125 +-
service/src/gen/thrift/gen-php/Types.php | 30 +-
.../gen-py/TCLIService/TCLIService-remote | 102 +-
.../thrift/gen-py/TCLIService/TCLIService.py | 439 +-
.../gen/thrift/gen-py/TCLIService/constants.py | 2 +-
.../src/gen/thrift/gen-py/TCLIService/ttypes.py | 478 +-
.../gen-py/hive_service/ThriftHive-remote | 1014 +-
.../thrift/gen-py/hive_service/ThriftHive.py | 266 +-
.../gen/thrift/gen-py/hive_service/constants.py | 2 +-
.../gen/thrift/gen-py/hive_service/ttypes.py | 19 +-
.../gen/thrift/gen-rb/hive_service_constants.rb | 2 +-
.../src/gen/thrift/gen-rb/hive_service_types.rb | 2 +-
.../src/gen/thrift/gen-rb/t_c_l_i_service.rb | 2 +-
.../thrift/gen-rb/t_c_l_i_service_constants.rb | 2 +-
.../gen/thrift/gen-rb/t_c_l_i_service_types.rb | 2 +-
service/src/gen/thrift/gen-rb/thrift_hive.rb | 2 +-
370 files changed, 67652 insertions(+), 22750 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d7413e4b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bb16f85..54e9cbb 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2211,6 +2211,13 @@ public class HiveConf extends Configuration {
"Channel logging level for remote Spark driver. One of {DEBUG, ERROR, INFO, TRACE, WARN}."),
SPARK_RPC_SASL_MECHANISM("hive.spark.client.rpc.sasl.mechanisms", "DIGEST-MD5",
"Name of the SASL mechanism to use for authentication."),
+ SPARK_DYNAMIC_PARTITION_PRUNING(
+ "hive.spark.dynamic.partition.pruning", false,
+ "When dynamic pruning is enabled, joins on partition keys will be processed by writing\n" +
+ "to a temporary HDFS file, and read later for removing unnecessary partitions."),
+ SPARK_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE(
+ "hive.spark.dynamic.partition.pruning.max.data.size", 100*1024*1024L,
+ "Maximum total data size in dynamic pruning."),
NWAYJOINREORDER("hive.reorder.nway.joins", true,
"Runs reordering of tables within single n-way join (i.e.: picks streamtable)"),
HIVE_LOG_N_RECORDS("hive.log.every.n.records", 0L, new RangeValidator(0L, null),
http://git-wip-us.apache.org/repos/asf/hive/blob/d7413e4b/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 1a60e8b..187b96e 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -429,7 +429,7 @@
templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm"
queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
queryFile="${qfile}"
- excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files}"
+ excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files}"
queryFileRegex="${qfile_regex}"
clusterMode="${clustermode}"
runDisabled="${run_disabled}"
http://git-wip-us.apache.org/repos/asf/hive/blob/d7413e4b/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 97c1df9..59a215b 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1097,6 +1097,12 @@ spark.query.files=add_part_multiple.q, \
vectorized_timestamp_funcs.q, \
windowing.q
+# Unlike "spark.query.files" above, these tests only run
+# under Spark engine.
+spark.only.query.files=spark_dynamic_partition_pruning.q,\
+ spark_dynamic_partition_pruning_2.q,\
+ spark_vectorized_dynamic_partition_pruning.q
+
miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\
bucket4.q,\
bucket5.q,\